From b336104bb0500221957a7d6e68a63cdd05b2abda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Tue, 25 Aug 2020 09:54:12 +0200
Subject: [PATCH 001/122] Split block announce processing into two parts

This pull requests splits the block announce processing into two parts.
Into a phase that is called pre-validation that will be async and call
the block announce validator and into a second phase that processes the
result of the pre-validation.

The important change here is that the pre-validation phase is async.
This will be required by Cumulus/parachains. When a parachain announces
a block, it adds the candidate message send by the relay chain as extra
data into the block announcement. To verify this candidate message, the
relay chain parent is required that was used when building this message.
Now it can happen that we first receive the block announcement before
fully importing the relay chain block and this leads to the parachain
block not being imported. By making the pre-validation async, we will be
able to wait for the relay chain block to be imported to verify the
candidate message.
---
 client/network/src/protocol.rs                |  93 +++++++---
 client/network/src/protocol/sync.rs           | 174 ++++++++++++++----
 client/network/test/src/sync.rs               |  53 ++++--
 .../consensus/common/src/block_validation.rs  |  17 +-
 4 files changed, 257 insertions(+), 80 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index ff95d8f12fc78..6b4dbe604d8c6 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -256,6 +256,10 @@ pub struct Protocol<B: BlockT, H: ExHashT> {
 	metrics: Option<Metrics>,
 	/// The `PeerId`'s of all boot nodes.
 	boot_node_ids: Arc<HashSet<PeerId>>,
+	/// All the block announcement pre-validations that are currently active.
+	block_announce_pre_validation: FuturesUnordered<
+		Box<dyn Future<Output = sync::PreValidateBlockAnnounce<B::Header>> + Send + Unpin>
+	>,
 }
 
 #[derive(Default)]
@@ -466,6 +470,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 				None
 			},
 			boot_node_ids,
+			block_announce_pre_validation: FuturesUnordered::new(),
 		};
 
 		Ok((protocol, peerset_handle))
@@ -554,7 +559,9 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 	pub fn update_chain(&mut self) {
 		let info = self.context_data.chain.info();
 		self.sync.update_chain_info(&info.best_hash, info.best_number);
-		self.behaviour.set_legacy_handshake_message(build_status_message(&self.config, &self.context_data.chain));
+		self.behaviour.set_legacy_handshake_message(
+			build_status_message(&self.config, &self.context_data.chain),
+		);
 		self.behaviour.set_notif_protocol_handshake(
 			&self.block_announces_protocol,
 			BlockAnnouncesHandshake::build(&self.config, &self.context_data.chain).encode()
@@ -585,11 +592,16 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		who: PeerId,
 		data: BytesMut,
 	) -> CustomMessageOutcome<B> {
-
 		let message = match <Message<B> as Decode>::decode(&mut &data[..]) {
 			Ok(message) => message,
 			Err(err) => {
-				debug!(target: "sync", "Couldn't decode packet sent by {}: {:?}: {}", who, data, err.what());
+				debug!(
+					target: "sync",
+					"Couldn't decode packet sent by {}: {:?}: {}",
+					who,
+					data,
+					err.what(),
+				);
 				self.peerset_handle.report_peer(who, rep::BAD_MESSAGE);
 				return CustomMessageOutcome::None;
 			}
@@ -609,9 +621,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 				return outcome
 			},
 			GenericMessage::BlockAnnounce(announce) => {
-				let outcome = self.on_block_announce(who.clone(), announce);
-				self.update_peer_info(&who);
-				return outcome;
+				self.pre_validate_block_announce(who.clone(), announce);
 			},
 			GenericMessage::Transactions(m) =>
 				self.on_transactions(who, m),
@@ -1300,42 +1310,62 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		}
 	}
 
-	fn on_block_announce(
+	/// Pre-validate the given block announcement.
+	///
+	/// The pre-validation is an async operation that will be run asyncly
+	/// until it is finished. After finishing the pre-validation it will
+	/// call the [`Protocol::process_block_announce_pre_validation`].
+	fn pre_validate_block_announce(
 		&mut self,
 		who: PeerId,
 		announce: BlockAnnounce<B::Header>,
-	) -> CustomMessageOutcome<B> {
+	) {
 		let hash = announce.header.hash();
-		let number = *announce.header.number();
 
 		if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) {
 			peer.known_blocks.insert(hash.clone());
 		}
 
-		let is_their_best = match announce.state.unwrap_or(message::BlockState::Best) {
+		let is_best = match announce.state.unwrap_or(message::BlockState::Best) {
 			message::BlockState::Best => true,
 			message::BlockState::Normal => false,
 		};
 
-		match self.sync.on_block_announce(&who, hash, &announce, is_their_best) {
-			sync::OnBlockAnnounce::Nothing => {
+		let future = self.sync.pre_validate_block_announce(who, &hash, announce, is_best);
+		self.block_announce_pre_validation.push(Box::new(future));
+	}
+
+	/// Process the result of the pre-validation of a block announcement.
+	fn process_block_announce_pre_validation(
+		&mut self,
+		pre_validation_result: sync::PreValidateBlockAnnounce<B::Header>,
+	) -> CustomMessageOutcome<B> {
+		let (header, is_best, who) = match self.sync.process_block_announce_pre_validation(pre_validation_result) {
+			sync::BlockAnnounceResult::Nothing { is_best, who, header } => {
+				self.update_peer_info(&who);
+
 				// `on_block_announce` returns `OnBlockAnnounce::ImportHeader`
 				// when we have all data required to import the block
 				// in the BlockAnnounce message. This is only when:
 				// 1) we're on light client;
 				// AND
 				// 2) parent block is already imported and not pruned.
-				if is_their_best {
-					return CustomMessageOutcome::PeerNewBest(who, number);
+				if is_best {
+					return CustomMessageOutcome::PeerNewBest(who, *header.number())
 				} else {
-					return CustomMessageOutcome::None;
+					return CustomMessageOutcome::None
 				}
 			}
-			sync::OnBlockAnnounce::ImportHeader => () // We proceed with the import.
-		}
+			sync::BlockAnnounceResult::ImportHeader { header, is_best, who } => {
+				self.update_peer_info(&who);
+				(header, is_best, who)
+			}
+		};
+
+		let number = *header.number();
 
-		// to import header from announced block let's construct response to request that normally would have
-		// been sent over network (but it is not in our case)
+		// to import header from announced block let's construct response to request that normally
+		// would have been sent over network (but it is not in our case)
 		let blocks_to_import = self.sync.on_block_data(
 			&who,
 			None,
@@ -1343,8 +1373,8 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 				id: 0,
 				blocks: vec![
 					message::generic::BlockData {
-						hash: hash,
-						header: Some(announce.header),
+						hash: header.hash(),
+						header: Some(header),
 						body: None,
 						receipt: None,
 						message_queue: None,
@@ -1354,8 +1384,10 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			},
 		);
 
-		if is_their_best {
-			self.pending_messages.push_back(CustomMessageOutcome::PeerNewBest(who, number));
+		if is_best {
+			self.pending_messages.push_back(
+				CustomMessageOutcome::PeerNewBest(who, number),
+			);
 		}
 
 		match blocks_to_import {
@@ -1786,13 +1818,12 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 					}
 					Some(Fallback::BlockAnnounce) => {
 						if let Ok(announce) = message::BlockAnnounce::decode(&mut message.as_ref()) {
-							let outcome = self.on_block_announce(peer_id.clone(), announce);
-							self.update_peer_info(&peer_id);
-							outcome
+							self.pre_validate_block_announce(peer_id, announce);
 						} else {
 							warn!(target: "sub-libp2p", "Failed to decode block announce");
-							CustomMessageOutcome::None
 						}
+
+						CustomMessageOutcome::None
 					}
 					None => {
 						error!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name);
@@ -1802,6 +1833,14 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 		};
 
 		if let CustomMessageOutcome::None = outcome {
+			// Check if any pre-validations are ready to be processed.
+			while let Poll::Ready(Some(result)) = self.block_announce_pre_validation.poll_next_unpin(cx) {
+				match self.process_block_announce_pre_validation(result) {
+					CustomMessageOutcome::None => continue,
+					outcome => return Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome))
+				}
+			}
+
 			Poll::Pending
 		} else {
 			Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome))
diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index bfd8c4fe218de..83055a5ea0d98 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -50,6 +50,7 @@ use sp_runtime::{
 };
 use sp_arithmetic::traits::Saturating;
 use std::{fmt, ops::Range, collections::{HashMap, HashSet, VecDeque}, sync::Arc};
+use futures::future::{Either as FEither, Future, ready, FutureExt};
 
 mod blocks;
 mod extra_requests;
@@ -306,13 +307,51 @@ pub enum OnBlockData<B: BlockT> {
 	Request(PeerId, BlockRequest<B>)
 }
 
-/// Result of [`ChainSync::on_block_announce`].
+/// Result of [`ChainSync::process_block_announce_pre_validation`].
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum OnBlockAnnounce {
+pub enum BlockAnnounceResult<H> {
 	/// The announcement does not require further handling.
-	Nothing,
+	Nothing {
+		/// Who sent the processed block announcement?
+		who: PeerId,
+		/// Was this their new best block?
+		is_best: bool,
+		/// The header of the announcement.
+		header: H,
+	},
 	/// The announcement header should be imported.
-	ImportHeader,
+	ImportHeader {
+		/// Who sent the processed block announcement?
+		who: PeerId,
+		/// Was this their new best block?
+		is_best: bool,
+		/// The header of the announcement.
+		header: H,
+	},
+}
+
+/// Result of [`ChainSync::pre_validate_block_announce`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum PreValidateBlockAnnounce<H> {
+	/// The announcement does not require further handling.
+	Nothing {
+		/// Who sent the processed block announcement?
+		who: PeerId,
+		/// Was this their new best block?
+		is_best: bool,
+		/// The announcement.
+		announce: BlockAnnounce<H>,
+	},
+	/// The pre-validation was sucessful and the announcement should be
+	/// further processed.
+	Process {
+		/// Is this the new best block of the peer?
+		is_new_best: bool,
+		/// The id of the peer that send us the announcement.
+		who: PeerId,
+		/// The announcement.
+		announce: BlockAnnounce<H>,
+	},
 }
 
 /// Result of [`ChainSync::on_block_justification`].
@@ -1156,60 +1195,119 @@ impl<B: BlockT> ChainSync<B> {
 		self.pending_requests.set_all();
 	}
 
-	/// Call when a node announces a new block.
+	/// Pre-validate the given block announcement.
 	///
-	/// If `OnBlockAnnounce::ImportHeader` is returned, then the caller MUST try to import passed
-	/// header (call `on_block_data`). The network request isn't sent
-	/// in this case. Both hash and header is passed as an optimization
-	/// to avoid rehashing the header.
-	pub fn on_block_announce(&mut self, who: &PeerId, hash: B::Hash, announce: &BlockAnnounce<B::Header>, is_best: bool)
-		-> OnBlockAnnounce
-	{
+	/// The pre-validation will return a [`Future`] that resolves to a
+	/// [`PreValidateBlockAnnounce`]. This result needs to be feed into
+	/// [`ChainSync::process_block_announce_pre_validation`] to finish
+	/// the block announcement processing.
+	pub fn pre_validate_block_announce(
+		&mut self,
+		who: PeerId,
+		hash: &B::Hash,
+		announce: BlockAnnounce<B::Header>,
+		is_best: bool,
+	) -> FEither<
+		impl Future<Output = PreValidateBlockAnnounce<B::Header>> + Send + Unpin,
+		impl Future<Output = PreValidateBlockAnnounce<B::Header>> + Send + Unpin
+	> {
 		let header = &announce.header;
 		let number = *header.number();
-		debug!(target: "sync", "Received block announcement {:?} with number {:?} from {}", hash, number, who);
+		debug!(
+			target: "sync",
+			"Pre-validating received block announcement {:?} with number {:?} from {}",
+			hash,
+			number,
+			who,
+		);
+
 		if number.is_zero() {
-			warn!(target: "sync", "💔 Ignored genesis block (#0) announcement from {}: {}", who, hash);
-			return OnBlockAnnounce::Nothing
+			warn!(
+				target: "sync",
+				"💔 Ignored genesis block (#0) announcement from {}: {}",
+				who,
+				hash,
+			);
+			return FEither::Left(ready(PreValidateBlockAnnounce::Nothing { is_best, who, announce }))
 		}
-		let parent_status = self.block_status(header.parent_hash()).ok().unwrap_or(BlockStatus::Unknown);
+
+		// Let external validator check the block announcement.
+		let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice());
+		let future = self.block_announce_validator.validate(&header, assoc_data);
+		let hash = hash.clone();
+
+		FEither::Right(async move {
+			match future.await {
+				Ok(Validation::Success { is_new_best }) =>
+					PreValidateBlockAnnounce::Process {
+						is_new_best: is_new_best || is_best,
+						announce,
+						who,
+					},
+				Ok(Validation::Failure) => {
+					debug!(
+						target: "sync",
+						"Block announcement validation of block {} from {} failed",
+						hash,
+						who,
+					);
+					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
+				}
+				Err(e) => {
+					error!(target: "sync", "💔 Block announcement validation errored: {}", e);
+					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
+				}
+			}
+		}.boxed())
+	}
+
+	/// Needs to be called with the result of [`ChainSync::pre_validate_block_announce`].
+	///
+	/// This will finish processing of the block announcement.
+	///
+	/// If [`BlockAnnounceResult::ImportHeader`] is returned, then the caller MUST try to import passed
+	/// header (call `on_block_data`). The network request isn't sent in this case.
+	pub fn process_block_announce_pre_validation(
+		&mut self,
+		pre_validation_result: PreValidateBlockAnnounce<B::Header>,
+	) -> BlockAnnounceResult<B::Header> {
+		let (announce, is_best, who) = match pre_validation_result {
+			PreValidateBlockAnnounce::Nothing { is_best, who, announce } =>
+				return BlockAnnounceResult::Nothing { is_best, who, header: announce.header },
+			PreValidateBlockAnnounce::Process { announce, is_new_best, who } =>
+				(announce, is_new_best, who),
+		};
+
+		let header = announce.header;
+		let number = *header.number();
+		let hash = header.hash();
+		let parent_status = self.block_status(header.parent_hash()).unwrap_or(BlockStatus::Unknown);
 		let known_parent = parent_status != BlockStatus::Unknown;
 		let ancient_parent = parent_status == BlockStatus::InChainPruned;
 
 		let known = self.is_known(&hash);
-		let peer = if let Some(peer) = self.peers.get_mut(who) {
+		let peer = if let Some(peer) = self.peers.get_mut(&who) {
 			peer
 		} else {
 			error!(target: "sync", "💔 Called on_block_announce with a bad peer ID");
-			return OnBlockAnnounce::Nothing
+			return BlockAnnounceResult::Nothing { is_best, who, header }
 		};
+
 		while peer.recently_announced.len() >= ANNOUNCE_HISTORY_SIZE {
 			peer.recently_announced.pop_front();
 		}
 		peer.recently_announced.push_back(hash.clone());
 
-		// Let external validator check the block announcement.
-		let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice());
-		let is_best = match self.block_announce_validator.validate(&header, assoc_data) {
-			Ok(Validation::Success { is_new_best }) => is_new_best || is_best,
-			Ok(Validation::Failure) => {
-				debug!(target: "sync", "Block announcement validation of block {} from {} failed", hash, who);
-				return OnBlockAnnounce::Nothing
-			}
-			Err(e) => {
-				error!(target: "sync", "💔 Block announcement validation errored: {}", e);
-				return OnBlockAnnounce::Nothing
-			}
-		};
-
 		if is_best {
 			// update their best block
 			peer.best_number = number;
 			peer.best_hash = hash;
 		}
+
 		if let PeerSyncState::AncestorSearch {..} = peer.state {
-			return OnBlockAnnounce::Nothing
+			return BlockAnnounceResult::Nothing { is_best, who, header }
 		}
+
 		// If the announced block is the best they have and is not ahead of us, our common number
 		// is either one further ahead or it's the one they just announced, if we know about it.
 		if is_best {
@@ -1221,7 +1319,7 @@ impl<B: BlockT> ChainSync<B> {
 				peer.common_number = number - One::one();
 			}
 		}
-		self.pending_requests.add(who);
+		self.pending_requests.add(&who);
 
 		// known block case
 		if known || self.is_already_downloading(&hash) {
@@ -1229,18 +1327,18 @@ impl<B: BlockT> ChainSync<B> {
 			if let Some(target) = self.fork_targets.get_mut(&hash) {
 				target.peers.insert(who.clone());
 			}
-			return OnBlockAnnounce::Nothing
+			return BlockAnnounceResult::Nothing { is_best, who, header }
 		}
 
 		if ancient_parent {
 			trace!(target: "sync", "Ignored ancient block announced from {}: {} {:?}", who, hash, header);
-			return OnBlockAnnounce::Nothing
+			return BlockAnnounceResult::Nothing { is_best, who, header }
 		}
 
 		let requires_additional_data = !self.role.is_light() || !known_parent;
 		if !requires_additional_data {
 			trace!(target: "sync", "Importing new header announced from {}: {} {:?}", who, hash, header);
-			return OnBlockAnnounce::ImportHeader
+			return BlockAnnounceResult::ImportHeader { is_best, header, who }
 		}
 
 		if number <= self.best_queued_number {
@@ -1258,7 +1356,7 @@ impl<B: BlockT> ChainSync<B> {
 				.peers.insert(who.clone());
 		}
 
-		OnBlockAnnounce::Nothing
+		BlockAnnounceResult::Nothing { is_best, who, header }
 	}
 
 	/// Call when a peer has disconnected.
diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs
index 1cf2a8fee3798..6b9465a3ea52e 100644
--- a/client/network/test/src/sync.rs
+++ b/client/network/test/src/sync.rs
@@ -18,7 +18,7 @@
 
 use sp_consensus::BlockOrigin;
 use std::time::Duration;
-use futures::executor::block_on;
+use futures::{Future, executor::block_on};
 use super::*;
 use sp_consensus::block_validation::Validation;
 use substrate_test_runtime::Header;
@@ -693,14 +693,7 @@ fn can_sync_to_peers_with_wrong_common_block() {
 	let fork_hash = net.peer(0).push_blocks_at(BlockId::Number(0), 2, false);
 	net.peer(1).push_blocks_at(BlockId::Number(0), 2, false);
 	// wait for connection
-	block_on(futures::future::poll_fn::<(), _>(|cx| {
-		net.poll(cx);
-		if net.peer(0).num_peers() == 0  || net.peer(1).num_peers() == 0 {
-			Poll::Pending
-		} else {
-			Poll::Ready(())
-		}
-	}));
+	net.block_until_connected();
 
 	// both peers re-org to the same fork without notifying each other
 	net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), Some(Vec::new()), true).unwrap();
@@ -720,8 +713,8 @@ impl BlockAnnounceValidator<Block> for NewBestBlockAnnounceValidator {
 		&mut self,
 		_: &Header,
 		_: &[u8],
-	) -> Result<Validation, Box<dyn std::error::Error + Send>> {
-		Ok(Validation::Success { is_new_best: true })
+	) -> Box<dyn Future<Output = Result<Validation, Box<dyn std::error::Error + Send>>> + Unpin + Send> {
+		Box::new(async { Ok(Validation::Success { is_new_best: true }) }.boxed())
 	}
 }
 
@@ -750,3 +743,41 @@ fn sync_blocks_when_block_announce_validator_says_it_is_new_best() {
 	// that flags all blocks as `is_new_best` and thus, it should have synced the blocks.
 	assert!(!net.peer(1).has_block(&block_hash));
 }
+
+/// Waits for some time until the validation is successfull.
+struct DeferredBlockAnnounceValidator;
+
+impl BlockAnnounceValidator<Block> for DeferredBlockAnnounceValidator {
+	fn validate(
+		&mut self,
+		_: &Header,
+		_: &[u8],
+	) -> Box<dyn Future<Output = Result<Validation, Box<dyn std::error::Error + Send>>> + Unpin + Send> {
+		Box::new(
+			async {
+				futures_timer::Delay::new(std::time::Duration::from_millis(500)).await;
+				Ok(Validation::Success { is_new_best: false })
+			}.boxed()
+		)
+	}
+}
+
+#[test]
+fn wait_until_deferred_block_announce_validation_is_ready() {
+	let _ = env_logger::try_init();
+	log::trace!(target: "sync", "Test");
+	let mut net = TestNet::with_fork_choice(ForkChoiceStrategy::Custom(false));
+	net.add_full_peer_with_config(Default::default());
+	net.add_full_peer_with_config(FullPeerConfig {
+		block_announce_validator: Some(Box::new(NewBestBlockAnnounceValidator)),
+		..Default::default()
+	});
+
+	net.block_until_connected();
+
+	let block_hash = net.peer(0).push_blocks(1, true);
+
+	while !net.peer(1).has_block(&block_hash) {
+		net.block_until_idle();
+	}
+}
diff --git a/primitives/consensus/common/src/block_validation.rs b/primitives/consensus/common/src/block_validation.rs
index 66f960f16fff3..3d1d6893b1c42 100644
--- a/primitives/consensus/common/src/block_validation.rs
+++ b/primitives/consensus/common/src/block_validation.rs
@@ -18,7 +18,8 @@
 
 use crate::BlockStatus;
 use sp_runtime::{generic::BlockId, traits::Block};
-use std::{error::Error, sync::Arc};
+use std::{error::Error, sync::Arc, future::Future};
+use futures::FutureExt;
 
 /// A type which provides access to chain information.
 pub trait Chain<B: Block> {
@@ -47,7 +48,11 @@ pub enum Validation {
 /// Type which checks incoming block announcements.
 pub trait BlockAnnounceValidator<B: Block> {
 	/// Validate the announced header and its associated data.
-	fn validate(&mut self, header: &B::Header, data: &[u8]) -> Result<Validation, Box<dyn Error + Send>>;
+	fn validate(
+		&mut self,
+		header: &B::Header,
+		data: &[u8],
+	) -> Box<dyn Future<Output = Result<Validation, Box<dyn Error + Send>>> + Send + Unpin>;
 }
 
 /// Default implementation of `BlockAnnounceValidator`.
@@ -55,7 +60,11 @@ pub trait BlockAnnounceValidator<B: Block> {
 pub struct DefaultBlockAnnounceValidator;
 
 impl<B: Block> BlockAnnounceValidator<B> for DefaultBlockAnnounceValidator {
-	fn validate(&mut self, _h: &B::Header, _d: &[u8]) -> Result<Validation, Box<dyn Error + Send>> {
-		Ok(Validation::Success { is_new_best: false })
+	fn validate(
+		&mut self,
+		_: &B::Header,
+		_: &[u8],
+	) -> Box<dyn Future<Output = Result<Validation, Box<dyn Error + Send>>> + Send + Unpin> {
+		Box::new(async { Ok(Validation::Success { is_new_best: false }) }.boxed())
 	}
 }

From 95c14205df92d0577cbc8eef8e2ffbb2c437fc9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Mon, 7 Sep 2020 19:49:26 +0200
Subject: [PATCH 002/122] Apply suggestions from code review

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
---
 client/network/src/protocol.rs                      |  4 ++--
 client/network/src/protocol/sync.rs                 |  9 ++-------
 primitives/consensus/common/src/block_validation.rs | 10 +++++-----
 3 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 6b4dbe604d8c6..36010df2d997b 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -258,7 +258,7 @@ pub struct Protocol<B: BlockT, H: ExHashT> {
 	boot_node_ids: Arc<HashSet<PeerId>>,
 	/// All the block announcement pre-validations that are currently active.
 	block_announce_pre_validation: FuturesUnordered<
-		Box<dyn Future<Output = sync::PreValidateBlockAnnounce<B::Header>> + Send + Unpin>
+		Pin<Box<dyn Future<Output = sync::PreValidateBlockAnnounce<B::Header>> + Send>>
 	>,
 }
 
@@ -1332,7 +1332,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		};
 
 		let future = self.sync.pre_validate_block_announce(who, &hash, announce, is_best);
-		self.block_announce_pre_validation.push(Box::new(future));
+		self.block_announce_pre_validation.push(Box::pin(future));
 	}
 
 	/// Process the result of the pre-validation of a block announcement.
diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 83055a5ea0d98..0f71a0075f729 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -1201,16 +1201,13 @@ impl<B: BlockT> ChainSync<B> {
 	/// [`PreValidateBlockAnnounce`]. This result needs to be feed into
 	/// [`ChainSync::process_block_announce_pre_validation`] to finish
 	/// the block announcement processing.
-	pub fn pre_validate_block_announce(
+	pub async fn pre_validate_block_announce(
 		&mut self,
 		who: PeerId,
 		hash: &B::Hash,
 		announce: BlockAnnounce<B::Header>,
 		is_best: bool,
-	) -> FEither<
-		impl Future<Output = PreValidateBlockAnnounce<B::Header>> + Send + Unpin,
-		impl Future<Output = PreValidateBlockAnnounce<B::Header>> + Send + Unpin
-	> {
+	) -> PreValidateBlockAnnounce<B::Header> {
 		let header = &announce.header;
 		let number = *header.number();
 		debug!(
@@ -1236,7 +1233,6 @@ impl<B: BlockT> ChainSync<B> {
 		let future = self.block_announce_validator.validate(&header, assoc_data);
 		let hash = hash.clone();
 
-		FEither::Right(async move {
 			match future.await {
 				Ok(Validation::Success { is_new_best }) =>
 					PreValidateBlockAnnounce::Process {
@@ -1258,7 +1254,6 @@ impl<B: BlockT> ChainSync<B> {
 					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
 				}
 			}
-		}.boxed())
 	}
 
 	/// Needs to be called with the result of [`ChainSync::pre_validate_block_announce`].
diff --git a/primitives/consensus/common/src/block_validation.rs b/primitives/consensus/common/src/block_validation.rs
index 3d1d6893b1c42..c3be7f3ffa8ba 100644
--- a/primitives/consensus/common/src/block_validation.rs
+++ b/primitives/consensus/common/src/block_validation.rs
@@ -18,8 +18,8 @@
 
 use crate::BlockStatus;
 use sp_runtime::{generic::BlockId, traits::Block};
-use std::{error::Error, sync::Arc, future::Future};
-use futures::FutureExt;
+use std::{error::Error, future::Future, pin::Pin, sync::Arc};
+use futures::FutureExt as _;
 
 /// A type which provides access to chain information.
 pub trait Chain<B: Block> {
@@ -52,7 +52,7 @@ pub trait BlockAnnounceValidator<B: Block> {
 		&mut self,
 		header: &B::Header,
 		data: &[u8],
-	) -> Box<dyn Future<Output = Result<Validation, Box<dyn Error + Send>>> + Send + Unpin>;
+	) -> Pin<Box<dyn Future<Output = Result<Validation, Box<dyn Error + Send>>> + Send>>;
 }
 
 /// Default implementation of `BlockAnnounceValidator`.
@@ -64,7 +64,7 @@ impl<B: Block> BlockAnnounceValidator<B> for DefaultBlockAnnounceValidator {
 		&mut self,
 		_: &B::Header,
 		_: &[u8],
-	) -> Box<dyn Future<Output = Result<Validation, Box<dyn Error + Send>>> + Send + Unpin> {
-		Box::new(async { Ok(Validation::Success { is_new_best: false }) }.boxed())
+	) -> Pin<Box<dyn Future<Output = Result<Validation, Box<dyn Error + Send>>> + Send>> {
+		async { Ok(Validation::Success { is_new_best: false }) }.boxed()
 	}
 }

From 0416a0b2ef6f85bd2683757b436369a4ffb39ddd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Tue, 15 Sep 2020 15:01:48 +0200
Subject: [PATCH 003/122] Update client/network/src/protocol/sync.rs

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
---
 client/network/src/protocol/sync.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 0f71a0075f729..5d922e67c2e58 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -1225,7 +1225,7 @@ impl<B: BlockT> ChainSync<B> {
 				who,
 				hash,
 			);
-			return FEither::Left(ready(PreValidateBlockAnnounce::Nothing { is_best, who, announce }))
+			return PreValidateBlockAnnounce::Nothing { is_best, who, announce }
 		}
 
 		// Let external validator check the block announcement.

From f974fe78ae0ce6a52ad5ffdc5b52a173e8bacac5 Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Mon, 24 Aug 2020 09:37:32 +0200
Subject: [PATCH 004/122] client/authority-discovery: Append PeerId to
 Multiaddr at most once (#6933)

* client/authority-discovery/worker: Extract address getter

* client/authority-discovery: Test for no duplicate p2p components

* client/authority-discovery: Append PeerId to Multiaddr at most once

When collecting the addresses to be published for the local node,
`addresses_to_publish` adds the local nodes `PeerId` to each
`Multiaddr`. Before doing so, ensure the `Multiaddr` does not already
contain one.

* client/authority-discovery: Remove explicit return
---
 Cargo.lock                                    |  1 +
 client/authority-discovery/Cargo.toml         |  1 +
 client/authority-discovery/src/worker.rs      | 43 ++++++-----
 .../authority-discovery/src/worker/tests.rs   | 71 ++++++++++++++++++-
 4 files changed, 97 insertions(+), 19 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 18a7266d3ccc1..03fa142c5b769 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6250,6 +6250,7 @@ version = "0.8.0-rc6"
 dependencies = [
  "bytes 0.5.6",
  "derive_more",
+ "either",
  "env_logger",
  "futures 0.3.5",
  "futures-timer 3.0.2",
diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml
index d154b35605761..8c898ab496422 100644
--- a/client/authority-discovery/Cargo.toml
+++ b/client/authority-discovery/Cargo.toml
@@ -19,6 +19,7 @@ prost-build = "0.6.1"
 bytes = "0.5.0"
 codec = { package = "parity-scale-codec", default-features = false, version = "1.3.4" }
 derive_more = "0.99.2"
+either = "1.5.3"
 futures = "0.3.4"
 futures-timer = "3.0.1"
 libp2p = { version = "0.24.0", default-features = false, features = ["kad"] }
diff --git a/client/authority-discovery/src/worker.rs b/client/authority-discovery/src/worker.rs
index 16f19489f94b7..09cdedd93a19e 100644
--- a/client/authority-discovery/src/worker.rs
+++ b/client/authority-discovery/src/worker.rs
@@ -30,7 +30,8 @@ use futures_timer::Delay;
 
 use addr_cache::AddrCache;
 use codec::Decode;
-use libp2p::core::multiaddr;
+use either::Either;
+use libp2p::{core::multiaddr, multihash::Multihash};
 use log::{debug, error, log_enabled};
 use prometheus_endpoint::{Counter, CounterVec, Gauge, Opts, U64, register};
 use prost::Message;
@@ -232,6 +233,26 @@ where
 		}
 	}
 
+	fn addresses_to_publish(&self) -> impl ExactSizeIterator<Item = Multiaddr> {
+		match &self.sentry_nodes {
+			Some(addrs) => Either::Left(addrs.clone().into_iter()),
+			None => {
+				let peer_id: Multihash = self.network.local_peer_id().into();
+				Either::Right(
+					self.network.external_addresses()
+						.into_iter()
+						.map(move |a| {
+							if a.iter().any(|p| matches!(p, multiaddr::Protocol::P2p(_))) {
+								a
+							} else {
+								a.with(multiaddr::Protocol::P2p(peer_id.clone()))
+							}
+						}),
+				)
+			}
+		}
+	}
+
 	/// Publish either our own or if specified the public addresses of our sentry nodes.
 	fn publish_ext_addresses(&mut self) -> Result<()> {
 		let key_store = match &self.role {
@@ -242,29 +263,15 @@ where
 			Role::Sentry => return Ok(()),
 		};
 
-		if let Some(metrics) = &self.metrics {
-			metrics.publish.inc()
-		}
-
-		let addresses: Vec<_> = match &self.sentry_nodes {
-			Some(addrs) => addrs.clone().into_iter()
-				.map(|a| a.to_vec())
-				.collect(),
-			None => self.network.external_addresses()
-				.into_iter()
-				.map(|a| a.with(multiaddr::Protocol::P2p(
-					self.network.local_peer_id().into(),
-				)))
-				.map(|a| a.to_vec())
-				.collect(),
-		};
+		let addresses = self.addresses_to_publish();
 
 		if let Some(metrics) = &self.metrics {
+			metrics.publish.inc();
 			metrics.amount_last_published.set(addresses.len() as u64);
 		}
 
 		let mut serialized_addresses = vec![];
-		schema::AuthorityAddresses { addresses }
+		schema::AuthorityAddresses { addresses: addresses.map(|a| a.to_vec()).collect() }
 			.encode(&mut serialized_addresses)
 			.map_err(Error::EncodingProto)?;
 
diff --git a/client/authority-discovery/src/worker/tests.rs b/client/authority-discovery/src/worker/tests.rs
index 68aadca7a7f30..4b16b9040b8dc 100644
--- a/client/authority-discovery/src/worker/tests.rs
+++ b/client/authority-discovery/src/worker/tests.rs
@@ -168,6 +168,7 @@ sp_api::mock_impl_runtime_apis! {
 
 pub struct TestNetwork {
 	peer_id: PeerId,
+	external_addresses: Vec<Multiaddr>,
 	// Whenever functions on `TestNetwork` are called, the function arguments are added to the
 	// vectors below.
 	pub put_value_call: Arc<Mutex<Vec<(kad::record::Key, Vec<u8>)>>>,
@@ -179,6 +180,10 @@ impl Default for TestNetwork {
 	fn default() -> Self {
 		TestNetwork {
 			peer_id: PeerId::random(),
+			external_addresses: vec![
+				"/ip6/2001:db8::/tcp/30333"
+					.parse().unwrap(),
+			],
 			put_value_call: Default::default(),
 			get_value_call: Default::default(),
 			set_priority_group_call: Default::default(),
@@ -212,7 +217,7 @@ impl NetworkStateInfo for TestNetwork {
 	}
 
 	fn external_addresses(&self) -> Vec<Multiaddr> {
-		vec!["/ip6/2001:db8::/tcp/30333".parse().unwrap()]
+		self.external_addresses.clone()
 	}
 }
 
@@ -691,3 +696,67 @@ fn do_not_cache_addresses_without_peer_id() {
 		"Expect worker to only cache `Multiaddr`s with `PeerId`s.",
 	);
 }
+
+#[test]
+fn addresses_to_publish_adds_p2p() {
+	let (_dht_event_tx, dht_event_rx) = channel(1000);
+	let network: Arc<TestNetwork> = Arc::new(Default::default());
+
+	assert!(!matches!(
+		network.external_addresses().pop().unwrap().pop().unwrap(),
+		multiaddr::Protocol::P2p(_)
+	));
+
+	let (_to_worker, from_service) = mpsc::channel(0);
+	let worker = Worker::new(
+		from_service,
+		Arc::new(TestApi {
+			authorities: vec![],
+		}),
+		network.clone(),
+		vec![],
+		dht_event_rx.boxed(),
+		Role::Authority(KeyStore::new()),
+		Some(prometheus_endpoint::Registry::new()),
+	);
+
+	assert!(
+		matches!(
+			worker.addresses_to_publish().next().unwrap().pop().unwrap(),
+			multiaddr::Protocol::P2p(_)
+		),
+		"Expect `addresses_to_publish` to append `p2p` protocol component.",
+	);
+}
+
+/// Ensure [`Worker::addresses_to_publish`] does not add an additional `p2p` protocol component in
+/// case one already exists.
+#[test]
+fn addresses_to_publish_respects_existing_p2p_protocol() {
+	let (_dht_event_tx, dht_event_rx) = channel(1000);
+	let network: Arc<TestNetwork> = Arc::new(TestNetwork {
+		external_addresses: vec![
+			"/ip6/2001:db8::/tcp/30333/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"
+				.parse().unwrap(),
+		],
+		.. Default::default()
+	});
+
+	let (_to_worker, from_service) = mpsc::channel(0);
+	let worker = Worker::new(
+		from_service,
+		Arc::new(TestApi {
+			authorities: vec![],
+		}),
+		network.clone(),
+		vec![],
+		dht_event_rx.boxed(),
+		Role::Authority(KeyStore::new()),
+		Some(prometheus_endpoint::Registry::new()),
+	);
+
+	assert_eq!(
+		network.external_addresses, worker.addresses_to_publish().collect::<Vec<_>>(),
+		"Expected Multiaddr from `TestNetwork` to not be altered.",
+	);
+}

From 7afdc676455f2cf72f21b8171fe1693a5a9beeec Mon Sep 17 00:00:00 2001
From: Xiliang Chen <xlchen1291@gmail.com>
Date: Mon, 24 Aug 2020 19:38:18 +1200
Subject: [PATCH 005/122] expose Deposit (#6943)

---
 frame/indices/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs
index c99beb463bc5f..e03cf4f1eea4d 100644
--- a/frame/indices/src/lib.rs
+++ b/frame/indices/src/lib.rs
@@ -121,6 +121,9 @@ decl_error! {
 
 decl_module! {
 	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system = frame_system {
+		/// The deposit needed for reserving an index.
+		const Deposit: BalanceOf<T> = T::Deposit::get();
+
 		fn deposit_event() = default;
 
 		/// Assign an previously unassigned index.

From 67d811bc6c4af38cdbe138166a8a8187f4658054 Mon Sep 17 00:00:00 2001
From: Ashley <ashley.ruglys@gmail.com>
Date: Mon, 24 Aug 2020 10:34:16 +0200
Subject: [PATCH 006/122] Add a `LightSyncState` field to the chain spec
 (#6894)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Reset code, almost ready for PR

* Improved build_hardcoded_spec

* Fix line widths

* Fix tests

* Fix sc-service-test

* Suggestions from code review

* Rename to LightSyncState

* It's not syncing :^(

* It syncs!

* Remove rpc call

* Convert spaces to tabs

* Moved sc-service things to export_sync_state.rs

* Fix tests

* Wait for syncing with network_status_sinks

* Remove sc-network from node-template

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Various changes, split the flag up into 2 pieces to make testing easier.

* Update client/cli/src/commands/build_spec_cmd.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Revert a lot of changes

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
---
 Cargo.lock                                 |  1 +
 client/api/src/in_mem.rs                   | 18 ++---
 client/api/src/light.rs                    | 19 +++--
 client/chain-spec/Cargo.toml               |  1 +
 client/chain-spec/src/chain_spec.rs        | 78 +++++++++++++++++---
 client/chain-spec/src/lib.rs               |  6 +-
 client/db/src/light.rs                     | 38 +++++-----
 client/service/src/chain_ops/build_spec.rs | 82 ++++++++++++++++++++++
 client/service/src/chain_ops/mod.rs        |  2 +
 client/service/test/src/client/light.rs    | 20 +++---
 10 files changed, 212 insertions(+), 53 deletions(-)
 create mode 100644 client/service/src/chain_ops/build_spec.rs

diff --git a/Cargo.lock b/Cargo.lock
index 03fa142c5b769..af32d89981dea 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6324,6 +6324,7 @@ name = "sc-chain-spec"
 version = "2.0.0-rc6"
 dependencies = [
  "impl-trait-for-tuples",
+ "parity-scale-codec",
  "sc-chain-spec-derive",
  "sc-network",
  "sc-telemetry",
diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs
index 306c3c2b2f10c..13ff7a01f9193 100644
--- a/client/api/src/in_mem.rs
+++ b/client/api/src/in_mem.rs
@@ -447,6 +447,16 @@ impl<Block: BlockT> light::Storage<Block> for Blockchain<Block>
 		Blockchain::finalize_header(self, id, None)
 	}
 
+	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
+		None
+	}
+
+	fn usage_info(&self) -> Option<UsageInfo> {
+		None
+	}
+}
+
+impl<Block: BlockT> light::ChtRootStorage<Block> for Blockchain<Block> {
 	fn header_cht_root(
 		&self,
 		_cht_size: NumberFor<Block>,
@@ -466,14 +476,6 @@ impl<Block: BlockT> light::Storage<Block> for Blockchain<Block>
 			.ok_or_else(|| sp_blockchain::Error::Backend(format!("Changes trie CHT for block {} not exists", block)))
 			.map(Some)
 	}
-
-	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
-		None
-	}
-
-	fn usage_info(&self) -> Option<UsageInfo> {
-		None
-	}
 }
 
 /// In-memory operation.
diff --git a/client/api/src/light.rs b/client/api/src/light.rs
index b359c1149eea6..f9aa002841caa 100644
--- a/client/api/src/light.rs
+++ b/client/api/src/light.rs
@@ -232,7 +232,9 @@ pub trait FetchChecker<Block: BlockT>: Send + Sync {
 
 
 /// Light client blockchain storage.
-pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetadata<Block, Error=ClientError> {
+pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block>
+	+ HeaderMetadata<Block, Error=ClientError> + ChtRootStorage<Block>
+{
 	/// Store new header. Should refuse to revert any finalized blocks.
 	///
 	/// Takes new authorities, the leaf state of the new block, and
@@ -254,6 +256,15 @@ pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetada
 	/// Get last finalized header.
 	fn last_finalized(&self) -> ClientResult<Block::Hash>;
 
+	/// Get storage cache.
+	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>>;
+
+	/// Get storage usage statistics.
+	fn usage_info(&self) -> Option<UsageInfo>;
+}
+
+/// Light client CHT root storage.
+pub trait ChtRootStorage<Block: BlockT> {
 	/// Get headers CHT root for given block. Returns None if the block is not pruned (not a part of any CHT).
 	fn header_cht_root(
 		&self,
@@ -267,12 +278,6 @@ pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block> + HeaderMetada
 		cht_size: NumberFor<Block>,
 		block: NumberFor<Block>,
 	) -> ClientResult<Option<Block::Hash>>;
-
-	/// Get storage cache.
-	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>>;
-
-	/// Get storage usage statistics.
-	fn usage_info(&self) -> Option<UsageInfo>;
 }
 
 /// Remote header.
diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml
index fb0addf461abd..fcfb80a720e25 100644
--- a/client/chain-spec/Cargo.toml
+++ b/client/chain-spec/Cargo.toml
@@ -21,3 +21,4 @@ serde_json = "1.0.41"
 sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
 sp-chain-spec = { version = "2.0.0-rc6", path = "../../primitives/chain-spec" }
 sc-telemetry = { version = "2.0.0-rc6", path = "../telemetry" }
+codec = { package = "parity-scale-codec", version = "1.3.4" }
diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs
index 52414f8687c96..20811394c56d7 100644
--- a/client/chain-spec/src/chain_spec.rs
+++ b/client/chain-spec/src/chain_spec.rs
@@ -17,6 +17,7 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 //! Substrate chain configurations.
+#![warn(missing_docs)]
 
 use std::{borrow::Cow, fs::File, path::PathBuf, sync::Arc, collections::HashMap};
 use serde::{Serialize, Deserialize};
@@ -26,6 +27,7 @@ use serde_json as json;
 use crate::{RuntimeGenesis, ChainType, extension::GetExtension, Properties};
 use sc_network::config::MultiaddrWithPeerId;
 use sc_telemetry::TelemetryEndpoints;
+use sp_runtime::traits::Block as BlockT;
 
 enum GenesisSource<G> {
 	File(PathBuf),
@@ -157,6 +159,7 @@ struct ClientSpec<E> {
 	consensus_engine: (),
 	#[serde(skip_serializing)]
 	genesis: serde::de::IgnoredAny,
+	light_sync_state: Option<SerializableLightSyncState>,
 }
 
 /// A type denoting empty extensions.
@@ -245,6 +248,7 @@ impl<G, E> ChainSpec<G, E> {
 			extensions,
 			consensus_engine: (),
 			genesis: Default::default(),
+			light_sync_state: None,
 		};
 
 		ChainSpec {
@@ -257,6 +261,11 @@ impl<G, E> ChainSpec<G, E> {
 	fn chain_type(&self) -> ChainType {
 		self.client_spec.chain_type.clone()
 	}
+
+	/// Hardcode infomation to allow light clients to sync quickly into the chain spec.
+	fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState) {
+		self.client_spec.light_sync_state = Some(light_sync_state);	
+	}
 }
 
 impl<G, E: serde::de::DeserializeOwned> ChainSpec<G, E> {
@@ -284,16 +293,15 @@ impl<G, E: serde::de::DeserializeOwned> ChainSpec<G, E> {
 	}
 }
 
-impl<G: RuntimeGenesis, E: serde::Serialize + Clone + 'static> ChainSpec<G, E> {
-	/// Dump to json string.
-	pub fn as_json(&self, raw: bool) -> Result<String, String> {
-		#[derive(Serialize, Deserialize)]
-		struct Container<G, E> {
-			#[serde(flatten)]
-			client_spec: ClientSpec<E>,
-			genesis: Genesis<G>,
+#[derive(Serialize, Deserialize)]
+struct JsonContainer<G, E> {
+	#[serde(flatten)]
+	client_spec: ClientSpec<E>,
+	genesis: Genesis<G>,
+}
 
-		};
+impl<G: RuntimeGenesis, E: serde::Serialize + Clone + 'static> ChainSpec<G, E> {
+	fn json_container(&self, raw: bool) -> Result<JsonContainer<G, E>, String> {
 		let genesis = match (raw, self.genesis.resolve()?) {
 			(true, Genesis::Runtime(g)) => {
 				let storage = g.build_storage()?;
@@ -313,10 +321,15 @@ impl<G: RuntimeGenesis, E: serde::Serialize + Clone + 'static> ChainSpec<G, E> {
 			},
 			(_, genesis) => genesis,
 		};
-		let container = Container {
+		Ok(JsonContainer {
 			client_spec: self.client_spec.clone(),
 			genesis,
-		};
+		})
+	}
+
+	/// Dump to json string.
+	pub fn as_json(&self, raw: bool) -> Result<String, String> {
+		let container = self.json_container(raw)?;
 		json::to_string_pretty(&container)
 			.map_err(|e| format!("Error generating spec json: {}", e))
 	}
@@ -378,6 +391,49 @@ where
 	fn set_storage(&mut self, storage: Storage) {
 		self.genesis = GenesisSource::Storage(storage);
 	}
+
+	fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState) {
+		ChainSpec::set_light_sync_state(self, light_sync_state)
+	}
+}
+
+/// Hardcoded infomation that allows light clients to sync quickly.
+pub struct LightSyncState<Block: BlockT> {
+	/// The header of the best finalized block.
+	pub header: <Block as BlockT>::Header,
+	/// A list of all CHTs in the chain.
+	pub chts: Vec<<Block as BlockT>::Hash>,
+}
+
+impl<Block: BlockT> LightSyncState<Block> {
+	/// Convert into a `SerializableLightSyncState`.
+	pub fn to_serializable(&self) -> SerializableLightSyncState {
+		use codec::Encode;
+
+		SerializableLightSyncState {
+			header: StorageData(self.header.encode()),
+			chts: self.chts.iter().map(|hash| StorageData(hash.encode())).collect(),
+		}
+	}
+
+	/// Convert from a `SerializableLightSyncState`.
+	pub fn from_serializable(serialized: &SerializableLightSyncState) -> Result<Self, codec::Error> {
+		Ok(Self {
+			header: codec::Decode::decode(&mut &serialized.header.0[..])?,
+			chts: serialized.chts.iter()
+				.map(|cht| codec::Decode::decode(&mut &cht.0[..]))
+				.collect::<Result<_, _>>()?,
+		})
+	}
+}
+
+/// The serializable form of `LightSyncState`. Created using `LightSyncState::serialize`.
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
+pub struct SerializableLightSyncState {
+	header: StorageData,
+	chts: Vec<StorageData>,
 }
 
 #[cfg(test)]
diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs
index 8901a9a682224..f5afe496f1980 100644
--- a/client/chain-spec/src/lib.rs
+++ b/client/chain-spec/src/lib.rs
@@ -108,7 +108,9 @@
 mod chain_spec;
 mod extension;
 
-pub use chain_spec::{ChainSpec as GenericChainSpec, NoExtension};
+pub use chain_spec::{
+	ChainSpec as GenericChainSpec, NoExtension, LightSyncState, SerializableLightSyncState,
+};
 pub use extension::{Group, Fork, Forks, Extension, GetExtension, get_extension};
 pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup};
 pub use sp_chain_spec::{Properties, ChainType};
@@ -155,6 +157,8 @@ pub trait ChainSpec: BuildStorage + Send {
 	///
 	/// This will be used as storage at genesis.
 	fn set_storage(&mut self, storage: Storage);
+	/// Hardcode infomation to allow light clients to sync quickly into the chain spec.
+	fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState);
 }
 
 impl std::fmt::Debug for dyn ChainSpec {
diff --git a/client/db/src/light.rs b/client/db/src/light.rs
index 139ecf3b22c72..adf9a98d35e2a 100644
--- a/client/db/src/light.rs
+++ b/client/db/src/light.rs
@@ -27,7 +27,7 @@ use sc_client_api::{
 	blockchain::{
 		BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo,
 	},
-	Storage
+	Storage, ChtRootStorage,
 };
 use sp_blockchain::{
 	CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
@@ -523,22 +523,6 @@ impl<Block> Storage<Block> for LightStorage<Block>
 		}
 	}
 
-	fn header_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>> {
-		self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block)
-	}
-
-	fn changes_trie_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>> {
-		self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block)
-	}
-
 	fn finalize_header(&self, id: BlockId<Block>) -> ClientResult<()> {
 		if let Some(header) = self.header(id)? {
 			let mut transaction = Transaction::new();
@@ -612,6 +596,26 @@ impl<Block> Storage<Block> for LightStorage<Block>
 	}
 }
 
+impl<Block> ChtRootStorage<Block> for LightStorage<Block>
+	where Block: BlockT,
+{
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<Block::Hash>> {
+		self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block)
+	}
+
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<Block::Hash>> {
+		self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block)
+	}
+}
+
 /// Build the key for inserting header-CHT at given block.
 fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
 	let mut key = [cht_type; 5];
diff --git a/client/service/src/chain_ops/build_spec.rs b/client/service/src/chain_ops/build_spec.rs
new file mode 100644
index 0000000000000..c84c1c754adac
--- /dev/null
+++ b/client/service/src/chain_ops/build_spec.rs
@@ -0,0 +1,82 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One};
+use sp_blockchain::HeaderBackend;
+use crate::{TFullBackend, TLightBackend};
+use std::sync::Arc;
+use sp_runtime::generic::BlockId;
+
+/// An error for if this function is being called on a full node.
+pub const CHT_ROOT_ERROR: &str =
+	"Backend doesn't store CHT roots. Make sure you're calling this on a light client.";
+
+/// Something that might allow access to a `ChtRootStorage`.
+pub trait MaybeChtRootStorageProvider<Block> {
+	/// Potentially get a reference to a `ChtRootStorage`.
+	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>>;
+}
+
+impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TFullBackend<Block> {
+	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
+		None
+	}
+}
+
+impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TLightBackend<Block> {
+	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
+		Some(self.blockchain().storage())
+	}
+}
+
+/// Build a `LightSyncState` from the CHT roots stored in a backend.
+pub fn build_light_sync_state<TBl, TCl, TBackend>(
+	client: Arc<TCl>,
+	backend: Arc<TBackend>,
+) -> Result<sc_chain_spec::LightSyncState<TBl>, sp_blockchain::Error>
+	where
+		TBl: BlockT,
+		TCl: HeaderBackend<TBl>,
+		TBackend: MaybeChtRootStorageProvider<TBl>,
+{
+	let storage = backend.cht_root_storage().ok_or(CHT_ROOT_ERROR)?;
+
+	let finalized_hash = client.info().finalized_hash;
+	let finalized_number = client.info().finalized_number;
+
+	use sc_client_api::cht;
+
+	let mut chts = Vec::new();
+
+	// We can't fetch a CHT root later than `finalized_number - 2 * cht_size`.
+	let cht_size_x_2 = cht::size::<NumberFor::<TBl>>() * NumberFor::<TBl>::from(2);
+
+	let mut number = NumberFor::<TBl>::one();
+
+	while number <= finalized_number.saturating_sub(cht_size_x_2) {
+		match storage.header_cht_root(cht::size(), number)? {
+			Some(cht_root) => chts.push(cht_root),
+			None => log::error!("No CHT found for block {}", number),
+		}
+
+		number += cht::size();
+	}
+
+	Ok(sc_chain_spec::LightSyncState {
+		header: client.header(BlockId::Hash(finalized_hash))?.unwrap(),
+		chts,
+	})
+}
diff --git a/client/service/src/chain_ops/mod.rs b/client/service/src/chain_ops/mod.rs
index af6e6f632fc06..19f5e346820aa 100644
--- a/client/service/src/chain_ops/mod.rs
+++ b/client/service/src/chain_ops/mod.rs
@@ -21,9 +21,11 @@ mod export_blocks;
 mod export_raw_state;
 mod import_blocks;
 mod revert_chain;
+mod build_spec;
 
 pub use check_block::*;
 pub use export_blocks::*;
 pub use export_raw_state::*;
 pub use import_blocks::*;
 pub use revert_chain::*;
+pub use build_spec::*;
diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs
index ffc84ad47b8f3..515d7d1532677 100644
--- a/client/service/test/src/client/light.rs
+++ b/client/service/test/src/client/light.rs
@@ -42,7 +42,7 @@ use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVer
 use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor};
 use sc_client_api::{
 	blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider,
-	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain},
+	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ChtRootStorage,
 	AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation,
 	RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest,
 	RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend,
@@ -164,6 +164,16 @@ impl Storage<Block> for DummyStorage {
 		Err(ClientError::Backend("Test error".into()))
 	}
 
+	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
+		None
+	}
+
+	fn usage_info(&self) -> Option<sc_client_api::UsageInfo> {
+		None
+	}
+}
+
+impl ChtRootStorage<Block> for DummyStorage {
 	fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult<Option<Hash>> {
 		Err(ClientError::Backend("Test error".into()))
 	}
@@ -177,14 +187,6 @@ impl Storage<Block> for DummyStorage {
 			).into())
 			.map(Some)
 	}
-
-	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
-		None
-	}
-
-	fn usage_info(&self) -> Option<sc_client_api::UsageInfo> {
-		None
-	}
 }
 
 struct DummyCallExecutor;

From e7146411fb830b029b057bc417248972964fc9e6 Mon Sep 17 00:00:00 2001
From: Ashley <ashley.ruglys@gmail.com>
Date: Mon, 24 Aug 2020 15:11:21 +0200
Subject: [PATCH 007/122] Dynamically generate CHT roots on a full client
 (#6944)

* Generate CHT roots on a full client

* add changes_trie_root function

* Add a test

* Line widths

* Fix sc-service-test

* Clarify comments

* Revert comments
---
 client/api/src/backend.rs                  | 18 +++++
 client/api/src/in_mem.rs                   |  4 +-
 client/api/src/light.rs                    | 21 +----
 client/db/src/lib.rs                       | 91 +++++++++++++++++++++-
 client/db/src/light.rs                     |  6 +-
 client/light/src/blockchain.rs             | 20 ++++-
 client/service/src/chain_ops/build_spec.rs | 31 ++------
 client/service/test/src/client/light.rs    |  4 +-
 8 files changed, 139 insertions(+), 56 deletions(-)

diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs
index efc5ca4ee8ca0..47fec977f5e82 100644
--- a/client/api/src/backend.rs
+++ b/client/api/src/backend.rs
@@ -536,3 +536,21 @@ pub fn changes_tries_state_at_block<'a, Block: BlockT>(
 		None => Ok(None),
 	}
 }
+
+/// Provide CHT roots. These are stored on a light client and generated dynamically on a full
+/// client.
+pub trait ProvideChtRoots<Block: BlockT> {
+	/// Get headers CHT root for given block. Returns None if the block is not a part of any CHT.
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>>;
+
+	/// Get changes trie CHT root for given block. Returns None if the block is not a part of any CHT.
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>>;
+}
diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs
index 13ff7a01f9193..ded030fb8046f 100644
--- a/client/api/src/in_mem.rs
+++ b/client/api/src/in_mem.rs
@@ -35,7 +35,7 @@ use sp_state_machine::{
 use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata};
 
 use crate::{
-	backend::{self, NewBlockState},
+	backend::{self, NewBlockState, ProvideChtRoots},
 	blockchain::{
 		self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId
 	},
@@ -456,7 +456,7 @@ impl<Block: BlockT> light::Storage<Block> for Blockchain<Block>
 	}
 }
 
-impl<Block: BlockT> light::ChtRootStorage<Block> for Blockchain<Block> {
+impl<Block: BlockT> ProvideChtRoots<Block> for Blockchain<Block> {
 	fn header_cht_root(
 		&self,
 		_cht_size: NumberFor<Block>,
diff --git a/client/api/src/light.rs b/client/api/src/light.rs
index f9aa002841caa..144851dac0075 100644
--- a/client/api/src/light.rs
+++ b/client/api/src/light.rs
@@ -32,7 +32,7 @@ use sp_blockchain::{
 	HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache,
 	Error as ClientError, Result as ClientResult,
 };
-use crate::{backend::{AuxStore, NewBlockState}, UsageInfo};
+use crate::{backend::{AuxStore, NewBlockState}, UsageInfo, ProvideChtRoots};
 
 /// Remote call request.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -233,7 +233,7 @@ pub trait FetchChecker<Block: BlockT>: Send + Sync {
 
 /// Light client blockchain storage.
 pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block>
-	+ HeaderMetadata<Block, Error=ClientError> + ChtRootStorage<Block>
+	+ HeaderMetadata<Block, Error=ClientError> + ProvideChtRoots<Block>
 {
 	/// Store new header. Should refuse to revert any finalized blocks.
 	///
@@ -263,23 +263,6 @@ pub trait Storage<Block: BlockT>: AuxStore + HeaderBackend<Block>
 	fn usage_info(&self) -> Option<UsageInfo>;
 }
 
-/// Light client CHT root storage.
-pub trait ChtRootStorage<Block: BlockT> {
-	/// Get headers CHT root for given block. Returns None if the block is not pruned (not a part of any CHT).
-	fn header_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>>;
-
-	/// Get changes trie CHT root for given block. Returns None if the block is not pruned (not a part of any CHT).
-	fn changes_trie_cht_root(
-		&self,
-		cht_size: NumberFor<Block>,
-		block: NumberFor<Block>,
-	) -> ClientResult<Option<Block::Hash>>;
-}
-
 /// Remote header.
 #[derive(Debug)]
 pub enum LocalOrRemote<Data, Request> {
diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs
index d854c80bf3535..bd438f4dd71b2 100644
--- a/client/db/src/lib.rs
+++ b/client/db/src/lib.rs
@@ -54,8 +54,8 @@ use std::collections::{HashMap, HashSet};
 
 use sc_client_api::{
 	UsageInfo, MemoryInfo, IoInfo, MemorySize,
-	backend::{NewBlockState, PrunableStateChangesTrieStorage},
-	leaves::{LeafSet, FinalizationDisplaced},
+	backend::{NewBlockState, PrunableStateChangesTrieStorage, ProvideChtRoots},
+	leaves::{LeafSet, FinalizationDisplaced}, cht,
 };
 use sp_blockchain::{
 	Result as ClientResult, Error as ClientError,
@@ -70,7 +70,7 @@ use sp_core::ChangesTrieConfiguration;
 use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges};
 use sp_core::storage::{well_known_keys, ChildInfo};
 use sp_arithmetic::traits::Saturating;
-use sp_runtime::{generic::BlockId, Justification, Storage};
+use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Storage};
 use sp_runtime::traits::{
 	Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor,
 };
@@ -405,6 +405,14 @@ impl<Block: BlockT> BlockchainDb<Block> {
 			meta.finalized_hash = hash;
 		}
 	}
+
+	// Get block changes trie root, if available.
+	fn changes_trie_root(&self, block: BlockId<Block>) -> ClientResult<Option<Block::Hash>> {
+		self.header(block)
+			.map(|header| header.and_then(|header|
+				header.digest().log(DigestItem::as_changes_trie_root)
+					.cloned()))
+	}
 }
 
 impl<Block: BlockT> sc_client_api::blockchain::HeaderBackend<Block> for BlockchainDb<Block> {
@@ -525,6 +533,58 @@ impl<Block: BlockT> HeaderMetadata<Block> for BlockchainDb<Block> {
 	}
 }
 
+impl<Block: BlockT> ProvideChtRoots<Block> for BlockchainDb<Block> {
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		let cht_number = match cht::block_to_cht_number(cht_size, block) {
+			Some(number) => number,
+			None => return Ok(None),
+		};
+
+		let cht_start: NumberFor<Block> = cht::start_number(cht::size(), cht_number);
+
+		let mut current_num = cht_start;
+		let cht_range = ::std::iter::from_fn(|| {
+			let old_current_num = current_num;
+			current_num = current_num + One::one();
+			Some(old_current_num)
+		});
+
+		cht::compute_root::<Block::Header, HashFor<Block>, _>(
+			cht::size(), cht_number, cht_range.map(|num| self.hash(num))
+		).map(Some)
+	}
+
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		let cht_number = match cht::block_to_cht_number(cht_size, block) {
+			Some(number) => number,
+			None => return Ok(None),
+		};
+
+		let cht_start: NumberFor<Block> = cht::start_number(cht::size(), cht_number);
+
+		let mut current_num = cht_start;
+		let cht_range = ::std::iter::from_fn(|| {
+			let old_current_num = current_num;
+			current_num = current_num + One::one();
+			Some(old_current_num)
+		});
+
+		cht::compute_root::<Block::Header, HashFor<Block>, _>(
+			cht::size(),
+			cht_number,
+			cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))),
+		).map(Some)
+	}
+}
+
 /// Database transaction
 pub struct BlockImportOperation<Block: BlockT> {
 	old_state: SyncingCachingState<RefTrackingState<Block>, Block>,
@@ -2329,4 +2389,29 @@ pub(crate) mod tests {
 			backend.commit_operation(op).unwrap_err();
 		}
 	}
+
+	#[test]
+	fn header_cht_root_works() {
+		use sc_client_api::ProvideChtRoots;
+
+		let backend = Backend::<Block>::new_test(10, 10);
+
+		// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
+		let mut prev_hash = insert_header(&backend, 0, Default::default(), None, Default::default());
+		let cht_size: u64 = cht::size();
+		for i in 1..1 + cht_size + cht_size + 1 {
+			prev_hash = insert_header(&backend, i, prev_hash, None, Default::default());
+		}
+
+		let blockchain = backend.blockchain();
+
+		let cht_root_1 = blockchain.header_cht_root(cht_size, cht::start_number(cht_size, 0))
+			.unwrap().unwrap();
+		let cht_root_2 = blockchain.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2)
+			.unwrap().unwrap();
+		let cht_root_3 = blockchain.header_cht_root(cht_size, cht::end_number(cht_size, 0))
+			.unwrap().unwrap();
+		assert_eq!(cht_root_1, cht_root_2);
+		assert_eq!(cht_root_2, cht_root_3);
+	}
 }
diff --git a/client/db/src/light.rs b/client/db/src/light.rs
index adf9a98d35e2a..acfb6217ce9e0 100644
--- a/client/db/src/light.rs
+++ b/client/db/src/light.rs
@@ -23,11 +23,11 @@ use std::convert::TryInto;
 use parking_lot::RwLock;
 
 use sc_client_api::{
-	cht, backend::{AuxStore, NewBlockState}, UsageInfo,
+	cht, backend::{AuxStore, NewBlockState, ProvideChtRoots}, UsageInfo,
 	blockchain::{
 		BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo,
 	},
-	Storage, ChtRootStorage,
+	Storage,
 };
 use sp_blockchain::{
 	CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
@@ -596,7 +596,7 @@ impl<Block> Storage<Block> for LightStorage<Block>
 	}
 }
 
-impl<Block> ChtRootStorage<Block> for LightStorage<Block>
+impl<Block> ProvideChtRoots<Block> for LightStorage<Block>
 	where Block: BlockT,
 {
 	fn header_cht_root(
diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs
index 9d557db887d29..3b5753f2849d5 100644
--- a/client/light/src/blockchain.rs
+++ b/client/light/src/blockchain.rs
@@ -29,7 +29,7 @@ use sp_blockchain::{
 };
 pub use sc_client_api::{
 	backend::{
-		AuxStore, NewBlockState
+		AuxStore, NewBlockState, ProvideChtRoots,
 	},
 	blockchain::{
 		Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache,
@@ -173,3 +173,21 @@ impl<S, Block: BlockT> RemoteBlockchain<Block> for Blockchain<S>
 		}))
 	}
 }
+
+impl<S: Storage<Block>, Block: BlockT> ProvideChtRoots<Block> for Blockchain<S> {
+	fn header_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		self.storage().header_cht_root(cht_size, block)
+	}
+
+	fn changes_trie_cht_root(
+		&self,
+		cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> sp_blockchain::Result<Option<Block::Hash>> {
+		self.storage().changes_trie_cht_root(cht_size, block)
+	}
+}
diff --git a/client/service/src/chain_ops/build_spec.rs b/client/service/src/chain_ops/build_spec.rs
index c84c1c754adac..40d591d81f02b 100644
--- a/client/service/src/chain_ops/build_spec.rs
+++ b/client/service/src/chain_ops/build_spec.rs
@@ -16,31 +16,9 @@
 
 use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One};
 use sp_blockchain::HeaderBackend;
-use crate::{TFullBackend, TLightBackend};
 use std::sync::Arc;
 use sp_runtime::generic::BlockId;
-
-/// An error for if this function is being called on a full node.
-pub const CHT_ROOT_ERROR: &str =
-	"Backend doesn't store CHT roots. Make sure you're calling this on a light client.";
-
-/// Something that might allow access to a `ChtRootStorage`.
-pub trait MaybeChtRootStorageProvider<Block> {
-	/// Potentially get a reference to a `ChtRootStorage`.
-	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>>;
-}
-
-impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TFullBackend<Block> {
-	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
-		None
-	}
-}
-
-impl<Block: BlockT> MaybeChtRootStorageProvider<Block> for TLightBackend<Block> {
-	fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage<Block>> {
-		Some(self.blockchain().storage())
-	}
-}
+use sc_client_api::ProvideChtRoots;
 
 /// Build a `LightSyncState` from the CHT roots stored in a backend.
 pub fn build_light_sync_state<TBl, TCl, TBackend>(
@@ -50,9 +28,10 @@ pub fn build_light_sync_state<TBl, TCl, TBackend>(
 	where
 		TBl: BlockT,
 		TCl: HeaderBackend<TBl>,
-		TBackend: MaybeChtRootStorageProvider<TBl>,
+		TBackend: sc_client_api::Backend<TBl>,
+		<TBackend as sc_client_api::Backend<TBl>>::Blockchain: ProvideChtRoots<TBl>,
 {
-	let storage = backend.cht_root_storage().ok_or(CHT_ROOT_ERROR)?;
+	let cht_root_provider = backend.blockchain();
 
 	let finalized_hash = client.info().finalized_hash;
 	let finalized_number = client.info().finalized_number;
@@ -67,7 +46,7 @@ pub fn build_light_sync_state<TBl, TCl, TBackend>(
 	let mut number = NumberFor::<TBl>::one();
 
 	while number <= finalized_number.saturating_sub(cht_size_x_2) {
-		match storage.header_cht_root(cht::size(), number)? {
+		match cht_root_provider.header_cht_root(cht::size(), number)? {
 			Some(cht_root) => chts.push(cht_root),
 			None => log::error!("No CHT found for block {}", number),
 		}
diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs
index 515d7d1532677..f38aef008e11c 100644
--- a/client/service/test/src/client/light.rs
+++ b/client/service/test/src/client/light.rs
@@ -42,7 +42,7 @@ use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVer
 use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor};
 use sc_client_api::{
 	blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider,
-	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ChtRootStorage,
+	in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ProvideChtRoots,
 	AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation,
 	RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest,
 	RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend,
@@ -173,7 +173,7 @@ impl Storage<Block> for DummyStorage {
 	}
 }
 
-impl ChtRootStorage<Block> for DummyStorage {
+impl ProvideChtRoots<Block> for DummyStorage {
 	fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult<Option<Hash>> {
 		Err(ClientError::Backend("Test error".into()))
 	}

From 508dfcd42c4e079b5d7fa77bc7622a2c3a00ea05 Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Mon, 24 Aug 2020 15:24:00 +0200
Subject: [PATCH 008/122] Enable verification logic when executing benchmarks
 (#6929)

* Add `--verify` flag to benchmark execution

* make it so `--verify` can be used for getting the actual benchmarks

* undo manual testing

* oops

* use benchmark config struct

* verify is default on, docs update

* remove clone

* improve formatting

* fix test

* bump impl for ci
---
 bin/node/runtime/src/lib.rs                 |  10 +-
 frame/benchmarking/src/lib.rs               | 179 ++++++++++----------
 frame/benchmarking/src/tests.rs             |  23 ++-
 frame/benchmarking/src/utils.rs             |  43 +++--
 frame/staking/src/benchmarking.rs           |   3 +-
 utils/frame/benchmarking-cli/src/command.rs |   1 +
 utils/frame/benchmarking-cli/src/lib.rs     |   4 +
 7 files changed, 150 insertions(+), 113 deletions(-)

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index e7842e5c4ba3b..17c02eca17b4a 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -1134,13 +1134,7 @@ impl_runtime_apis! {
 	#[cfg(feature = "runtime-benchmarks")]
 	impl frame_benchmarking::Benchmark<Block> for Runtime {
 		fn dispatch_benchmark(
-			pallet: Vec<u8>,
-			benchmark: Vec<u8>,
-			lowest_range_values: Vec<u32>,
-			highest_range_values: Vec<u32>,
-			steps: Vec<u32>,
-			repeat: u32,
-			extra: bool,
+			config: frame_benchmarking::BenchmarkConfig
 		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
 			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey};
 			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
@@ -1170,7 +1164,7 @@ impl_runtime_apis! {
 			];
 
 			let mut batches = Vec::<BenchmarkBatch>::new();
-			let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist, extra);
+			let params = (&config, &whitelist);
 
 			add_benchmark!(params, batches, pallet_babe, Babe);
 			add_benchmark!(params, batches, pallet_balances, Balances);
diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs
index cebdcbcfecd25..03d60dbec5851 100644
--- a/frame/benchmarking/src/lib.rs
+++ b/frame/benchmarking/src/lib.rs
@@ -648,30 +648,11 @@ macro_rules! benchmark_backend {
 				]
 			}
 
-			fn instance(&self, components: &[($crate::BenchmarkParameter, u32)])
-				-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
-			{
-				$(
-					let $common = $common_from;
-				)*
-				$(
-					// Prepare instance
-					let $param = components.iter()
-						.find(|&c| c.0 == $crate::BenchmarkParameter::$param)
-						.unwrap().1;
-				)*
-				$(
-					let $pre_id : $pre_ty = $pre_ex;
-				)*
-				$( $param_instancer ; )*
-				$( $post )*
-
-				Ok(Box::new(move || -> Result<(), &'static str> { $eval; Ok(()) }))
-			}
-
-			fn verify(&self, components: &[($crate::BenchmarkParameter, u32)])
-				-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
-			{
+			fn instance(
+				&self,
+				components: &[($crate::BenchmarkParameter, u32)],
+				verify: bool
+			) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str> {
 				$(
 					let $common = $common_from;
 				)*
@@ -687,7 +668,13 @@ macro_rules! benchmark_backend {
 				$( $param_instancer ; )*
 				$( $post )*
 
-				Ok(Box::new(move || -> Result<(), &'static str> { $eval; $postcode; Ok(()) }))
+				Ok(Box::new(move || -> Result<(), &'static str> {
+					$eval;
+					if verify {
+						$postcode;
+					}
+					Ok(())
+				}))
 			}
 		}
 	};
@@ -736,26 +723,16 @@ macro_rules! selected_benchmark {
 				}
 			}
 
-			fn instance(&self, components: &[($crate::BenchmarkParameter, u32)])
-				-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
-			{
-				match self {
-					$(
-						Self::$bench => <
-							$bench as $crate::BenchmarkingSetup<T $(, $bench_inst)? >
-						>::instance(&$bench, components),
-					)*
-				}
-			}
-
-			fn verify(&self, components: &[($crate::BenchmarkParameter, u32)])
-				-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
-			{
+			fn instance(
+				&self,
+				components: &[($crate::BenchmarkParameter, u32)],
+				verify: bool
+			) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str> {
 				match self {
 					$(
 						Self::$bench => <
 							$bench as $crate::BenchmarkingSetup<T $(, $bench_inst)? >
-						>::verify(&$bench, components),
+						>::instance(&$bench, components, verify),
 					)*
 				}
 			}
@@ -791,7 +768,8 @@ macro_rules! impl_benchmark {
 				highest_range_values: &[u32],
 				steps: &[u32],
 				repeat: u32,
-				whitelist: &[$crate::TrackedStorageKey]
+				whitelist: &[$crate::TrackedStorageKey],
+				verify: bool,
 			) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
 				// Map the input to the selected benchmark.
 				let extrinsic = sp_std::str::from_utf8(extrinsic)
@@ -826,6 +804,7 @@ macro_rules! impl_benchmark {
 					repeat: u32,
 					c: Vec<($crate::BenchmarkParameter, u32)>,
 					results: &mut Vec<$crate::BenchmarkResults>,
+					verify: bool,
 				| -> Result<(), &'static str> {
 					// Run the benchmark `repeat` times.
 					for _ in 0..repeat {
@@ -833,7 +812,7 @@ macro_rules! impl_benchmark {
 						// benchmark.
 						let closure_to_benchmark = <
 							SelectedBenchmark as $crate::BenchmarkingSetup<T $(, $instance)?>
-						>::instance(&selected_benchmark, &c)?;
+						>::instance(&selected_benchmark, &c, verify)?;
 
 						// Set the block number to at least 1 so events are deposited.
 						if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
@@ -847,43 +826,49 @@ macro_rules! impl_benchmark {
 						// Reset the read/write counter so we don't count operations in the setup process.
 						$crate::benchmarking::reset_read_write_count();
 
-						// Time the extrinsic logic.
-						frame_support::debug::trace!(
-							target: "benchmark",
-							"Start Benchmark: {:?}", c
-						);
+						if verify {
+							closure_to_benchmark()?;
+						} else {
+							// Time the extrinsic logic.
+							frame_support::debug::trace!(
+								target: "benchmark",
+								"Start Benchmark: {:?}", c
+							);
 
-						let start_extrinsic = $crate::benchmarking::current_time();
-						closure_to_benchmark()?;
-						let finish_extrinsic = $crate::benchmarking::current_time();
-						let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
-						// Commit the changes to get proper write count
-						$crate::benchmarking::commit_db();
-						frame_support::debug::trace!(
-							target: "benchmark",
-							"End Benchmark: {} ns", elapsed_extrinsic
-						);
-						let read_write_count = $crate::benchmarking::read_write_count();
-						frame_support::debug::trace!(
-							target: "benchmark",
-							"Read/Write Count {:?}", read_write_count
-						);
+							let start_extrinsic = $crate::benchmarking::current_time();
 
-						// Time the storage root recalculation.
-						let start_storage_root = $crate::benchmarking::current_time();
-						$crate::storage_root();
-						let finish_storage_root = $crate::benchmarking::current_time();
-						let elapsed_storage_root = finish_storage_root - start_storage_root;
+							closure_to_benchmark()?;
 
-						results.push($crate::BenchmarkResults {
-							components: c.clone(),
-							extrinsic_time: elapsed_extrinsic,
-							storage_root_time: elapsed_storage_root,
-							reads: read_write_count.0,
-							repeat_reads: read_write_count.1,
-							writes: read_write_count.2,
-							repeat_writes: read_write_count.3,
-						});
+							let finish_extrinsic = $crate::benchmarking::current_time();
+							let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
+							// Commit the changes to get proper write count
+							$crate::benchmarking::commit_db();
+							frame_support::debug::trace!(
+								target: "benchmark",
+								"End Benchmark: {} ns", elapsed_extrinsic
+							);
+							let read_write_count = $crate::benchmarking::read_write_count();
+							frame_support::debug::trace!(
+								target: "benchmark",
+								"Read/Write Count {:?}", read_write_count
+							);
+
+							// Time the storage root recalculation.
+							let start_storage_root = $crate::benchmarking::current_time();
+							$crate::storage_root();
+							let finish_storage_root = $crate::benchmarking::current_time();
+							let elapsed_storage_root = finish_storage_root - start_storage_root;
+
+							results.push($crate::BenchmarkResults {
+								components: c.clone(),
+								extrinsic_time: elapsed_extrinsic,
+								storage_root_time: elapsed_storage_root,
+								reads: read_write_count.0,
+								repeat_reads: read_write_count.1,
+								writes: read_write_count.2,
+								repeat_writes: read_write_count.3,
+							});
+						}
 
 						// Wipe the DB back to the genesis state.
 						$crate::benchmarking::wipe_db();
@@ -893,7 +878,11 @@ macro_rules! impl_benchmark {
 				};
 
 				if components.is_empty() {
-					repeat_benchmark(repeat, Default::default(), &mut results)?;
+					if verify {
+						// If `--verify` is used, run the benchmark once to verify it would complete.
+						repeat_benchmark(1, Default::default(), &mut Vec::new(), true)?;
+					}
+					repeat_benchmark(repeat, Default::default(), &mut results, false)?;
 				} else {
 					// Select the component we will be benchmarking. Each component will be benchmarked.
 					for (idx, (name, low, high)) in components.iter().enumerate() {
@@ -929,7 +918,11 @@ macro_rules! impl_benchmark {
 								)
 								.collect();
 
-							repeat_benchmark(repeat, c, &mut results)?;
+							if verify {
+								// If `--verify` is used, run the benchmark once to verify it would complete.
+								repeat_benchmark(1, Default::default(), &mut Vec::new(), true)?;
+							}
+							repeat_benchmark(repeat, c, &mut results, false)?;
 						}
 					}
 				}
@@ -962,17 +955,17 @@ macro_rules! impl_benchmark_test {
 				let execute_benchmark = |
 					c: Vec<($crate::BenchmarkParameter, u32)>
 				| -> Result<(), &'static str> {
-					// Set up the verification state
+					// Set up the benchmark, return execution + verification function.
 					let closure_to_verify = <
 						SelectedBenchmark as $crate::BenchmarkingSetup<T, _>
-					>::verify(&selected_benchmark, &c)?;
+					>::instance(&selected_benchmark, &c, true)?;
 
 					// Set the block number to at least 1 so events are deposited.
 					if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
 						frame_system::Module::<T>::set_block_number(1.into());
 					}
 
-					// Run verification
+					// Run execution + verification
 					closure_to_verify()?;
 
 					// Reset the state
@@ -1015,7 +1008,7 @@ macro_rules! impl_benchmark_test {
 /// First create an object that holds in the input parameters for the benchmark:
 ///
 /// ```ignore
-/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist);
+/// let params = (&config, &whitelist);
 /// ```
 ///
 /// The `whitelist` is a parameter you pass to control the DB read/write tracking.
@@ -1059,18 +1052,29 @@ macro_rules! impl_benchmark_test {
 macro_rules! add_benchmark {
 	( $params:ident, $batches:ident, $name:ident, $( $location:tt )* ) => (
 		let name_string = stringify!($name).as_bytes();
-		let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist, extra) = $params;
+		let (config, whitelist) = $params;
+		let $crate::BenchmarkConfig {
+			pallet,
+			benchmark,
+			lowest_range_values,
+			highest_range_values,
+			steps,
+			repeat,
+			verify,
+			extra,
+		} = config;
 		if &pallet[..] == &name_string[..] || &pallet[..] == &b"*"[..] {
 			if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
-				for benchmark in $( $location )*::benchmarks(extra).into_iter() {
+				for benchmark in $( $location )*::benchmarks(*extra).into_iter() {
 					$batches.push($crate::BenchmarkBatch {
 						results: $( $location )*::run_benchmark(
 							benchmark,
 							&lowest_range_values[..],
 							&highest_range_values[..],
 							&steps[..],
-							repeat,
+							*repeat,
 							whitelist,
+							*verify,
 						)?,
 						pallet: name_string.to_vec(),
 						benchmark: benchmark.to_vec(),
@@ -1083,8 +1087,9 @@ macro_rules! add_benchmark {
 						&lowest_range_values[..],
 						&highest_range_values[..],
 						&steps[..],
-						repeat,
+						*repeat,
 						whitelist,
+						*verify,
 					)?,
 					pallet: name_string.to_vec(),
 					benchmark: benchmark.clone(),
diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs
index 127645d430564..94f3574100739 100644
--- a/frame/benchmarking/src/tests.rs
+++ b/frame/benchmarking/src/tests.rs
@@ -176,10 +176,11 @@ fn benchmarks_macro_works() {
 	let closure = <SelectedBenchmark as BenchmarkingSetup<Test>>::instance(
 		&selected,
 		&[(BenchmarkParameter::b, 1)],
+		true,
 	).expect("failed to create closure");
 
 	new_test_ext().execute_with(|| {
-		assert_eq!(closure(), Ok(()));
+		assert_ok!(closure());
 	});
 }
 
@@ -193,6 +194,7 @@ fn benchmarks_macro_rename_works() {
 	let closure = <SelectedBenchmark as BenchmarkingSetup<Test>>::instance(
 		&selected,
 		&[(BenchmarkParameter::b, 1)],
+		true,
 	).expect("failed to create closure");
 
 	new_test_ext().execute_with(|| {
@@ -210,9 +212,10 @@ fn benchmarks_macro_works_for_non_dispatchable() {
 	let closure = <SelectedBenchmark as BenchmarkingSetup<Test>>::instance(
 		&selected,
 		&[(BenchmarkParameter::x, 1)],
+		true,
 	).expect("failed to create closure");
 
-	assert_eq!(closure(), Ok(()));
+	assert_ok!(closure());
 }
 
 #[test]
@@ -220,14 +223,28 @@ fn benchmarks_macro_verify_works() {
 	// Check postcondition for benchmark `set_value` is valid.
 	let selected = SelectedBenchmark::set_value;
 
-	let closure = <SelectedBenchmark as BenchmarkingSetup<Test>>::verify(
+	let closure = <SelectedBenchmark as BenchmarkingSetup<Test>>::instance(
 		&selected,
 		&[(BenchmarkParameter::b, 1)],
+		true,
 	).expect("failed to create closure");
 
 	new_test_ext().execute_with(|| {
 		assert_ok!(closure());
 	});
+
+	// Check postcondition for benchmark `bad_verify` is invalid.
+	let selected = SelectedBenchmark::bad_verify;
+
+	let closure = <SelectedBenchmark as BenchmarkingSetup<Test>>::instance(
+		&selected,
+		&[(BenchmarkParameter::x, 10000)],
+		true,
+	).expect("failed to create closure");
+
+	new_test_ext().execute_with(|| {
+		assert_err!(closure(), "You forgot to sort!");
+	});
 }
 
 #[test]
diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs
index 8c25f035802fc..347334e24d5f6 100644
--- a/frame/benchmarking/src/utils.rs
+++ b/frame/benchmarking/src/utils.rs
@@ -63,19 +63,32 @@ pub struct BenchmarkResults {
 	pub repeat_writes: u32,
 }
 
+/// Configuration used to setup and run runtime benchmarks.
+#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
+pub struct BenchmarkConfig {
+	/// The encoded name of the pallet to benchmark.
+	pub pallet: Vec<u8>,
+	/// The encoded name of the benchmark/extrinsic to run.
+	pub benchmark: Vec<u8>,
+	/// An optional manual override to the lowest values used in the `steps` range.
+	pub lowest_range_values: Vec<u32>,
+	/// An optional manual override to the highest values used in the `steps` range.
+	pub highest_range_values: Vec<u32>,
+	/// The number of samples to take across the range of values for components.
+	pub steps: Vec<u32>,
+	/// The number of times to repeat a benchmark.
+	pub repeat: u32,
+	/// Enable an extra benchmark iteration which runs the verification logic for a benchmark.
+	pub verify: bool,
+	/// Enable benchmarking of "extra" extrinsics, i.e. those that are not directly used in a pallet.
+	pub extra: bool,
+}
+
 sp_api::decl_runtime_apis! {
 	/// Runtime api for benchmarking a FRAME runtime.
 	pub trait Benchmark {
 		/// Dispatch the given benchmark.
-		fn dispatch_benchmark(
-			pallet: Vec<u8>,
-			benchmark: Vec<u8>,
-			lowest_range_values: Vec<u32>,
-			highest_range_values: Vec<u32>,
-			steps: Vec<u32>,
-			repeat: u32,
-			extra: bool,
-		) -> Result<Vec<BenchmarkBatch>, RuntimeString>;
+		fn dispatch_benchmark(config: BenchmarkConfig) -> Result<Vec<BenchmarkBatch>, RuntimeString>;
 	}
 }
 
@@ -175,7 +188,8 @@ pub trait Benchmarking<T> {
 		highest_range_values: &[u32],
 		steps: &[u32],
 		repeat: u32,
-		whitelist: &[TrackedStorageKey]
+		whitelist: &[TrackedStorageKey],
+		verify: bool,
 	) -> Result<Vec<T>, &'static str>;
 }
 
@@ -188,10 +202,11 @@ pub trait BenchmarkingSetup<T, I = ()> {
 	fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>;
 
 	/// Set up the storage, and prepare a closure to run the benchmark.
-	fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>;
-
-	/// Set up the storage, and prepare a closure to test and verify the benchmark
-	fn verify(&self, components: &[(BenchmarkParameter, u32)]) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>;
+	fn instance(
+		&self,
+		components: &[(BenchmarkParameter, u32)],
+		verify: bool
+	) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>;
 }
 
 /// Grab an account, seeded by a name and index.
diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs
index 77eecb2ef04d5..156b2f81c8429 100644
--- a/frame/staking/src/benchmarking.rs
+++ b/frame/staking/src/benchmarking.rs
@@ -715,7 +715,8 @@ mod tests {
 			let closure_to_benchmark =
 				<SelectedBenchmark as frame_benchmarking::BenchmarkingSetup<Test>>::instance(
 					&selected_benchmark,
-					&c
+					&c,
+					true
 				).unwrap();
 
 			assert_ok!(closure_to_benchmark());
diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs
index 688e393bd605a..b0791f88ce5f6 100644
--- a/utils/frame/benchmarking-cli/src/command.rs
+++ b/utils/frame/benchmarking-cli/src/command.rs
@@ -75,6 +75,7 @@ impl BenchmarkCmd {
 				self.highest_range_values.clone(),
 				self.steps.clone(),
 				self.repeat,
+				!self.no_verify,
 				self.extra,
 			).encode(),
 			extensions,
diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs
index c2a228fc86aef..8cbb3c786876c 100644
--- a/utils/frame/benchmarking-cli/src/lib.rs
+++ b/utils/frame/benchmarking-cli/src/lib.rs
@@ -72,6 +72,10 @@ pub struct BenchmarkCmd {
 	#[structopt(long)]
 	pub heap_pages: Option<u64>,
 
+	/// Disable verification logic when running benchmarks.
+	#[structopt(long)]
+	pub no_verify: bool,
+
 	/// Display and run extra benchmarks that would otherwise not be needed for weight construction.
 	#[structopt(long)]
 	pub extra: bool,

From 43cd1d0dff44fe9e7d103bb28790d7c8c0113bf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Silva?=
 <123550+andresilva@users.noreply.github.com>
Date: Mon, 24 Aug 2020 14:29:17 +0100
Subject: [PATCH 009/122] grandpa: always create and send justification if
 there are any subscribers (#6935)

* grandpa: use bytes type for justification rpc notification

* grandpa: always create justification if there are rpc subscribers

* grandpa: wording

* grandpa: replace notify_justification macro with function

* grandpa: prefer Option<&T> over &Option<T>
---
 client/finality-grandpa/rpc/Cargo.toml        |  3 +-
 client/finality-grandpa/rpc/src/lib.rs        |  4 +-
 .../finality-grandpa/rpc/src/notification.rs  |  4 +-
 client/finality-grandpa/src/environment.rs    | 68 ++++++++++++-------
 client/finality-grandpa/src/import.rs         |  3 +-
 client/finality-grandpa/src/notification.rs   | 23 +++++--
 client/finality-grandpa/src/observer.rs       |  9 ++-
 7 files changed, 72 insertions(+), 42 deletions(-)

diff --git a/client/finality-grandpa/rpc/Cargo.toml b/client/finality-grandpa/rpc/Cargo.toml
index 28197405c8db7..6f3014644eaa4 100644
--- a/client/finality-grandpa/rpc/Cargo.toml
+++ b/client/finality-grandpa/rpc/Cargo.toml
@@ -8,9 +8,10 @@ edition = "2018"
 license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
 
 [dependencies]
+sc-finality-grandpa = { version = "0.8.0-rc6", path = "../" }
 sc-rpc = { version = "2.0.0-rc6", path = "../../rpc" }
+sp-core = { version = "2.0.0-rc6", path = "../../../primitives/core" }
 sp-runtime = { version = "2.0.0-rc6", path = "../../../primitives/runtime" }
-sc-finality-grandpa = { version = "0.8.0-rc6", path = "../" }
 finality-grandpa = { version = "0.12.3", features = ["derive-codec"] }
 jsonrpc-core = "14.2.0"
 jsonrpc-core-client = "14.2.0"
diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs
index c00c95c5f776f..5606da42d5947 100644
--- a/client/finality-grandpa/rpc/src/lib.rs
+++ b/client/finality-grandpa/rpc/src/lib.rs
@@ -406,7 +406,7 @@ mod tests {
 
 		// Notify with a header and justification
 		let justification = create_justification();
-		let _ = justification_sender.notify(justification.clone()).unwrap();
+		justification_sender.notify(|| Ok(justification.clone())).unwrap();
 
 		// Inspect what we received
 		let recv = receiver.take(1).wait().flatten().collect::<Vec<_>>();
@@ -418,7 +418,7 @@ mod tests {
 
 		let recv_sub_id: String =
 			serde_json::from_value(json_map["subscription"].take()).unwrap();
-		let recv_justification: Vec<u8> =
+		let recv_justification: sp_core::Bytes =
 			serde_json::from_value(json_map["result"].take()).unwrap();
 		let recv_justification: GrandpaJustification<Block> =
 			Decode::decode(&mut &recv_justification[..]).unwrap();
diff --git a/client/finality-grandpa/rpc/src/notification.rs b/client/finality-grandpa/rpc/src/notification.rs
index 831f4681549a7..fd03a622b2196 100644
--- a/client/finality-grandpa/rpc/src/notification.rs
+++ b/client/finality-grandpa/rpc/src/notification.rs
@@ -23,10 +23,10 @@ use sc_finality_grandpa::GrandpaJustification;
 
 /// An encoded justification proving that the given header has been finalized
 #[derive(Clone, Serialize, Deserialize)]
-pub struct JustificationNotification(Vec<u8>);
+pub struct JustificationNotification(sp_core::Bytes);
 
 impl<Block: BlockT> From<GrandpaJustification<Block>> for JustificationNotification {
 	fn from(notification: GrandpaJustification<Block>) -> Self {
-		JustificationNotification(notification.encode())
+		JustificationNotification(notification.encode().into())
 	}
 }
diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs
index a7a29fe0e8a65..d862372770518 100644
--- a/client/finality-grandpa/src/environment.rs
+++ b/client/finality-grandpa/src/environment.rs
@@ -645,7 +645,8 @@ pub(crate) fn ancestry<Block: BlockT, Client>(
 	client: &Arc<Client>,
 	base: Block::Hash,
 	block: Block::Hash,
-) -> Result<Vec<Block::Hash>, GrandpaError> where
+) -> Result<Vec<Block::Hash>, GrandpaError>
+where
 	Client: HeaderMetadata<Block, Error = sp_blockchain::Error>,
 {
 	if base == block { return Err(GrandpaError::NotDescendent) }
@@ -671,15 +672,14 @@ pub(crate) fn ancestry<Block: BlockT, Client>(
 	Ok(tree_route.retracted().iter().skip(1).map(|e| e.hash).collect())
 }
 
-impl<B, Block: BlockT, C, N, SC, VR>
-	voter::Environment<Block::Hash, NumberFor<Block>>
-for Environment<B, Block, C, N, SC, VR>
+impl<B, Block: BlockT, C, N, SC, VR> voter::Environment<Block::Hash, NumberFor<Block>>
+	for Environment<B, Block, C, N, SC, VR>
 where
 	Block: 'static,
 	B: Backend<Block>,
 	C: crate::ClientForGrandpa<Block, B> + 'static,
 	C::Api: GrandpaApi<Block, Error = sp_blockchain::Error>,
- 	N: NetworkT<Block> + 'static + Send + Sync,
+	N: NetworkT<Block> + 'static + Send + Sync,
 	SC: SelectChain<Block> + 'static,
 	VR: VotingRule<Block, C>,
 	NumberFor<Block>: BlockNumberOps,
@@ -1023,7 +1023,7 @@ where
 			number,
 			(round, commit).into(),
 			false,
-			&self.justification_sender,
+			self.justification_sender.as_ref(),
 		)
 	}
 
@@ -1088,9 +1088,10 @@ pub(crate) fn finalize_block<BE, Block, Client>(
 	number: NumberFor<Block>,
 	justification_or_commit: JustificationOrCommit<Block>,
 	initial_sync: bool,
-	justification_sender: &Option<GrandpaJustificationSender<Block>>,
-) -> Result<(), CommandOrError<Block::Hash, NumberFor<Block>>> where
-	Block:  BlockT,
+	justification_sender: Option<&GrandpaJustificationSender<Block>>,
+) -> Result<(), CommandOrError<Block::Hash, NumberFor<Block>>>
+where
+	Block: BlockT,
 	BE: Backend<Block>,
 	Client: crate::ClientForGrandpa<Block, BE>,
 {
@@ -1154,6 +1155,18 @@ pub(crate) fn finalize_block<BE, Block, Client>(
 			}
 		}
 
+		// send a justification notification if a sender exists and in case of error log it.
+		fn notify_justification<Block: BlockT>(
+			justification_sender: Option<&GrandpaJustificationSender<Block>>,
+			justification: impl FnOnce() -> Result<GrandpaJustification<Block>, Error>,
+		) {
+			if let Some(sender) = justification_sender {
+				if let Err(err) = sender.notify(justification) {
+					warn!(target: "afg", "Error creating justification for subscriber: {:?}", err);
+				}
+			}
+		}
+
 		// NOTE: this code assumes that honest voters will never vote past a
 		// transition block, thus we don't have to worry about the case where
 		// we have a transition with `effective_block = N`, but we finalize
@@ -1161,7 +1174,10 @@ pub(crate) fn finalize_block<BE, Block, Client>(
 		// justifications for transition blocks which will be requested by
 		// syncing clients.
 		let justification = match justification_or_commit {
-			JustificationOrCommit::Justification(justification) => Some(justification),
+			JustificationOrCommit::Justification(justification) => {
+				notify_justification(justification_sender, || Ok(justification.clone()));
+				Some(justification.encode())
+			},
 			JustificationOrCommit::Commit((round_number, commit)) => {
 				let mut justification_required =
 					// justification is always required when block that enacts new authorities
@@ -1181,29 +1197,31 @@ pub(crate) fn finalize_block<BE, Block, Client>(
 					}
 				}
 
+				// NOTE: the code below is a bit more verbose because we
+				// really want to avoid creating a justification if it isn't
+				// needed (e.g. if there's no subscribers), and also to avoid
+				// creating it twice. depending on the vote tree for the round,
+				// creating a justification might require multiple fetches of
+				// headers from the database.
+				let justification = || GrandpaJustification::from_commit(
+					&client,
+					round_number,
+					commit,
+				);
+
 				if justification_required {
-					let justification = GrandpaJustification::from_commit(
-						&client,
-						round_number,
-						commit,
-					)?;
+					let justification = justification()?;
+					notify_justification(justification_sender, || Ok(justification.clone()));
 
-					Some(justification)
+					Some(justification.encode())
 				} else {
+					notify_justification(justification_sender, justification);
+
 					None
 				}
 			},
 		};
 
-		// Notify any registered listeners in case we have a justification
-		if let Some(sender) = justification_sender {
-			if let Some(ref justification) = justification {
-				let _ = sender.notify(justification.clone());
-			}
-		}
-
-		let justification = justification.map(|j| j.encode());
-
 		debug!(target: "afg", "Finalizing blocks up to ({:?}, {})", number, hash);
 
 		// ideally some handle to a synchronization oracle would be used
diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs
index d5b0a650096ca..04df95a3187e1 100644
--- a/client/finality-grandpa/src/import.rs
+++ b/client/finality-grandpa/src/import.rs
@@ -619,7 +619,6 @@ where
 	Client: crate::ClientForGrandpa<Block, BE>,
 	NumberFor<Block>: finality_grandpa::BlockNumberOps,
 {
-
 	/// Import a block justification and finalize the block.
 	///
 	/// If `enacts_change` is set to true, then finalizing this block *must*
@@ -653,7 +652,7 @@ where
 			number,
 			justification.into(),
 			initial_sync,
-			&Some(self.justification_sender.clone()),
+			Some(&self.justification_sender),
 		);
 
 		match result {
diff --git a/client/finality-grandpa/src/notification.rs b/client/finality-grandpa/src/notification.rs
index 16f705f0eeb1f..8415583051902 100644
--- a/client/finality-grandpa/src/notification.rs
+++ b/client/finality-grandpa/src/notification.rs
@@ -20,9 +20,10 @@ use std::sync::Arc;
 use parking_lot::Mutex;
 
 use sp_runtime::traits::Block as BlockT;
-use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver};
+use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
 
 use crate::justification::GrandpaJustification;
+use crate::Error;
 
 // Stream of justifications returned when subscribing.
 type JustificationStream<Block> = TracingUnboundedReceiver<GrandpaJustification<Block>>;
@@ -54,10 +55,22 @@ impl<Block: BlockT> GrandpaJustificationSender<Block> {
 
 	/// Send out a notification to all subscribers that a new justification
 	/// is available for a block.
-	pub fn notify(&self, notification: GrandpaJustification<Block>) -> Result<(), ()> {
-		self.subscribers.lock().retain(|n| {
-			!n.is_closed() && n.unbounded_send(notification.clone()).is_ok()
-		});
+	pub fn notify(
+		&self,
+		justification: impl FnOnce() -> Result<GrandpaJustification<Block>, Error>,
+	) -> Result<(), Error> {
+		let mut subscribers = self.subscribers.lock();
+
+		// do an initial prune on closed subscriptions
+		subscribers.retain(|n| !n.is_closed());
+
+		// if there's no subscribers we avoid creating
+		// the justification which is a costly operation
+		if !subscribers.is_empty() {
+			let justification = justification()?;
+			subscribers.retain(|n| n.unbounded_send(justification.clone()).is_ok());
+		}
+
 		Ok(())
 	}
 }
diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs
index 8fb536a369751..6a9955aa86d81 100644
--- a/client/finality-grandpa/src/observer.rs
+++ b/client/finality-grandpa/src/observer.rs
@@ -74,11 +74,10 @@ fn grandpa_observer<BE, Block: BlockT, Client, S, F>(
 	last_finalized_number: NumberFor<Block>,
 	commits: S,
 	note_round: F,
-) -> impl Future<Output=Result<(), CommandOrError<Block::Hash, NumberFor<Block>>>> where
+) -> impl Future<Output = Result<(), CommandOrError<Block::Hash, NumberFor<Block>>>>
+where
 	NumberFor<Block>: BlockNumberOps,
-	S: Stream<
-		Item = Result<CommunicationIn<Block>, CommandOrError<Block::Hash, NumberFor<Block>>>,
-	>,
+	S: Stream<Item = Result<CommunicationIn<Block>, CommandOrError<Block::Hash, NumberFor<Block>>>>,
 	F: Fn(u64),
 	BE: Backend<Block>,
 	Client: crate::ClientForGrandpa<Block, BE>,
@@ -130,7 +129,7 @@ fn grandpa_observer<BE, Block: BlockT, Client, S, F>(
 				finalized_number,
 				(round, commit).into(),
 				false,
-				&justification_sender,
+				justification_sender.as_ref(),
 			) {
 				Ok(_) => {},
 				Err(e) => return future::err(e),

From 0de545129d1b74fda4f6d105310b1ab44287a181 Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Mon, 24 Aug 2020 15:37:07 +0200
Subject: [PATCH 010/122] .maintain/monitoring/alerting-rules: Add fd alert
 (#6946)

Alert on high file descriptor allocation.
---
 .../monitoring/alerting-rules/alerting-rules.yaml   | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.maintain/monitoring/alerting-rules/alerting-rules.yaml b/.maintain/monitoring/alerting-rules/alerting-rules.yaml
index 12f46e17ad85b..7f36fedb4ba67 100644
--- a/.maintain/monitoring/alerting-rules/alerting-rules.yaml
+++ b/.maintain/monitoring/alerting-rules/alerting-rules.yaml
@@ -109,6 +109,19 @@ groups:
       message: 'The node {{ $labels.instance }} has less than 3 peers for more
       than 15 minutes'
 
+  ##############################################################################
+  # System
+  ##############################################################################
+
+  - alert: HighNumberOfFileDescriptors
+    expr: 'node_filefd_allocated{domain=~"kusama|polkadot"} > 10000'
+    for: 3m
+    labels:
+      severity: warning
+    annotations:
+      message: 'The node {{ $labels.instance }} has more than 10_000 file
+      descriptors allocated for more than 3 minutes'
+
   ##############################################################################
   # Others
   ##############################################################################

From 20d4ca4b8c186c82aa180034b069c3b12f51c3c1 Mon Sep 17 00:00:00 2001
From: Guillaume Thiolliere <gui.thiolliere@gmail.com>
Date: Mon, 24 Aug 2020 15:40:16 +0200
Subject: [PATCH 011/122] Fix benchmark read/write key tracker for keys in
 child storages. (#6905)

* WIP: read child trie and write child trie

* add test

* refactor a bit + improve log

* better naming

* trigger CI

* Revert "trigger CI"

This reverts commit d0aadaeb6a12fc6c39f01b3c1b5725d19f085865.
---
 client/db/src/bench.rs                  | 150 +++++++++++++++++++-----
 primitives/state-machine/src/backend.rs |  10 +-
 primitives/state-machine/src/ext.rs     |   1 +
 3 files changed, 128 insertions(+), 33 deletions(-)

diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs
index 93b8048529fc4..1c9be87faa0f3 100644
--- a/client/db/src/bench.rs
+++ b/client/db/src/bench.rs
@@ -30,7 +30,9 @@ use sp_core::{
 };
 use sp_runtime::traits::{Block as BlockT, HashFor};
 use sp_runtime::Storage;
-use sp_state_machine::{DBValue, backend::Backend as StateBackend, StorageCollection};
+use sp_state_machine::{
+	DBValue, backend::Backend as StateBackend, StorageCollection, ChildStorageCollection
+};
 use kvdb::{KeyValueDB, DBTransaction};
 use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};
 
@@ -96,7 +98,11 @@ pub struct BenchmarkingState<B: BlockT> {
 	genesis: HashMap<Vec<u8>, (Vec<u8>, i32)>,
 	record: Cell<Vec<Vec<u8>>>,
 	shared_cache: SharedCache<B>, // shared cache is always empty
-	key_tracker: RefCell<HashMap<Vec<u8>, KeyTracker>>,
+	/// Key tracker for keys in the main trie.
+	main_key_tracker: RefCell<HashMap<Vec<u8>, KeyTracker>>,
+	/// Key tracker for keys in a child trie.
+	/// Child trie are identified by their storage key (i.e. `ChildInfo::storage_key()`)
+	child_key_tracker: RefCell<HashMap<Vec<u8>, HashMap<Vec<u8>, KeyTracker>>>,
 	read_write_tracker: RefCell<ReadWriteTracker>,
 	whitelist: RefCell<Vec<TrackedStorageKey>>,
 }
@@ -116,7 +122,8 @@ impl<B: BlockT> BenchmarkingState<B> {
 			genesis_root: Default::default(),
 			record: Default::default(),
 			shared_cache: new_shared_cache(0, (1, 10)),
-			key_tracker: Default::default(),
+			main_key_tracker: Default::default(),
+			child_key_tracker: Default::default(),
 			read_write_tracker: Default::default(),
 			whitelist: Default::default(),
 		};
@@ -134,7 +141,7 @@ impl<B: BlockT> BenchmarkingState<B> {
 		);
 		state.genesis = transaction.clone().drain();
 		state.genesis_root = root.clone();
-		state.commit(root, transaction, Vec::new())?;
+		state.commit(root, transaction, Vec::new(), Vec::new())?;
 		state.record.take();
 		Ok(state)
 	}
@@ -156,7 +163,7 @@ impl<B: BlockT> BenchmarkingState<B> {
 	}
 
 	fn add_whitelist_to_tracker(&self) {
-		let mut key_tracker = self.key_tracker.borrow_mut();
+		let mut main_key_tracker = self.main_key_tracker.borrow_mut();
 
 		let whitelist = self.whitelist.borrow();
 
@@ -165,25 +172,29 @@ impl<B: BlockT> BenchmarkingState<B> {
 				has_been_read: key.has_been_read,
 				has_been_written: key.has_been_written,
 			};
-			key_tracker.insert(key.key.clone(), whitelisted);
+			main_key_tracker.insert(key.key.clone(), whitelisted);
 		});
 	}
 
 	fn wipe_tracker(&self) {
-		*self.key_tracker.borrow_mut() = HashMap::new();
+		*self.main_key_tracker.borrow_mut() = HashMap::new();
 		self.add_whitelist_to_tracker();
 		*self.read_write_tracker.borrow_mut() = Default::default();
 	}
 
-	fn add_read_key(&self, key: &[u8]) {
-		log::trace!(target: "benchmark", "Read: {}", HexDisplay::from(&key));
-
-		let mut key_tracker = self.key_tracker.borrow_mut();
+	// Childtrie is identified by its storage key (i.e. `ChildInfo::storage_key`)
+	fn add_read_key(&self, childtrie: Option<&[u8]>, key: &[u8]) {
 		let mut read_write_tracker = self.read_write_tracker.borrow_mut();
+		let mut child_key_tracker = self.child_key_tracker.borrow_mut();
+		let mut main_key_tracker = self.main_key_tracker.borrow_mut();
 
-		let maybe_tracker = key_tracker.get(key);
+		let key_tracker = if let Some(childtrie) = childtrie {
+			child_key_tracker.entry(childtrie.to_vec()).or_insert_with(|| HashMap::new())
+	 	} else {
+			&mut main_key_tracker
+		};
 
-		match maybe_tracker {
+		let read = match key_tracker.get(key) {
 			None => {
 				let has_been_read = KeyTracker {
 					has_been_read: true,
@@ -191,6 +202,7 @@ impl<B: BlockT> BenchmarkingState<B> {
 				};
 				key_tracker.insert(key.to_vec(), has_been_read);
 				read_write_tracker.add_read();
+				true
 			},
 			Some(tracker) => {
 				if !tracker.has_been_read {
@@ -200,20 +212,37 @@ impl<B: BlockT> BenchmarkingState<B> {
 					};
 					key_tracker.insert(key.to_vec(), has_been_read);
 					read_write_tracker.add_read();
+					true
 				} else {
 					read_write_tracker.add_repeat_read();
+					false
 				}
 			}
+		};
+
+		if read {
+			if let Some(childtrie) = childtrie {
+				log::trace!(
+					target: "benchmark",
+					"Childtrie Read: {} {}", HexDisplay::from(&childtrie), HexDisplay::from(&key)
+				);
+			} else {
+				log::trace!(target: "benchmark", "Read: {}", HexDisplay::from(&key));
+			}
 		}
 	}
 
-	fn add_write_key(&self, key: &[u8]) {
-		log::trace!(target: "benchmark", "Write: {}", HexDisplay::from(&key));
-
-		let mut key_tracker = self.key_tracker.borrow_mut();
+	// Childtrie is identified by its storage key (i.e. `ChildInfo::storage_key`)
+	fn add_write_key(&self, childtrie: Option<&[u8]>, key: &[u8]) {
 		let mut read_write_tracker = self.read_write_tracker.borrow_mut();
+		let mut child_key_tracker = self.child_key_tracker.borrow_mut();
+		let mut main_key_tracker = self.main_key_tracker.borrow_mut();
 
-		let maybe_tracker = key_tracker.get(key);
+		let key_tracker = if let Some(childtrie) = childtrie {
+			child_key_tracker.entry(childtrie.to_vec()).or_insert_with(|| HashMap::new())
+	 	} else {
+			&mut main_key_tracker
+		};
 
 		// If we have written to the key, we also consider that we have read from it.
 		let has_been_written = KeyTracker {
@@ -221,19 +250,33 @@ impl<B: BlockT> BenchmarkingState<B> {
 			has_been_written: true,
 		};
 
-		match maybe_tracker {
+		let write = match key_tracker.get(key) {
 			None => {
 				key_tracker.insert(key.to_vec(), has_been_written);
 				read_write_tracker.add_write();
+				true
 			},
 			Some(tracker) => {
 				if !tracker.has_been_written {
 					key_tracker.insert(key.to_vec(), has_been_written);
 					read_write_tracker.add_write();
+					true
 				} else {
 					read_write_tracker.add_repeat_write();
+					false
 				}
 			}
+		};
+
+		if write {
+			if let Some(childtrie) = childtrie {
+				log::trace!(
+					target: "benchmark",
+					"Childtrie Write: {} {}", HexDisplay::from(&childtrie), HexDisplay::from(&key)
+				);
+			} else {
+				log::trace!(target: "benchmark", "Write: {}", HexDisplay::from(&key));
+			}
 		}
 	}
 }
@@ -248,12 +291,12 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 	type TrieBackendStorage = <DbState<B> as StateBackend<HashFor<B>>>::TrieBackendStorage;
 
 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		self.add_read_key(key);
+		self.add_read_key(None, key);
 		self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key)
 	}
 
 	fn storage_hash(&self, key: &[u8]) -> Result<Option<B::Hash>, Self::Error> {
-		self.add_read_key(key);
+		self.add_read_key(None, key);
 		self.state.borrow().as_ref().ok_or_else(state_err)?.storage_hash(key)
 	}
 
@@ -262,12 +305,12 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 		child_info: &ChildInfo,
 		key: &[u8],
 	) -> Result<Option<Vec<u8>>, Self::Error> {
-		self.add_read_key(key);
+		self.add_read_key(Some(child_info.storage_key()), key);
 		self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(child_info, key)
 	}
 
 	fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
-		self.add_read_key(key);
+		self.add_read_key(None, key);
 		self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key)
 	}
 
@@ -276,12 +319,12 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 		child_info: &ChildInfo,
 		key: &[u8],
 	) -> Result<bool, Self::Error> {
-		self.add_read_key(key);
+		self.add_read_key(Some(child_info.storage_key()), key);
 		self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(child_info, key)
 	}
 
 	fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		self.add_read_key(key);
+		self.add_read_key(None, key);
 		self.state.borrow().as_ref().ok_or_else(state_err)?.next_storage_key(key)
 	}
 
@@ -290,7 +333,7 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 		child_info: &ChildInfo,
 		key: &[u8],
 	) -> Result<Option<Vec<u8>>, Self::Error> {
-		self.add_read_key(key);
+		self.add_read_key(Some(child_info.storage_key()), key);
 		self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(child_info, key)
 	}
 
@@ -367,9 +410,9 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 	fn commit(&self,
 		storage_root: <HashFor<B> as Hasher>::Out,
 		mut transaction: Self::Transaction,
-		storage_changes: StorageCollection,
-	) -> Result<(), Self::Error>
-	{
+		main_storage_changes: StorageCollection,
+		child_storage_changes: ChildStorageCollection,
+	) -> Result<(), Self::Error> {
 		if let Some(db) = self.db.take() {
 			let mut db_transaction = DBTransaction::new();
 			let changes = transaction.drain();
@@ -390,8 +433,13 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 			self.db.set(Some(db));
 
 			// Track DB Writes
-			storage_changes.iter().for_each(|(key, _)| {
-				self.add_write_key(key);
+			main_storage_changes.iter().for_each(|(key, _)| {
+				self.add_write_key(None, key);
+			});
+			child_storage_changes.iter().for_each(|(child_storage_key, storage_changes)| {
+				storage_changes.iter().for_each(|(key, _)| {
+					self.add_write_key(Some(child_storage_key), key);
+				})
 			});
 		} else {
 			return Err("Trying to commit to a closed db".into())
@@ -453,3 +501,43 @@ impl<Block: BlockT> std::fmt::Debug for BenchmarkingState<Block> {
 		write!(f, "Bench DB")
 	}
 }
+
+#[cfg(test)]
+mod test {
+	use crate::bench::BenchmarkingState;
+	use sp_state_machine::backend::Backend as _;
+
+	#[test]
+	fn read_to_main_and_child_tries() {
+		let bench_state = BenchmarkingState::<crate::tests::Block>::new(Default::default(), None)
+			.unwrap();
+
+		let child1 = sp_core::storage::ChildInfo::new_default(b"child1");
+		let child2 = sp_core::storage::ChildInfo::new_default(b"child2");
+
+		bench_state.storage(b"foo").unwrap();
+		bench_state.child_storage(&child1, b"foo").unwrap();
+		bench_state.child_storage(&child2, b"foo").unwrap();
+
+		bench_state.storage(b"bar").unwrap();
+		bench_state.child_storage(&child1, b"bar").unwrap();
+		bench_state.child_storage(&child2, b"bar").unwrap();
+
+		bench_state.commit(
+			Default::default(),
+			Default::default(),
+			vec![
+				("foo".as_bytes().to_vec(), None)
+			],
+			vec![
+				("child1".as_bytes().to_vec(), vec![("foo".as_bytes().to_vec(), None)])
+			]
+		).unwrap();
+
+		let rw_tracker = bench_state.read_write_tracker.borrow();
+		assert_eq!(rw_tracker.reads, 6);
+		assert_eq!(rw_tracker.repeat_reads, 0);
+		assert_eq!(rw_tracker.writes, 2);
+		assert_eq!(rw_tracker.repeat_writes, 0);
+	}
+}
diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs
index cfff2c6fc6967..6ced5ed0e521e 100644
--- a/primitives/state-machine/src/backend.rs
+++ b/primitives/state-machine/src/backend.rs
@@ -26,7 +26,7 @@ use sp_core::{
 use crate::{
 	trie_backend::TrieBackend,
 	trie_backend_essence::TrieBackendStorage,
-	UsageInfo, StorageKey, StorageValue, StorageCollection,
+	UsageInfo, StorageKey, StorageValue, StorageCollection, ChildStorageCollection,
 };
 
 /// A state backend is used to read state data and can have changes committed
@@ -215,7 +215,13 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
 	}
 
 	/// Commit given transaction to storage.
-	fn commit(&self, _: H::Out, _: Self::Transaction, _: StorageCollection) -> Result<(), Self::Error> {
+	fn commit(
+		&self,
+		_: H::Out,
+		_: Self::Transaction,
+		_: StorageCollection,
+		_: ChildStorageCollection,
+	) -> Result<(), Self::Error> {
 		unimplemented!()
 	}
 
diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs
index e57636b300a5f..e36964716f8c9 100644
--- a/primitives/state-machine/src/ext.rs
+++ b/primitives/state-machine/src/ext.rs
@@ -594,6 +594,7 @@ where
 			changes.transaction_storage_root,
 			changes.transaction,
 			changes.main_storage_changes,
+			changes.child_storage_changes,
 		).expect(EXT_NOT_ALLOWED_TO_FAIL);
 		self.mark_dirty();
 		self.overlay

From 4b3197c98084634e1a7b4453555f06485fb3f7a0 Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Mon, 24 Aug 2020 16:17:39 +0200
Subject: [PATCH 012/122] client/authority-discovery: Limit number of addresses
 per authority (#6947)

* client/authority-discovery: Test addresses per authority limit

* client/authority-discovery: Limit number of addresses per authority
---
 client/authority-discovery/src/worker.rs      |  4 ++
 .../authority-discovery/src/worker/tests.rs   | 66 +++++++++++++++++++
 2 files changed, 70 insertions(+)

diff --git a/client/authority-discovery/src/worker.rs b/client/authority-discovery/src/worker.rs
index 09cdedd93a19e..232e59d08dd78 100644
--- a/client/authority-discovery/src/worker.rs
+++ b/client/authority-discovery/src/worker.rs
@@ -67,6 +67,9 @@ const LIBP2P_KADEMLIA_BOOTSTRAP_TIME: Duration = Duration::from_secs(30);
 /// discovery module.
 const AUTHORITIES_PRIORITY_GROUP_NAME: &'static str = "authorities";
 
+/// Maximum number of addresses cached per authority. Additional addresses are discarded.
+const MAX_ADDRESSES_PER_AUTHORITY: usize = 10;
+
 /// Role an authority discovery module can run as.
 pub enum Role {
 	/// Actual authority as well as a reference to its key store.
@@ -496,6 +499,7 @@ where
 
 				false // `protocol` is not a [`Protocol::P2p`], let's keep looking.
 			}))
+			.take(MAX_ADDRESSES_PER_AUTHORITY)
 			.collect();
 
 		if !remote_addresses.is_empty() {
diff --git a/client/authority-discovery/src/worker/tests.rs b/client/authority-discovery/src/worker/tests.rs
index 4b16b9040b8dc..baa6bd0fc7d62 100644
--- a/client/authority-discovery/src/worker/tests.rs
+++ b/client/authority-discovery/src/worker/tests.rs
@@ -617,6 +617,72 @@ fn never_add_own_address_to_priority_group() {
 	);
 }
 
+#[test]
+fn limit_number_of_addresses_added_to_cache_per_authority() {
+	let remote_key_store = KeyStore::new();
+	let remote_public = remote_key_store
+		.write()
+		.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)
+		.unwrap();
+
+	let dht_event = {
+		let addresses = (0..100).map(|_| {
+			let peer_id = PeerId::random();
+			let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
+			address.with(multiaddr::Protocol::P2p(
+				peer_id.into(),
+			)).to_vec()
+		}).collect();
+
+		let mut serialized_addresses = vec![];
+		schema::AuthorityAddresses { addresses }
+		.encode(&mut serialized_addresses)
+			.map_err(Error::EncodingProto)
+			.unwrap();
+
+		let signature = remote_key_store.read()
+			.sign_with(
+				key_types::AUTHORITY_DISCOVERY,
+				&remote_public.clone().into(),
+				serialized_addresses.as_slice(),
+			)
+			.map_err(|_| Error::Signing)
+			.unwrap();
+
+		let mut signed_addresses = vec![];
+		schema::SignedAuthorityAddresses {
+			addresses: serialized_addresses.clone(),
+			signature,
+		}
+			.encode(&mut signed_addresses)
+			.map_err(Error::EncodingProto)
+			.unwrap();
+
+		let key = hash_authority_id(&remote_public.to_raw_vec());
+		let value = signed_addresses;
+		(key, value)
+	};
+
+	let (_dht_event_tx, dht_event_rx) = channel(1);
+
+	let (_to_worker, from_service) = mpsc::channel(0);
+	let mut worker = Worker::new(
+		from_service,
+		Arc::new(TestApi { authorities: vec![remote_public.into()] }),
+		Arc::new(TestNetwork::default()),
+		vec![],
+		dht_event_rx.boxed(),
+		Role::Sentry,
+		None,
+	);
+
+	worker.handle_dht_value_found_event(vec![dht_event]).unwrap();
+	assert_eq!(
+		MAX_ADDRESSES_PER_AUTHORITY,
+		worker.addr_cache.get_addresses_by_authority_id(&remote_public.into()).unwrap().len(),
+	);
+}
+
 #[test]
 fn do_not_cache_addresses_without_peer_id() {
 	let remote_key_store = KeyStore::new();

From 9f590dfe60ca4257857df62a7b389aee72ae39fc Mon Sep 17 00:00:00 2001
From: Swezey <matt@swezey.io>
Date: Mon, 24 Aug 2020 09:22:25 -0500
Subject: [PATCH 013/122] =?UTF-8?q?=E2=9B=93=20=E2=9C=A8Add=20ShiftNrg=20N?=
 =?UTF-8?q?etwork=20SS58=20address=20type=20(#6942)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 primitives/core/src/crypto.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs
index ba0ed12568d4b..a8d84eb57cff9 100644
--- a/primitives/core/src/crypto.rs
+++ b/primitives/core/src/crypto.rs
@@ -467,6 +467,8 @@ ss58_address_format!(
 		(18, "darwinia", "Darwinia Chain mainnet, standard account (*25519).")
 	StafiAccount =>
 		(20, "stafi", "Stafi mainnet, standard account (*25519).")
+	ShiftNrg =>
+		(23, "shift", "ShiftNrg mainnet, standard account (*25519).")
 	SubsocialAccount =>
 		(28, "subsocial", "Subsocial network, standard account (*25519).")
 	PhalaAccount =>

From 761ec7cee844025e9eeb0f923a1c0e656c7151fb Mon Sep 17 00:00:00 2001
From: Guillaume Thiolliere <gui.thiolliere@gmail.com>
Date: Mon, 24 Aug 2020 17:33:15 +0200
Subject: [PATCH 014/122] update tracing attribute (#6950)

---
 Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index af32d89981dea..65f5935a1e900 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9395,9 +9395,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.9"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0693bf8d6f2bf22c690fc61a9d21ac69efdbb894a17ed596b9af0f01e64b84b"
+checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada"
 dependencies = [
  "proc-macro2",
  "quote",

From f7791d042fa092868c05bd0edc865597e44f7ac8 Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Tue, 25 Aug 2020 23:40:27 +0200
Subject: [PATCH 015/122] Fix unwraps and other issues with benchmarks (#6957)

* Fix unwraps and other issues with benchmarks

* undo changes to contracts pallet
---
 frame/benchmarking/src/lib.rs | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs
index 03d60dbec5851..6a457d2a5e912 100644
--- a/frame/benchmarking/src/lib.rs
+++ b/frame/benchmarking/src/lib.rs
@@ -660,7 +660,8 @@ macro_rules! benchmark_backend {
 					// Prepare instance
 					let $param = components.iter()
 						.find(|&c| c.0 == $crate::BenchmarkParameter::$param)
-						.unwrap().1;
+						.ok_or("Could not find component in benchmark preparation.")?
+						.1;
 				)*
 				$(
 					let $pre_id : $pre_ty = $pre_ex;
@@ -778,6 +779,10 @@ macro_rules! impl_benchmark {
 					$( stringify!($name) => SelectedBenchmark::$name, )*
 					_ => return Err("Could not find extrinsic."),
 				};
+				let mut results: Vec<$crate::BenchmarkResults> = Vec::new();
+				if repeat == 0 {
+					return Ok(results);
+				}
 
 				// Add whitelist to DB including whitelisted caller
 				let mut whitelist = whitelist.to_vec();
@@ -795,14 +800,13 @@ macro_rules! impl_benchmark {
 				let components = <
 					SelectedBenchmark as $crate::BenchmarkingSetup<T $(, $instance)?>
 				>::components(&selected_benchmark);
-				let mut results: Vec<$crate::BenchmarkResults> = Vec::new();
 
 				// Default number of steps for a component.
 				let mut prev_steps = 10;
 
 				let repeat_benchmark = |
 					repeat: u32,
-					c: Vec<($crate::BenchmarkParameter, u32)>,
+					c: &[($crate::BenchmarkParameter, u32)],
 					results: &mut Vec<$crate::BenchmarkResults>,
 					verify: bool,
 				| -> Result<(), &'static str> {
@@ -812,7 +816,7 @@ macro_rules! impl_benchmark {
 						// benchmark.
 						let closure_to_benchmark = <
 							SelectedBenchmark as $crate::BenchmarkingSetup<T $(, $instance)?>
-						>::instance(&selected_benchmark, &c, verify)?;
+						>::instance(&selected_benchmark, c, verify)?;
 
 						// Set the block number to at least 1 so events are deposited.
 						if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
@@ -860,7 +864,7 @@ macro_rules! impl_benchmark {
 							let elapsed_storage_root = finish_storage_root - start_storage_root;
 
 							results.push($crate::BenchmarkResults {
-								components: c.clone(),
+								components: c.to_vec(),
 								extrinsic_time: elapsed_extrinsic,
 								storage_root_time: elapsed_storage_root,
 								reads: read_write_count.0,
@@ -920,9 +924,9 @@ macro_rules! impl_benchmark {
 
 							if verify {
 								// If `--verify` is used, run the benchmark once to verify it would complete.
-								repeat_benchmark(1, Default::default(), &mut Vec::new(), true)?;
+								repeat_benchmark(1, &c, &mut Vec::new(), true)?;
 							}
-							repeat_benchmark(repeat, c, &mut results, false)?;
+							repeat_benchmark(repeat, &c, &mut results, false)?;
 						}
 					}
 				}

From 127673fb835e99f505efc7c110f1d5fe3cfb7af1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Wed, 26 Aug 2020 09:05:49 +0200
Subject: [PATCH 016/122] Remove implementation of `Randomness for ()` (#6959)

---
 frame/society/src/mock.rs   | 5 +++--
 frame/support/src/traits.rs | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs
index f29dbc8cb17ad..1ca828bf37196 100644
--- a/frame/society/src/mock.rs
+++ b/frame/society/src/mock.rs
@@ -20,7 +20,8 @@
 use super::*;
 
 use frame_support::{
-	impl_outer_origin, parameter_types, ord_parameter_types, traits::{OnInitialize, OnFinalize}
+	impl_outer_origin, parameter_types, ord_parameter_types,
+	traits::{OnInitialize, OnFinalize, TestRandomness},
 };
 use sp_core::H256;
 use sp_runtime::{
@@ -99,7 +100,7 @@ impl pallet_balances::Trait for Test {
 impl Trait for Test {
 	type Event = ();
 	type Currency = pallet_balances::Module<Self>;
-	type Randomness = ();
+	type Randomness = TestRandomness;
 	type CandidateDeposit = CandidateDeposit;
 	type WrongSideDeduction = WrongSideDeduction;
 	type MaxStrikes = MaxStrikes;
diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs
index 72a3850d2d37e..6f50f38a23369 100644
--- a/frame/support/src/traits.rs
+++ b/frame/support/src/traits.rs
@@ -1345,7 +1345,10 @@ pub trait Randomness<Output> {
 	}
 }
 
-impl<Output: Decode + Default> Randomness<Output> for () {
+/// Provides an implementation of [`Randomness`] that should only be used in tests!
+pub struct TestRandomness;
+
+impl<Output: Decode + Default> Randomness<Output> for TestRandomness {
 	fn random(subject: &[u8]) -> Output {
 		Output::decode(&mut TrailingZeroInput::new(subject)).unwrap_or_default()
 	}

From 144192cef8d8e360662a7fc8efee06042e5bbd37 Mon Sep 17 00:00:00 2001
From: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Date: Wed, 26 Aug 2020 10:15:12 +0200
Subject: [PATCH 017/122] Fix staking fuzzer. (#6954)

---
 frame/staking/fuzzer/src/submit_solution.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frame/staking/fuzzer/src/submit_solution.rs b/frame/staking/fuzzer/src/submit_solution.rs
index 6812a739c4929..9158331726ab3 100644
--- a/frame/staking/fuzzer/src/submit_solution.rs
+++ b/frame/staking/fuzzer/src/submit_solution.rs
@@ -166,7 +166,7 @@ fn main() {
 							DispatchError::Module {
 								index: 0,
 								error: 16,
-								message: Some("PhragmenWeakSubmission"),
+								message: Some("OffchainElectionWeakSubmission"),
 							},
 						);
 					},

From 1f33b6efa6c7b8f595ff2df8288f70d36048ed47 Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Wed, 26 Aug 2020 14:27:30 +0200
Subject: [PATCH 018/122] Enforce that ProtocolId is a string (#6953)

* Enforce that ProtocolId is a string

* Fix test
---
 client/network/src/behaviour.rs               | 18 ++++++-------
 client/network/src/block_requests.rs          | 20 +++++++--------
 client/network/src/config.rs                  | 25 ++++++++++++-------
 client/network/src/discovery.rs               | 12 ++++-----
 client/network/src/finality_requests.rs       |  2 +-
 client/network/src/gossip/tests.rs            |  2 +-
 client/network/src/light_client_handler.rs    |  6 ++---
 client/network/src/protocol.rs                |  4 +--
 .../src/protocol/generic_proto/tests.rs       |  2 +-
 .../protocol/generic_proto/upgrade/legacy.rs  |  2 +-
 client/network/src/service.rs                 | 17 ++++++-------
 client/network/src/service/tests.rs           |  2 +-
 client/network/test/src/lib.rs                |  4 +--
 client/service/src/builder.rs                 |  2 +-
 14 files changed, 61 insertions(+), 57 deletions(-)

diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs
index a43c61054d974..20b5adf76b832 100644
--- a/client/network/src/behaviour.rs
+++ b/client/network/src/behaviour.rs
@@ -80,7 +80,7 @@ pub enum BehaviourOut<B: BlockT> {
 		/// Peer which sent us a request.
 		peer: PeerId,
 		/// Protocol name of the request.
-		protocol: Vec<u8>,
+		protocol: String,
 		/// Time it took to build the response.
 		build_time: Duration,
 	},
@@ -88,14 +88,14 @@ pub enum BehaviourOut<B: BlockT> {
 	RequestStarted {
 		peer: PeerId,
 		/// Protocol name of the request.
-		protocol: Vec<u8>,
+		protocol: String,
 	},
 	/// Finished, successfully or not, a previously-started request.
 	RequestFinished {
 		/// Who we were requesting.
 		peer: PeerId,
 		/// Protocol name of the request.
-		protocol: Vec<u8>,
+		protocol: String,
 		/// How long before the response came or the request got cancelled.
 		request_duration: Duration,
 	},
@@ -300,18 +300,18 @@ Behaviour<B, H> {
 					block_requests::SendRequestOutcome::Ok => {
 						self.events.push_back(BehaviourOut::RequestStarted {
 							peer: target,
-							protocol: self.block_requests.protocol_name().to_vec(),
+							protocol: self.block_requests.protocol_name().to_owned(),
 						});
 					},
 					block_requests::SendRequestOutcome::Replaced { request_duration, .. } => {
 						self.events.push_back(BehaviourOut::RequestFinished {
 							peer: target.clone(),
-							protocol: self.block_requests.protocol_name().to_vec(),
+							protocol: self.block_requests.protocol_name().to_owned(),
 							request_duration,
 						});
 						self.events.push_back(BehaviourOut::RequestStarted {
 							peer: target,
-							protocol: self.block_requests.protocol_name().to_vec(),
+							protocol: self.block_requests.protocol_name().to_owned(),
 						});
 					}
 					block_requests::SendRequestOutcome::NotConnected |
@@ -364,14 +364,14 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviourEventProcess<block_requests::Event<B
 			block_requests::Event::AnsweredRequest { peer, total_handling_time } => {
 				self.events.push_back(BehaviourOut::AnsweredRequest {
 					peer,
-					protocol: self.block_requests.protocol_name().to_vec(),
+					protocol: self.block_requests.protocol_name().to_owned(),
 					build_time: total_handling_time,
 				});
 			},
 			block_requests::Event::Response { peer, original_request: _, response, request_duration } => {
 				self.events.push_back(BehaviourOut::RequestFinished {
 					peer: peer.clone(),
-					protocol: self.block_requests.protocol_name().to_vec(),
+					protocol: self.block_requests.protocol_name().to_owned(),
 					request_duration,
 				});
 				let ev = self.substrate.on_block_response(peer, response);
@@ -383,7 +383,7 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviourEventProcess<block_requests::Event<B
 				// we process them by disconnecting the node.
 				self.events.push_back(BehaviourOut::RequestFinished {
 					peer: peer.clone(),
-					protocol: self.block_requests.protocol_name().to_vec(),
+					protocol: self.block_requests.protocol_name().to_owned(),
 					request_duration,
 				});
 				self.substrate.on_block_request_failed(&peer);
diff --git a/client/network/src/block_requests.rs b/client/network/src/block_requests.rs
index d7a12816dd4e1..f1fbe8fb2523e 100644
--- a/client/network/src/block_requests.rs
+++ b/client/network/src/block_requests.rs
@@ -124,7 +124,7 @@ pub struct Config {
 	max_response_len: usize,
 	inactivity_timeout: Duration,
 	request_timeout: Duration,
-	protocol: Bytes,
+	protocol: String,
 }
 
 impl Config {
@@ -143,7 +143,7 @@ impl Config {
 			max_response_len: 16 * 1024 * 1024,
 			inactivity_timeout: Duration::from_secs(15),
 			request_timeout: Duration::from_secs(40),
-			protocol: Bytes::new(),
+			protocol: String::new(),
 		};
 		c.set_protocol(id);
 		c
@@ -184,11 +184,11 @@ impl Config {
 
 	/// Set protocol to use for upgrade negotiation.
 	pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self {
-		let mut v = Vec::new();
-		v.extend_from_slice(b"/");
-		v.extend_from_slice(id.as_bytes());
-		v.extend_from_slice(b"/sync/2");
-		self.protocol = v.into();
+		let mut s = String::new();
+		s.push_str("/");
+		s.push_str(id.as_ref());
+		s.push_str("/sync/2");
+		self.protocol = s;
 		self
 	}
 }
@@ -258,7 +258,7 @@ where
 	}
 
 	/// Returns the libp2p protocol name used on the wire (e.g. `/foo/sync/2`).
-	pub fn protocol_name(&self) -> &[u8] {
+	pub fn protocol_name(&self) -> &str {
 		&self.config.protocol
 	}
 
@@ -322,7 +322,7 @@ where
 				request: buf,
 				original_request: req,
 				max_response_size: self.config.max_response_len,
-				protocol: self.config.protocol.clone(),
+				protocol: self.config.protocol.as_bytes().to_vec().into(),
 			},
 		});
 
@@ -472,7 +472,7 @@ where
 	fn new_handler(&mut self) -> Self::ProtocolsHandler {
 		let p = InboundProtocol {
 			max_request_len: self.config.max_request_len,
-			protocol: self.config.protocol.clone(),
+			protocol: self.config.protocol.as_bytes().to_owned().into(),
 			marker: PhantomData,
 		};
 		let mut cfg = OneShotHandlerConfig::default();
diff --git a/client/network/src/config.rs b/client/network/src/config.rs
index 94b2993b4e6bd..bde34a0a5716a 100644
--- a/client/network/src/config.rs
+++ b/client/network/src/config.rs
@@ -48,6 +48,7 @@ use std::{
 	io::{self, Write},
 	net::Ipv4Addr,
 	path::{Path, PathBuf},
+	str,
 	sync::Arc,
 };
 use zeroize::Zeroize;
@@ -233,20 +234,26 @@ impl<H: ExHashT + Default, B: BlockT> TransactionPool<H, B> for EmptyTransaction
 	fn transaction(&self, _h: &H) -> Option<B::Extrinsic> { None }
 }
 
-/// Name of a protocol, transmitted on the wire. Should be unique for each chain.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+/// Name of a protocol, transmitted on the wire. Should be unique for each chain. Always UTF-8.
+#[derive(Clone, PartialEq, Eq, Hash)]
 pub struct ProtocolId(smallvec::SmallVec<[u8; 6]>);
 
-impl<'a> From<&'a [u8]> for ProtocolId {
-	fn from(bytes: &'a [u8]) -> ProtocolId {
-		ProtocolId(bytes.into())
+impl<'a> From<&'a str> for ProtocolId {
+	fn from(bytes: &'a str) -> ProtocolId {
+		ProtocolId(bytes.as_bytes().into())
 	}
 }
 
-impl ProtocolId {
-	/// Exposes the `ProtocolId` as bytes.
-	pub fn as_bytes(&self) -> &[u8] {
-		self.0.as_ref()
+impl AsRef<str> for ProtocolId {
+	fn as_ref(&self) -> &str {
+		str::from_utf8(&self.0[..])
+			.expect("the only way to build a ProtocolId is through a UTF-8 String; qed")
+	}
+}
+
+impl fmt::Debug for ProtocolId {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		fmt::Debug::fmt(self.as_ref(), f)
 	}
 }
 
diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs
index e349b08c41dc3..51ee224a9378b 100644
--- a/client/network/src/discovery.rs
+++ b/client/network/src/discovery.rs
@@ -752,7 +752,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 // `DiscoveryBehaviour::new_handler` is still correct.
 fn protocol_name_from_protocol_id(id: &ProtocolId) -> Vec<u8> {
 	let mut v = vec![b'/'];
-	v.extend_from_slice(id.as_bytes());
+	v.extend_from_slice(id.as_ref().as_bytes());
 	v.extend_from_slice(b"/kad");
 	v
 }
@@ -773,7 +773,7 @@ mod tests {
 	#[test]
 	fn discovery_working() {
 		let mut first_swarm_peer_id_and_addr = None;
-		let protocol_id = ProtocolId::from(b"dot".as_ref());
+		let protocol_id = ProtocolId::from("dot");
 
 		// Build swarms whose behaviour is `DiscoveryBehaviour`, each aware of
 		// the first swarm via `with_user_defined`.
@@ -877,8 +877,8 @@ mod tests {
 
 	#[test]
 	fn discovery_ignores_peers_with_unknown_protocols() {
-		let supported_protocol_id = ProtocolId::from(b"a".as_ref());
-		let unsupported_protocol_id = ProtocolId::from(b"b".as_ref());
+		let supported_protocol_id = ProtocolId::from("a");
+		let unsupported_protocol_id = ProtocolId::from("b");
 
 		let mut discovery = {
 			let keypair = Keypair::generate_ed25519();
@@ -929,8 +929,8 @@ mod tests {
 
 	#[test]
 	fn discovery_adds_peer_to_kademlia_of_same_protocol_only() {
-		let protocol_a = ProtocolId::from(b"a".as_ref());
-		let protocol_b = ProtocolId::from(b"b".as_ref());
+		let protocol_a = ProtocolId::from("a");
+		let protocol_b = ProtocolId::from("b");
 
 		let mut discovery = {
 			let keypair = Keypair::generate_ed25519();
diff --git a/client/network/src/finality_requests.rs b/client/network/src/finality_requests.rs
index de737cdd20a4e..9b99521ba681b 100644
--- a/client/network/src/finality_requests.rs
+++ b/client/network/src/finality_requests.rs
@@ -129,7 +129,7 @@ impl Config {
 	pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self {
 		let mut v = Vec::new();
 		v.extend_from_slice(b"/");
-		v.extend_from_slice(id.as_bytes());
+		v.extend_from_slice(id.as_ref().as_bytes());
 		v.extend_from_slice(b"/finality-proof/1");
 		self.protocol = v.into();
 		self
diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs
index 9b16e057461bf..6c3e26da13c64 100644
--- a/client/network/src/gossip/tests.rs
+++ b/client/network/src/gossip/tests.rs
@@ -100,7 +100,7 @@ fn build_test_full_node(config: config::NetworkConfiguration)
 		finality_proof_request_builder: None,
 		on_demand: None,
 		transaction_pool: Arc::new(crate::config::EmptyTransactionPool),
-		protocol_id: config::ProtocolId::from(&b"/test-protocol-name"[..]),
+		protocol_id: config::ProtocolId::from("/test-protocol-name"),
 		import_queue,
 		block_announce_validator: Box::new(
 			sp_consensus::block_validation::DefaultBlockAnnounceValidator,
diff --git a/client/network/src/light_client_handler.rs b/client/network/src/light_client_handler.rs
index 678a717a898ff..98af34092ab21 100644
--- a/client/network/src/light_client_handler.rs
+++ b/client/network/src/light_client_handler.rs
@@ -156,13 +156,13 @@ impl Config {
 	pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self {
 		let mut vl = Vec::new();
 		vl.extend_from_slice(b"/");
-		vl.extend_from_slice(id.as_bytes());
+		vl.extend_from_slice(id.as_ref().as_bytes());
 		vl.extend_from_slice(b"/light/2");
 		self.light_protocol = vl.into();
 
 		let mut vb = Vec::new();
 		vb.extend_from_slice(b"/");
-		vb.extend_from_slice(id.as_bytes());
+		vb.extend_from_slice(id.as_ref().as_bytes());
 		vb.extend_from_slice(b"/sync/2");
 		self.block_protocol = vb.into();
 
@@ -1447,7 +1447,7 @@ mod tests {
 	}
 
 	fn make_config() -> super::Config {
-		super::Config::new(&ProtocolId::from(&b"foo"[..]))
+		super::Config::new(&ProtocolId::from("foo"))
 	}
 
 	fn dummy_header() -> sp_test_primitives::Header {
diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 36010df2d997b..a97aeec1ec4b5 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -423,7 +423,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 
 		let transactions_protocol: Cow<'static, [u8]> = Cow::from({
 			let mut proto = b"/".to_vec();
-			proto.extend(protocol_id.as_bytes());
+			proto.extend(protocol_id.as_ref().as_bytes());
 			proto.extend(b"/transactions/1");
 			proto
 		});
@@ -432,7 +432,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 
 		let block_announces_protocol: Cow<'static, [u8]> = Cow::from({
 			let mut proto = b"/".to_vec();
-			proto.extend(protocol_id.as_bytes());
+			proto.extend(protocol_id.as_ref().as_bytes());
 			proto.extend(b"/block-announces/1");
 			proto
 		});
diff --git a/client/network/src/protocol/generic_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs
index cf9f72b89ba52..15c4a17df8d3a 100644
--- a/client/network/src/protocol/generic_proto/tests.rs
+++ b/client/network/src/protocol/generic_proto/tests.rs
@@ -83,7 +83,7 @@ fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
 		});
 
 		let behaviour = CustomProtoWithAddr {
-			inner: GenericProto::new(local_peer_id, &b"test"[..], &[1], vec![], peerset),
+			inner: GenericProto::new(local_peer_id, "test", &[1], vec![], peerset),
 			addrs: addrs
 				.iter()
 				.enumerate()
diff --git a/client/network/src/protocol/generic_proto/upgrade/legacy.rs b/client/network/src/protocol/generic_proto/upgrade/legacy.rs
index f56ab2450d43e..0937a7798be98 100644
--- a/client/network/src/protocol/generic_proto/upgrade/legacy.rs
+++ b/client/network/src/protocol/generic_proto/upgrade/legacy.rs
@@ -49,7 +49,7 @@ impl RegisteredProtocol {
 		-> Self {
 		let protocol = protocol.into();
 		let mut base_name = b"/substrate/".to_vec();
-		base_name.extend_from_slice(protocol.as_bytes());
+		base_name.extend_from_slice(protocol.as_ref().as_bytes());
 		base_name.extend_from_slice(b"/");
 
 		RegisteredProtocol {
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index c9213d4dde286..6f7751f430751 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -1497,28 +1497,28 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::AnsweredRequest { protocol, build_time, .. })) => {
 					if let Some(metrics) = this.metrics.as_ref() {
 						metrics.requests_in_total
-							.with_label_values(&[&maybe_utf8_bytes_to_string(&protocol)])
+							.with_label_values(&[&protocol])
 							.observe(build_time.as_secs_f64());
 					}
 				},
 				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestStarted { protocol, .. })) => {
 					if let Some(metrics) = this.metrics.as_ref() {
 						metrics.requests_out_started_total
-							.with_label_values(&[&maybe_utf8_bytes_to_string(&protocol)])
+							.with_label_values(&[&protocol])
 							.inc();
 					}
 				},
 				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { protocol, request_duration, .. })) => {
 					if let Some(metrics) = this.metrics.as_ref() {
 						metrics.requests_out_finished
-							.with_label_values(&[&maybe_utf8_bytes_to_string(&protocol)])
+							.with_label_values(&[&protocol])
 							.observe(request_duration.as_secs_f64());
 					}
 				},
 				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted(protocol))) => {
 					if let Some(metrics) = this.metrics.as_ref() {
 						metrics.kademlia_random_queries_total
-							.with_label_values(&[&maybe_utf8_bytes_to_string(protocol.as_bytes())])
+							.with_label_values(&[&protocol.as_ref()])
 							.inc();
 					}
 				},
@@ -1776,16 +1776,13 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 		if let Some(metrics) = this.metrics.as_ref() {
 			metrics.is_major_syncing.set(is_major_syncing as u64);
 			for (proto, num_entries) in this.network_service.num_kbuckets_entries() {
-				let proto = maybe_utf8_bytes_to_string(proto.as_bytes());
-				metrics.kbuckets_num_nodes.with_label_values(&[&proto]).set(num_entries as u64);
+				metrics.kbuckets_num_nodes.with_label_values(&[&proto.as_ref()]).set(num_entries as u64);
 			}
 			for (proto, num_entries) in this.network_service.num_kademlia_records() {
-				let proto = maybe_utf8_bytes_to_string(proto.as_bytes());
-				metrics.kademlia_records_count.with_label_values(&[&proto]).set(num_entries as u64);
+				metrics.kademlia_records_count.with_label_values(&[&proto.as_ref()]).set(num_entries as u64);
 			}
 			for (proto, num_entries) in this.network_service.kademlia_records_total_size() {
-				let proto = maybe_utf8_bytes_to_string(proto.as_bytes());
-				metrics.kademlia_records_sizes_total.with_label_values(&[&proto]).set(num_entries as u64);
+				metrics.kademlia_records_sizes_total.with_label_values(&[&proto.as_ref()]).set(num_entries as u64);
 			}
 			metrics.peers_count.set(num_connected_peers as u64);
 			metrics.peerset_num_discovered.set(this.network_service.user_protocol().num_discovered_peers() as u64);
diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs
index 797942e1c2490..5090362e37606 100644
--- a/client/network/src/service/tests.rs
+++ b/client/network/src/service/tests.rs
@@ -101,7 +101,7 @@ fn build_test_full_node(config: config::NetworkConfiguration)
 		finality_proof_request_builder: None,
 		on_demand: None,
 		transaction_pool: Arc::new(crate::config::EmptyTransactionPool),
-		protocol_id: config::ProtocolId::from(&b"/test-protocol-name"[..]),
+		protocol_id: config::ProtocolId::from("/test-protocol-name"),
 		import_queue,
 		block_announce_validator: Box::new(
 			sp_consensus::block_validation::DefaultBlockAnnounceValidator,
diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs
index 35587cbdc08b3..d269842386cdd 100644
--- a/client/network/test/src/lib.rs
+++ b/client/network/test/src/lib.rs
@@ -675,7 +675,7 @@ pub trait TestNetFactory: Sized {
 			finality_proof_request_builder,
 			on_demand: None,
 			transaction_pool: Arc::new(EmptyTransactionPool),
-			protocol_id: ProtocolId::from(&b"test-protocol-name"[..]),
+			protocol_id: ProtocolId::from("test-protocol-name"),
 			import_queue,
 			block_announce_validator: config.block_announce_validator
 				.unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)),
@@ -755,7 +755,7 @@ pub trait TestNetFactory: Sized {
 			finality_proof_request_builder,
 			on_demand: None,
 			transaction_pool: Arc::new(EmptyTransactionPool),
-			protocol_id: ProtocolId::from(&b"test-protocol-name"[..]),
+			protocol_id: ProtocolId::from("test-protocol-name"),
 			import_queue,
 			block_announce_validator: Box::new(DefaultBlockAnnounceValidator),
 			metrics_registry: None,
diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs
index 8ad95511f77d3..5faf0899aa2e3 100644
--- a/client/service/src/builder.rs
+++ b/client/service/src/builder.rs
@@ -854,7 +854,7 @@ pub fn build_network<TBl, TExPool, TImpQu, TCl>(
 				);
 				DEFAULT_PROTOCOL_ID
 			}
-		}.as_bytes();
+		};
 		sc_network::config::ProtocolId::from(protocol_id_full)
 	};
 

From ebd6d5c21e6311e95ae639e342a5e1259bad68c2 Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Wed, 26 Aug 2020 18:26:33 +0200
Subject: [PATCH 019/122] Support Staking Payout to Any Account (#6832)

* Support staking payout to any account

* fix offences benchmarks
---
 frame/offences/benchmarking/src/lib.rs |  2 +-
 frame/staking/src/lib.rs               | 15 ++++++----
 frame/staking/src/tests.rs             | 38 ++++++++++++++++++++++----
 3 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/frame/offences/benchmarking/src/lib.rs b/frame/offences/benchmarking/src/lib.rs
index 1aa9fed85b127..e35050992368a 100644
--- a/frame/offences/benchmarking/src/lib.rs
+++ b/frame/offences/benchmarking/src/lib.rs
@@ -125,7 +125,7 @@ fn create_offender<T: Trait>(n: u32, nominators: u32) -> Result<Offender<T>, &'s
 			RawOrigin::Signed(nominator_stash.clone()).into(),
 			nominator_controller_lookup.clone(),
 			amount.clone(),
-			reward_destination,
+			reward_destination.clone(),
 		)?;
 
 		let selected_validators: Vec<LookupSourceOf<T>> = vec![controller_lookup.clone()];
diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs
index 8f5b8561eb420..a15b7ac5d7248 100644
--- a/frame/staking/src/lib.rs
+++ b/frame/staking/src/lib.rs
@@ -425,16 +425,18 @@ pub enum StakerStatus<AccountId> {
 
 /// A destination account for payment.
 #[derive(PartialEq, Eq, Copy, Clone, Encode, Decode, RuntimeDebug)]
-pub enum RewardDestination {
+pub enum RewardDestination<AccountId> {
 	/// Pay into the stash account, increasing the amount at stake accordingly.
 	Staked,
 	/// Pay into the stash account, not increasing the amount at stake.
 	Stash,
 	/// Pay into the controller account.
 	Controller,
+	/// Pay into a specified account.
+	Account(AccountId),
 }
 
-impl Default for RewardDestination {
+impl<AccountId> Default for RewardDestination<AccountId> {
 	fn default() -> Self {
 		RewardDestination::Staked
 	}
@@ -1049,7 +1051,7 @@ decl_storage! {
 			=> Option<StakingLedger<T::AccountId, BalanceOf<T>>>;
 
 		/// Where the reward payment should be made. Keyed by stash.
-		pub Payee get(fn payee): map hasher(twox_64_concat) T::AccountId => RewardDestination;
+		pub Payee get(fn payee): map hasher(twox_64_concat) T::AccountId => RewardDestination<T::AccountId>;
 
 		/// The map from (wannabe) validator stash key to the preferences of that validator.
 		pub Validators get(fn validators):
@@ -1496,7 +1498,7 @@ decl_module! {
 		pub fn bond(origin,
 			controller: <T::Lookup as StaticLookup>::Source,
 			#[compact] value: BalanceOf<T>,
-			payee: RewardDestination,
+			payee: RewardDestination<T::AccountId>,
 		) {
 			let stash = ensure_signed(origin)?;
 
@@ -1830,7 +1832,7 @@ decl_module! {
 		///     - Write: Payee
 		/// # </weight>
 		#[weight = 11 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(1, 1)]
-		fn set_payee(origin, payee: RewardDestination) {
+		fn set_payee(origin, payee: RewardDestination<T::AccountId>) {
 			let controller = ensure_signed(origin)?;
 			let ledger = Self::ledger(&controller).ok_or(Error::<T>::NotController)?;
 			let stash = &ledger.stash;
@@ -2489,6 +2491,9 @@ impl<T: Trait> Module<T> {
 					Self::update_ledger(&controller, &l);
 					r
 				}),
+			RewardDestination::Account(dest_account) => {
+				Some(T::Currency::deposit_creating(&dest_account, amount))
+			}
 		}
 	}
 
diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs
index 278e532387672..d27654d1feae6 100644
--- a/frame/staking/src/tests.rs
+++ b/frame/staking/src/tests.rs
@@ -4359,7 +4359,7 @@ fn test_payout_stakers() {
 	// We also test that `payout_extra_nominators` works.
 	ExtBuilder::default().has_stakers(false).build_and_execute(|| {
 		let balance = 1000;
-		// Create three validators:
+		// Create a validator:
 		bond_validator(11, 10, balance); // Default(64)
 
 		// Create nominators, targeting stash of validators
@@ -4597,15 +4597,12 @@ fn on_initialize_weight_is_correct() {
 	});
 }
 
-
 #[test]
 fn payout_creates_controller() {
-	// Here we will test validator can set `max_nominators_payout` and it works.
-	// We also test that `payout_extra_nominators` works.
 	ExtBuilder::default().has_stakers(false).build_and_execute(|| {
 		let balance = 1000;
-		// Create three validators:
-		bond_validator(11, 10, balance); // Default(64)
+		// Create a validator:
+		bond_validator(11, 10, balance);
 
 		// Create a stash/controller pair
 		bond_nominator(1234, 1337, 100, vec![11]);
@@ -4626,3 +4623,32 @@ fn payout_creates_controller() {
 		assert!(Balances::free_balance(1337) > 0);
 	})
 }
+
+#[test]
+fn payout_to_any_account_works() {
+	ExtBuilder::default().has_stakers(false).build_and_execute(|| {
+		let balance = 1000;
+		// Create a validator:
+		bond_validator(11, 10, balance); // Default(64)
+
+		// Create a stash/controller pair
+		bond_nominator(1234, 1337, 100, vec![11]);
+
+		// Update payout location
+		assert_ok!(Staking::set_payee(Origin::signed(1337), RewardDestination::Account(42)));
+
+		// Reward Destination account doesn't exist
+		assert_eq!(Balances::free_balance(42), 0);
+
+		mock::start_era(1);
+		Staking::reward_by_ids(vec![(11, 1)]);
+		// Compute total payout now for whole duration as other parameter won't change
+		let total_payout_0 = current_total_payout_for_duration(3 * 1000);
+		assert!(total_payout_0 > 100); // Test is meaningful if reward something
+		mock::start_era(2);
+		assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 1));
+
+		// Payment is successful
+		assert!(Balances::free_balance(42) > 0);
+	})
+}

From ce89cc9cac63f2bd8131f9f97fc0180bc962e7c5 Mon Sep 17 00:00:00 2001
From: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Date: Wed, 26 Aug 2020 18:26:56 +0200
Subject: [PATCH 020/122] Better prime election. (#6939)

* Better prime election.

* improve docs

* more sensible variable names

* link to Borda count wiki

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
---
 frame/elections-phragmen/src/lib.rs | 33 +++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs
index 50c5de9bc0de4..9d1922576ad42 100644
--- a/frame/elections-phragmen/src/lib.rs
+++ b/frame/elections-phragmen/src/lib.rs
@@ -87,7 +87,7 @@ use codec::{Encode, Decode};
 use sp_std::prelude::*;
 use sp_runtime::{
 	DispatchError, RuntimeDebug, Perbill,
-	traits::{Zero, StaticLookup, Convert},
+	traits::{Zero, StaticLookup, Convert, Saturating},
 };
 use frame_support::{
 	decl_storage, decl_event, ensure, decl_module, decl_error,
@@ -904,14 +904,20 @@ impl<T: Trait> Module<T> {
 			to_votes(Self::locked_stake_of(who))
 		};
 
-		let voters_and_votes = Voting::<T>::iter()
-			.map(|(voter, (stake, targets))| { (voter, to_votes(stake), targets) })
+		// used for prime election.
+		let voters_and_stakes = Voting::<T>::iter()
+			.map(|(voter, (stake, targets))| { (voter, stake, targets) })
+			.collect::<Vec<_>>();
+		// used for phragmen.
+		let voters_and_votes = voters_and_stakes.iter()
+			.cloned()
+			.map(|(voter, stake, targets)| { (voter, to_votes(stake), targets)} )
 			.collect::<Vec<_>>();
 		let maybe_phragmen_result = sp_npos_elections::seq_phragmen::<T::AccountId, Perbill>(
 			num_to_elect,
 			0,
 			candidates,
-			voters_and_votes.clone(),
+			voters_and_votes,
 		);
 
 		if let Some(ElectionResult { winners, assignments }) = maybe_phragmen_result {
@@ -965,17 +971,26 @@ impl<T: Trait> Module<T> {
 			// save the members, sorted based on account id.
 			new_members.sort_by(|i, j| i.0.cmp(&j.0));
 
-			let mut prime_votes: Vec<_> = new_members.iter().map(|c| (&c.0, VoteWeight::zero())).collect();
-			for (_, stake, targets) in voters_and_votes.into_iter() {
-				for (votes, who) in targets.iter()
+			// Now we select a prime member using a [Borda count](https://en.wikipedia.org/wiki/Borda_count).
+			// We weigh everyone's vote for that new member by a multiplier based on the order
+			// of the votes. i.e. the first person a voter votes for gets a 16x multiplier,
+			// the next person gets a 15x multiplier, an so on... (assuming `MAXIMUM_VOTE` = 16)
+			let mut prime_votes: Vec<_> = new_members.iter().map(|c| (&c.0, BalanceOf::<T>::zero())).collect();
+			for (_, stake, targets) in voters_and_stakes.into_iter() {
+				for (vote_multiplier, who) in targets.iter()
 					.enumerate()
-					.map(|(votes, who)| ((MAXIMUM_VOTE - votes) as u32, who))
+					.map(|(vote_position, who)| ((MAXIMUM_VOTE - vote_position) as u32, who))
 				{
 					if let Ok(i) = prime_votes.binary_search_by_key(&who, |k| k.0) {
-						prime_votes[i].1 += stake * votes as VoteWeight;
+						prime_votes[i].1 = prime_votes[i].1.saturating_add(
+							stake.saturating_mul(vote_multiplier.into())
+						);
 					}
 				}
 			}
+			// We then select the new member with the highest weighted stake. In the case of
+			// a tie, the last person in the list with the tied score is selected. This is
+			// the person with the "highest" account id based on the sort above.
 			let prime = prime_votes.into_iter().max_by_key(|x| x.1).map(|x| x.0.clone());
 
 			// new_members_ids is sorted by account id.

From 117bb452b72ce208ac0ee6b742d4ba85a788bdf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Silva?=
 <123550+andresilva@users.noreply.github.com>
Date: Wed, 26 Aug 2020 17:27:44 +0100
Subject: [PATCH 021/122] babe: fix report_equivocation weight (#6936)

* babe: fix report_equivocation weight

* node: bump spec_version

* babe: fix floor in report_equivocation weight calculation

Co-authored-by: Gavin Wood <gavin@parity.io>

* grandpa: fix floor in report_equivocation weight calculation

* babe, grandpa: add test for weight_for::report_equivocation

Co-authored-by: Gavin Wood <gavin@parity.io>
---
 frame/babe/src/lib.rs      | 29 ++++++++++++++++++++---------
 frame/babe/src/tests.rs    | 23 +++++++++++++++++++++++
 frame/grandpa/src/lib.rs   |  2 +-
 frame/grandpa/src/tests.rs | 23 +++++++++++++++++++++++
 4 files changed, 67 insertions(+), 10 deletions(-)

diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs
index f80ac186434d0..891411e8ede57 100644
--- a/frame/babe/src/lib.rs
+++ b/frame/babe/src/lib.rs
@@ -255,7 +255,7 @@ decl_module! {
 		/// the equivocation proof and validate the given key ownership proof
 		/// against the extracted offender. If both are valid, the offence will
 		/// be reported.
-		#[weight = weight::weight_for_report_equivocation::<T>()]
+		#[weight = weight_for::report_equivocation::<T>(key_owner_proof.validator_count())]
 		fn report_equivocation(
 			origin,
 			equivocation_proof: EquivocationProof<T::Header>,
@@ -278,7 +278,7 @@ decl_module! {
 		/// block authors will call it (validated in `ValidateUnsigned`), as such
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
-		#[weight = weight::weight_for_report_equivocation::<T>()]
+		#[weight = weight_for::report_equivocation::<T>(key_owner_proof.validator_count())]
 		fn report_equivocation_unsigned(
 			origin,
 			equivocation_proof: EquivocationProof<T::Header>,
@@ -295,24 +295,35 @@ decl_module! {
 	}
 }
 
-mod weight {
+mod weight_for {
 	use frame_support::{
 		traits::Get,
-		weights::{constants::WEIGHT_PER_MICROS, Weight},
+		weights::{
+			constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS},
+			Weight,
+		},
 	};
 
-	pub fn weight_for_report_equivocation<T: super::Trait>() -> Weight {
+	pub fn report_equivocation<T: super::Trait>(validator_count: u32) -> Weight {
+		// we take the validator set count from the membership proof to
+		// calculate the weight but we set a floor of 100 validators.
+		let validator_count = validator_count.max(100) as u64;
+
+		// worst case we are considering is that the given offender
+		// is backed by 200 nominators
+		const MAX_NOMINATORS: u64 = 200;
+
 		// checking membership proof
 		(35 * WEIGHT_PER_MICROS)
+			.saturating_add((175 * WEIGHT_PER_NANOS).saturating_mul(validator_count))
 			.saturating_add(T::DbWeight::get().reads(5))
 			// check equivocation proof
 			.saturating_add(110 * WEIGHT_PER_MICROS)
 			// report offence
 			.saturating_add(110 * WEIGHT_PER_MICROS)
-			// worst case we are considering is that the given offender
-			// is backed by 200 nominators
-			.saturating_add(T::DbWeight::get().reads(14 + 3 * 200))
-			.saturating_add(T::DbWeight::get().writes(10 + 3 * 200))
+			.saturating_add(25 * WEIGHT_PER_MICROS * MAX_NOMINATORS)
+			.saturating_add(T::DbWeight::get().reads(14 + 3 * MAX_NOMINATORS))
+			.saturating_add(T::DbWeight::get().writes(10 + 3 * MAX_NOMINATORS))
 	}
 }
 
diff --git a/frame/babe/src/tests.rs b/frame/babe/src/tests.rs
index bdd6748c3b351..2b24e1208de1d 100644
--- a/frame/babe/src/tests.rs
+++ b/frame/babe/src/tests.rs
@@ -585,3 +585,26 @@ fn report_equivocation_validate_unsigned_prevents_duplicates() {
 		);
 	});
 }
+
+#[test]
+fn report_equivocation_has_valid_weight() {
+	// the weight depends on the size of the validator set,
+	// but there's a lower bound of 100 validators.
+	assert!(
+		(1..=100)
+			.map(weight_for::report_equivocation::<Test>)
+			.collect::<Vec<_>>()
+			.windows(2)
+			.all(|w| w[0] == w[1])
+	);
+
+	// after 100 validators the weight should keep increasing
+	// with every extra validator.
+	assert!(
+		(100..=1000)
+			.map(weight_for::report_equivocation::<Test>)
+			.collect::<Vec<_>>()
+			.windows(2)
+			.all(|w| w[0] < w[1])
+	);
+}
diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs
index 961c099460752..09d32662d349a 100644
--- a/frame/grandpa/src/lib.rs
+++ b/frame/grandpa/src/lib.rs
@@ -376,7 +376,7 @@ mod weight_for {
 	pub fn report_equivocation<T: super::Trait>(validator_count: u32) -> Weight {
 		// we take the validator set count from the membership proof to
 		// calculate the weight but we set a floor of 100 validators.
-		let validator_count = validator_count.min(100) as u64;
+		let validator_count = validator_count.max(100) as u64;
 
 		// worst case we are considering is that the given offender
 		// is backed by 200 nominators
diff --git a/frame/grandpa/src/tests.rs b/frame/grandpa/src/tests.rs
index 9eca2cc381371..aa1b48681d402 100644
--- a/frame/grandpa/src/tests.rs
+++ b/frame/grandpa/src/tests.rs
@@ -842,3 +842,26 @@ fn always_schedules_a_change_on_new_session_when_stalled() {
 		assert_eq!(Grandpa::current_set_id(), 2);
 	});
 }
+
+#[test]
+fn report_equivocation_has_valid_weight() {
+	// the weight depends on the size of the validator set,
+	// but there's a lower bound of 100 validators.
+	assert!(
+		(1..=100)
+			.map(weight_for::report_equivocation::<Test>)
+			.collect::<Vec<_>>()
+			.windows(2)
+			.all(|w| w[0] == w[1])
+	);
+
+	// after 100 validators the weight should keep increasing
+	// with every extra validator.
+	assert!(
+		(100..=1000)
+			.map(weight_for::report_equivocation::<Test>)
+			.collect::<Vec<_>>()
+			.windows(2)
+			.all(|w| w[0] < w[1])
+	);
+}

From 4c0a477e0f80f94ff957671fe3ec66fa11d06b86 Mon Sep 17 00:00:00 2001
From: Guillaume Thiolliere <gui.thiolliere@gmail.com>
Date: Wed, 26 Aug 2020 20:36:37 +0200
Subject: [PATCH 022/122] fix bench db wipe (#6965)

---
 client/db/src/bench.rs | 59 +++++++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 27 deletions(-)

diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs
index 1c9be87faa0f3..f3c8f1aff9e14 100644
--- a/client/db/src/bench.rs
+++ b/client/db/src/bench.rs
@@ -178,6 +178,7 @@ impl<B: BlockT> BenchmarkingState<B> {
 
 	fn wipe_tracker(&self) {
 		*self.main_key_tracker.borrow_mut() = HashMap::new();
+		*self.child_key_tracker.borrow_mut() = HashMap::new();
 		self.add_whitelist_to_tracker();
 		*self.read_write_tracker.borrow_mut() = Default::default();
 	}
@@ -512,32 +513,36 @@ mod test {
 		let bench_state = BenchmarkingState::<crate::tests::Block>::new(Default::default(), None)
 			.unwrap();
 
-		let child1 = sp_core::storage::ChildInfo::new_default(b"child1");
-		let child2 = sp_core::storage::ChildInfo::new_default(b"child2");
-
-		bench_state.storage(b"foo").unwrap();
-		bench_state.child_storage(&child1, b"foo").unwrap();
-		bench_state.child_storage(&child2, b"foo").unwrap();
-
-		bench_state.storage(b"bar").unwrap();
-		bench_state.child_storage(&child1, b"bar").unwrap();
-		bench_state.child_storage(&child2, b"bar").unwrap();
-
-		bench_state.commit(
-			Default::default(),
-			Default::default(),
-			vec![
-				("foo".as_bytes().to_vec(), None)
-			],
-			vec![
-				("child1".as_bytes().to_vec(), vec![("foo".as_bytes().to_vec(), None)])
-			]
-		).unwrap();
-
-		let rw_tracker = bench_state.read_write_tracker.borrow();
-		assert_eq!(rw_tracker.reads, 6);
-		assert_eq!(rw_tracker.repeat_reads, 0);
-		assert_eq!(rw_tracker.writes, 2);
-		assert_eq!(rw_tracker.repeat_writes, 0);
+		for _ in 0..2 {
+			let child1 = sp_core::storage::ChildInfo::new_default(b"child1");
+			let child2 = sp_core::storage::ChildInfo::new_default(b"child2");
+
+			bench_state.storage(b"foo").unwrap();
+			bench_state.child_storage(&child1, b"foo").unwrap();
+			bench_state.child_storage(&child2, b"foo").unwrap();
+
+			bench_state.storage(b"bar").unwrap();
+			bench_state.child_storage(&child1, b"bar").unwrap();
+			bench_state.child_storage(&child2, b"bar").unwrap();
+
+			bench_state.commit(
+				Default::default(),
+				Default::default(),
+				vec![
+					("foo".as_bytes().to_vec(), None)
+				],
+				vec![
+					("child1".as_bytes().to_vec(), vec![("foo".as_bytes().to_vec(), None)])
+				]
+			).unwrap();
+
+			let rw_tracker = bench_state.read_write_tracker.borrow();
+			assert_eq!(rw_tracker.reads, 6);
+			assert_eq!(rw_tracker.repeat_reads, 0);
+			assert_eq!(rw_tracker.writes, 2);
+			assert_eq!(rw_tracker.repeat_writes, 0);
+			drop(rw_tracker);
+			bench_state.wipe().unwrap();
+		}
 	}
 }

From 3bddf6cc7eac5f8d562c1e0274af4c2604a803b7 Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Thu, 27 Aug 2020 14:53:20 +0200
Subject: [PATCH 023/122] Implement request-responses protocols (#6634)

* Implement request-responses protocols

* Add tests

* Fix sc-cli

* Apply suggestions from code review

Co-authored-by: Max Inden <mail@max-inden.de>

* Fix naming

* Fix other issues

* Other naming fix

* Fix error logging

* Max sizes to u64

* Don't kill connections on refusal to process

* Adjust comment

Co-authored-by: Max Inden <mail@max-inden.de>
---
 Cargo.lock                              |   1 +
 client/cli/src/params/network_params.rs |   1 +
 client/network/Cargo.toml               |   3 +-
 client/network/src/behaviour.rs         |  94 ++-
 client/network/src/config.rs            |  13 +-
 client/network/src/error.rs             |   9 +-
 client/network/src/lib.rs               |  10 +-
 client/network/src/request_responses.rs | 892 ++++++++++++++++++++++++
 client/network/src/service.rs           | 225 +++++-
 9 files changed, 1183 insertions(+), 65 deletions(-)
 create mode 100644 client/network/src/request_responses.rs

diff --git a/Cargo.lock b/Cargo.lock
index 65f5935a1e900..402dfb6ec9e1f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6911,6 +6911,7 @@ version = "0.8.0-rc6"
 dependencies = [
  "assert_matches",
  "async-std",
+ "async-trait",
  "bitflags",
  "bs58",
  "bytes 0.5.6",
diff --git a/client/cli/src/params/network_params.rs b/client/cli/src/params/network_params.rs
index 4a33644e8934e..faaf2c2bd210d 100644
--- a/client/cli/src/params/network_params.rs
+++ b/client/cli/src/params/network_params.rs
@@ -148,6 +148,7 @@ impl NetworkParams {
 			listen_addresses,
 			public_addresses,
 			notifications_protocols: Vec::new(),
+			request_response_protocols: Vec::new(),
 			node_key,
 			node_name: node_name.to_string(),
 			client_version: client_id.to_string(),
diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml
index de885bc65a45c..d5729ae06b2ca 100644
--- a/client/network/Cargo.toml
+++ b/client/network/Cargo.toml
@@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 prost-build = "0.6.1"
 
 [dependencies]
+async-trait = "0.1"
 async-std = { version = "1.6.2", features = ["unstable"] }
 bitflags = "1.2.0"
 bs58 = "0.3.1"
@@ -64,7 +65,7 @@ zeroize = "1.0.0"
 [dependencies.libp2p]
 version = "0.24.0"
 default-features = false
-features = ["identify", "kad", "mdns-async-std", "mplex", "noise", "ping", "tcp-async-std", "websocket", "yamux"]
+features = ["identify", "kad", "mdns-async-std", "mplex", "noise", "ping", "request-response", "tcp-async-std", "websocket", "yamux"]
 
 [dev-dependencies]
 assert_matches = "1.3"
diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs
index 20b5adf76b832..4a47a26f55c24 100644
--- a/client/network/src/behaviour.rs
+++ b/client/network/src/behaviour.rs
@@ -16,7 +16,7 @@
 
 use crate::{
 	config::{ProtocolId, Role}, block_requests, light_client_handler, finality_requests,
-	peer_info, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut},
+	peer_info, request_responses, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut},
 	protocol::{message::{self, Roles}, CustomMessageOutcome, NotificationsSink, Protocol},
 	ObservedRole, DhtEvent, ExHashT,
 };
@@ -39,6 +39,10 @@ use std::{
 	time::Duration,
 };
 
+pub use crate::request_responses::{
+	ResponseFailure, InboundFailure, RequestFailure, OutboundFailure, RequestId, SendRequestError
+};
+
 /// General behaviour of the network. Combines all protocols together.
 #[derive(NetworkBehaviour)]
 #[behaviour(out_event = "BehaviourOut<B>", poll_method = "poll")]
@@ -50,6 +54,8 @@ pub struct Behaviour<B: BlockT, H: ExHashT> {
 	peer_info: peer_info::PeerInfoBehaviour,
 	/// Discovers nodes of the network.
 	discovery: DiscoveryBehaviour,
+	/// Generic request-reponse protocols.
+	request_responses: request_responses::RequestResponsesBehaviour,
 	/// Block request handling.
 	block_requests: block_requests::BlockRequests<B>,
 	/// Finality proof request handling.
@@ -76,22 +82,40 @@ pub enum BehaviourOut<B: BlockT> {
 	RandomKademliaStarted(ProtocolId),
 
 	/// We have received a request from a peer and answered it.
-	AnsweredRequest {
+	///
+	/// This event is generated for statistics purposes.
+	InboundRequest {
 		/// Peer which sent us a request.
 		peer: PeerId,
 		/// Protocol name of the request.
-		protocol: String,
-		/// Time it took to build the response.
-		build_time: Duration,
+		protocol: Cow<'static, str>,
+		/// If `Ok`, contains the time elapsed between when we received the request and when we
+		/// sent back the response. If `Err`, the error that happened.
+		result: Result<Duration, ResponseFailure>,
 	},
+
+	/// A request initiated using [`Behaviour::send_request`] has succeeded or failed.
+	RequestFinished {
+		/// Request that has succeeded.
+		request_id: RequestId,
+		/// Response sent by the remote or reason for failure.
+		result: Result<Vec<u8>, RequestFailure>,
+	},
+
 	/// Started a new request with the given node.
-	RequestStarted {
+	///
+	/// This event is for statistics purposes only. The request and response handling are entirely
+	/// internal to the behaviour.
+	OpaqueRequestStarted {
 		peer: PeerId,
 		/// Protocol name of the request.
 		protocol: String,
 	},
 	/// Finished, successfully or not, a previously-started request.
-	RequestFinished {
+	///
+	/// This event is for statistics purposes only. The request and response handling are entirely
+	/// internal to the behaviour.
+	OpaqueRequestFinished {
 		/// Who we were requesting.
 		peer: PeerId,
 		/// Protocol name of the request.
@@ -161,17 +185,20 @@ impl<B: BlockT, H: ExHashT> Behaviour<B, H> {
 		finality_proof_requests: finality_requests::FinalityProofRequests<B>,
 		light_client_handler: light_client_handler::LightClientHandler<B>,
 		disco_config: DiscoveryConfig,
-	) -> Self {
-		Behaviour {
+		request_response_protocols: Vec<request_responses::ProtocolConfig>,
+	) -> Result<Self, request_responses::RegisterError> {
+		Ok(Behaviour {
 			substrate,
 			peer_info: peer_info::PeerInfoBehaviour::new(user_agent, local_public_key),
 			discovery: disco_config.finish(),
+			request_responses:
+				request_responses::RequestResponsesBehaviour::new(request_response_protocols.into_iter())?,
 			block_requests,
 			finality_proof_requests,
 			light_client_handler,
 			events: VecDeque::new(),
 			role,
-		}
+		})
 	}
 
 	/// Returns the list of nodes that we know exist in the network.
@@ -208,6 +235,16 @@ impl<B: BlockT, H: ExHashT> Behaviour<B, H> {
 		self.peer_info.node(peer_id)
 	}
 
+	/// Initiates sending a request.
+	///
+	/// An error is returned if we are not connected to the target peer of if the protocol doesn't
+	/// match one that has been registered.
+	pub fn send_request(&mut self, target: &PeerId, protocol: &str, request: Vec<u8>)
+		-> Result<RequestId, SendRequestError>
+	{
+		self.request_responses.send_request(target, protocol, request)
+	}
+
 	/// Registers a new notifications protocol.
 	///
 	/// Please call `event_stream` before registering a protocol, otherwise you may miss events
@@ -298,18 +335,18 @@ Behaviour<B, H> {
 			CustomMessageOutcome::BlockRequest { target, request } => {
 				match self.block_requests.send_request(&target, request) {
 					block_requests::SendRequestOutcome::Ok => {
-						self.events.push_back(BehaviourOut::RequestStarted {
+						self.events.push_back(BehaviourOut::OpaqueRequestStarted {
 							peer: target,
 							protocol: self.block_requests.protocol_name().to_owned(),
 						});
 					},
 					block_requests::SendRequestOutcome::Replaced { request_duration, .. } => {
-						self.events.push_back(BehaviourOut::RequestFinished {
+						self.events.push_back(BehaviourOut::OpaqueRequestFinished {
 							peer: target.clone(),
 							protocol: self.block_requests.protocol_name().to_owned(),
 							request_duration,
 						});
-						self.events.push_back(BehaviourOut::RequestStarted {
+						self.events.push_back(BehaviourOut::OpaqueRequestStarted {
 							peer: target,
 							protocol: self.block_requests.protocol_name().to_owned(),
 						});
@@ -358,18 +395,39 @@ Behaviour<B, H> {
 	}
 }
 
+impl<B: BlockT, H: ExHashT> NetworkBehaviourEventProcess<request_responses::Event> for Behaviour<B, H> {
+	fn inject_event(&mut self, event: request_responses::Event) {
+		match event {
+			request_responses::Event::InboundRequest { peer, protocol, result } => {
+				self.events.push_back(BehaviourOut::InboundRequest {
+					peer,
+					protocol,
+					result,
+				});
+			}
+
+			request_responses::Event::RequestFinished { request_id, result } => {
+				self.events.push_back(BehaviourOut::RequestFinished {
+					request_id,
+					result,
+				});
+			},
+		}
+	}
+}
+
 impl<B: BlockT, H: ExHashT> NetworkBehaviourEventProcess<block_requests::Event<B>> for Behaviour<B, H> {
 	fn inject_event(&mut self, event: block_requests::Event<B>) {
 		match event {
 			block_requests::Event::AnsweredRequest { peer, total_handling_time } => {
-				self.events.push_back(BehaviourOut::AnsweredRequest {
+				self.events.push_back(BehaviourOut::InboundRequest {
 					peer,
-					protocol: self.block_requests.protocol_name().to_owned(),
-					build_time: total_handling_time,
+					protocol: self.block_requests.protocol_name().to_owned().into(),
+					result: Ok(total_handling_time),
 				});
 			},
 			block_requests::Event::Response { peer, original_request: _, response, request_duration } => {
-				self.events.push_back(BehaviourOut::RequestFinished {
+				self.events.push_back(BehaviourOut::OpaqueRequestFinished {
 					peer: peer.clone(),
 					protocol: self.block_requests.protocol_name().to_owned(),
 					request_duration,
@@ -381,7 +439,7 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviourEventProcess<block_requests::Event<B
 			block_requests::Event::RequestTimeout { peer, request_duration, .. } => {
 				// There doesn't exist any mechanism to report cancellations or timeouts yet, so
 				// we process them by disconnecting the node.
-				self.events.push_back(BehaviourOut::RequestFinished {
+				self.events.push_back(BehaviourOut::OpaqueRequestFinished {
 					peer: peer.clone(),
 					protocol: self.block_requests.protocol_name().to_owned(),
 					request_duration,
diff --git a/client/network/src/config.rs b/client/network/src/config.rs
index bde34a0a5716a..5185befacf5ae 100644
--- a/client/network/src/config.rs
+++ b/client/network/src/config.rs
@@ -23,6 +23,7 @@
 
 pub use crate::chain::{Client, FinalityProofProvider};
 pub use crate::on_demand_layer::{AlwaysBadChecker, OnDemand};
+pub use crate::request_responses::{IncomingRequest, ProtocolConfig as RequestResponseConfig};
 pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multiaddr};
 
 // Note: this re-export shouldn't be part of the public API of the crate and will be removed in
@@ -34,9 +35,10 @@ use crate::ExHashT;
 
 use core::{fmt, iter};
 use futures::future;
-use libp2p::identity::{ed25519, Keypair};
-use libp2p::wasm_ext;
-use libp2p::{multiaddr, Multiaddr, PeerId};
+use libp2p::{
+	identity::{ed25519, Keypair},
+	multiaddr, wasm_ext, Multiaddr, PeerId,
+};
 use prometheus_endpoint::Registry;
 use sp_consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue};
 use sp_runtime::{traits::Block as BlockT, ConsensusEngineId};
@@ -414,6 +416,8 @@ pub struct NetworkConfiguration {
 	/// List of notifications protocols that the node supports. Must also include a
 	/// `ConsensusEngineId` for backwards-compatibility.
 	pub notifications_protocols: Vec<(ConsensusEngineId, Cow<'static, [u8]>)>,
+	/// List of request-response protocols that the node supports.
+	pub request_response_protocols: Vec<RequestResponseConfig>,
 	/// Maximum allowed number of incoming connections.
 	pub in_peers: u32,
 	/// Number of outgoing connections we're trying to maintain.
@@ -449,6 +453,7 @@ impl NetworkConfiguration {
 			boot_nodes: Vec::new(),
 			node_key,
 			notifications_protocols: Vec::new(),
+			request_response_protocols: Vec::new(),
 			in_peers: 25,
 			out_peers: 75,
 			reserved_nodes: Vec::new(),
@@ -465,9 +470,7 @@ impl NetworkConfiguration {
 			allow_non_globals_in_dht: false,
 		}
 	}
-}
 
-impl NetworkConfiguration {
 	/// Create new default configuration for localhost-only connection with random port (useful for testing)
 	pub fn new_local() -> NetworkConfiguration {
 		let mut config = NetworkConfiguration::new(
diff --git a/client/network/src/error.rs b/client/network/src/error.rs
index d5a4024ef53d7..7d7603ce92aab 100644
--- a/client/network/src/error.rs
+++ b/client/network/src/error.rs
@@ -21,7 +21,7 @@
 use crate::config::TransportConfig;
 use libp2p::{PeerId, Multiaddr};
 
-use std::fmt;
+use std::{borrow::Cow, fmt};
 
 /// Result type alias for the network.
 pub type Result<T> = std::result::Result<T, Error>;
@@ -61,6 +61,12 @@ pub enum Error {
 		/// The invalid addresses.
 		addresses: Vec<Multiaddr>,
 	},
+	/// The same request-response protocol has been registered multiple times.
+	#[display(fmt = "Request-response protocol registered multiple times: {}", protocol)]
+	DuplicateRequestResponseProtocol {
+		/// Name of the protocol registered multiple times.
+		protocol: Cow<'static, str>,
+	},
 }
 
 // Make `Debug` use the `Display` implementation.
@@ -78,6 +84,7 @@ impl std::error::Error for Error {
 			Error::DuplicateBootnode { .. } => None,
 			Error::Prometheus(ref err) => Some(err),
 			Error::AddressesForAnotherTransport { .. } => None,
+			Error::DuplicateRequestResponseProtocol { .. } => None,
 		}
 	}
 }
diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs
index e01b260263566..326d73c372110 100644
--- a/client/network/src/lib.rs
+++ b/client/network/src/lib.rs
@@ -253,6 +253,7 @@ mod finality_requests;
 mod light_client_handler;
 mod on_demand_layer;
 mod protocol;
+mod request_responses;
 mod schema;
 mod service;
 mod transport;
@@ -263,13 +264,10 @@ pub mod error;
 pub mod gossip;
 pub mod network_state;
 
-pub use service::{NetworkService, NetworkWorker};
-pub use protocol::PeerInfo;
-pub use protocol::event::{Event, DhtEvent, ObservedRole};
-pub use protocol::sync::SyncState;
-pub use libp2p::{Multiaddr, PeerId};
 #[doc(inline)]
-pub use libp2p::multiaddr;
+pub use libp2p::{multiaddr, Multiaddr, PeerId};
+pub use protocol::{event::{DhtEvent, Event, ObservedRole}, sync::SyncState, PeerInfo};
+pub use service::{NetworkService, NetworkWorker, RequestFailure, OutboundFailure};
 
 pub use sc_peerset::ReputationChange;
 use sp_runtime::traits::{Block as BlockT, NumberFor};
diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs
new file mode 100644
index 0000000000000..92233c77d6bd1
--- /dev/null
+++ b/client/network/src/request_responses.rs
@@ -0,0 +1,892 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Collection of request-response protocols.
+//!
+//! The [`RequestResponses`] struct defined in this module provides support for zero or more
+//! so-called "request-response" protocols.
+//!
+//! A request-response protocol works in the following way:
+//!
+//! - For every emitted request, a new substream is open and the protocol is negotiated. If the
+//! remote supports the protocol, the size of the request is sent as a LEB128 number, followed
+//! with the request itself. The remote then sends the size of the response as a LEB128 number,
+//! followed with the response.
+//!
+//! - Requests have a certain time limit before they time out. This time includes the time it
+//! takes to send/receive the request and response.
+//!
+//! - If provided, a ["requests processing"](RequestResponseConfig::inbound_queue) channel
+//! is used to handle incoming requests.
+//!
+
+use futures::{channel::{mpsc, oneshot}, prelude::*};
+use libp2p::{
+	core::{
+		connection::{ConnectionId, ListenerId},
+		ConnectedPoint, Multiaddr, PeerId,
+	},
+	request_response::{
+		RequestResponse, RequestResponseCodec, RequestResponseConfig, RequestResponseEvent,
+		RequestResponseMessage, ResponseChannel, ProtocolSupport
+	},
+	swarm::{
+		protocols_handler::multi::MultiHandler, NetworkBehaviour, NetworkBehaviourAction,
+		PollParameters, ProtocolsHandler,
+	},
+};
+use std::{
+	borrow::Cow, collections::{hash_map::Entry, HashMap}, convert::TryFrom as _, io, iter,
+	pin::Pin, task::{Context, Poll}, time::Duration,
+};
+
+pub use libp2p::request_response::{InboundFailure, OutboundFailure, RequestId};
+
+/// Configuration for a single request-response protocol.
+#[derive(Debug, Clone)]
+pub struct ProtocolConfig {
+    /// Name of the protocol on the wire. Should be something like `/foo/bar`.
+    pub name: Cow<'static, str>,
+
+    /// Maximum allowed size, in bytes, of a request.
+    ///
+    /// Any request larger than this value will be declined as a way to avoid allocating too
+    /// much memory for it.
+    pub max_request_size: u64,
+
+    /// Maximum allowed size, in bytes, of a response.
+    ///
+    /// Any response larger than this value will be declined as a way to avoid allocating too
+    /// much memory for it.
+    pub max_response_size: u64,
+
+    /// Duration after which emitted requests are considered timed out.
+    ///
+    /// If you expect the response to come back quickly, you should set this to a smaller duration.
+    pub request_timeout: Duration,
+
+    /// Channel on which the networking service will send incoming requests.
+    ///
+    /// Every time a peer sends a request to the local node using this protocol, the networking
+    /// service will push an element on this channel. The receiving side of this channel then has
+    /// to pull this element, process the request, and send back the response to send back to the
+    /// peer.
+    ///
+    /// The size of the channel has to be carefully chosen. If the channel is full, the networking
+    /// service will discard the incoming request send back an error to the peer. Consequently,
+    /// the channel being full is an indicator that the node is overloaded.
+    ///
+    /// You can typically set the size of the channel to `T / d`, where `T` is the
+    /// `request_timeout` and `d` is the expected average duration of CPU and I/O it takes to
+    /// build a response.
+    ///
+    /// Can be `None` if the local node does not support answering incoming requests.
+    /// If this is `None`, then the local node will not advertise support for this protocol towards
+    /// other peers. If this is `Some` but the channel is closed, then the local node will
+    /// advertise support for this protocol, but any incoming request will lead to an error being
+    /// sent back.
+    pub inbound_queue: Option<mpsc::Sender<IncomingRequest>>,
+}
+
+/// A single request received by a peer on a request-response protocol.
+#[derive(Debug)]
+pub struct IncomingRequest {
+	/// Who sent the request.
+	pub peer: PeerId,
+
+	/// Request sent by the remote. Will always be smaller than
+	/// [`RequestResponseConfig::max_request_size`].
+	pub payload: Vec<u8>,
+
+	/// Channel to send back the response to.
+	pub pending_response: oneshot::Sender<Vec<u8>>,
+}
+
+/// Event generated by the [`RequestResponsesBehaviour`].
+#[derive(Debug)]
+pub enum Event {
+	/// A remote sent a request and either we have successfully answered it or an error happened.
+	///
+	/// This event is generated for statistics purposes.
+	InboundRequest {
+		/// Peer which has emitted the request.
+		peer: PeerId,
+		/// Name of the protocol in question.
+		protocol: Cow<'static, str>,
+		/// If `Ok`, contains the time elapsed between when we received the request and when we
+		/// sent back the response. If `Err`, the error that happened.
+		result: Result<Duration, ResponseFailure>,
+	},
+
+	/// A request initiated using [`RequestResponsesBehaviour::send_request`] has succeeded or
+	/// failed.
+	RequestFinished {
+		/// Request that has succeeded.
+		request_id: RequestId,
+		/// Response sent by the remote or reason for failure.
+		result: Result<Vec<u8>, RequestFailure>,
+	},
+}
+
+/// Implementation of `NetworkBehaviour` that provides support for request-response protocols.
+pub struct RequestResponsesBehaviour {
+	/// The multiple sub-protocols, by name.
+	/// Contains the underlying libp2p `RequestResponse` behaviour, plus an optional
+	/// "response builder" used to build responses for incoming requests.
+	protocols: HashMap<
+		Cow<'static, str>,
+		(RequestResponse<GenericCodec>, Option<mpsc::Sender<IncomingRequest>>)
+	>,
+
+	/// Whenever an incoming request arrives, a `Future` is added to this list and will yield the
+	/// response to send back to the remote.
+	pending_responses: stream::FuturesUnordered<
+		Pin<Box<dyn Future<Output = RequestProcessingOutcome> + Send>>
+	>,
+}
+
+/// Generated by the response builder and waiting to be processed.
+enum RequestProcessingOutcome {
+	Response {
+		protocol: Cow<'static, str>,
+		inner_channel: ResponseChannel<Result<Vec<u8>, ()>>,
+		response: Vec<u8>,
+	},
+	Busy {
+		peer: PeerId,
+		protocol: Cow<'static, str>,
+	},
+}
+
+impl RequestResponsesBehaviour {
+	/// Creates a new behaviour. Must be passed a list of supported protocols. Returns an error if
+	/// the same protocol is passed twice.
+	pub fn new(list: impl Iterator<Item = ProtocolConfig>) -> Result<Self, RegisterError> {
+		let mut protocols = HashMap::new();
+		for protocol in list {
+			let mut cfg = RequestResponseConfig::default();
+			cfg.set_connection_keep_alive(Duration::from_secs(10));
+			cfg.set_request_timeout(protocol.request_timeout);
+
+			let protocol_support = if protocol.inbound_queue.is_some() {
+				ProtocolSupport::Full
+			} else {
+				ProtocolSupport::Outbound
+			};
+
+			let rq_rp = RequestResponse::new(GenericCodec {
+				max_request_size: protocol.max_request_size,
+				max_response_size: protocol.max_response_size,
+			}, iter::once((protocol.name.as_bytes().to_vec(), protocol_support)), cfg);
+
+			match protocols.entry(protocol.name) {
+				Entry::Vacant(e) => e.insert((rq_rp, protocol.inbound_queue)),
+				Entry::Occupied(e) =>
+					return Err(RegisterError::DuplicateProtocol(e.key().clone())),
+			};
+		}
+
+		Ok(Self {
+			protocols,
+			pending_responses: stream::FuturesUnordered::new(),
+		})
+	}
+
+	/// Initiates sending a request.
+	///
+	/// An error is returned if we are not connected to the target peer or if the protocol doesn't
+	/// match one that has been registered.
+	pub fn send_request(&mut self, target: &PeerId, protocol: &str, request: Vec<u8>)
+		-> Result<RequestId, SendRequestError>
+	{
+		if let Some((protocol, _)) = self.protocols.get_mut(protocol) {
+			if protocol.is_connected(target) {
+				Ok(protocol.send_request(target, request))
+			} else {
+				Err(SendRequestError::NotConnected)
+			}
+		} else {
+			Err(SendRequestError::UnknownProtocol)
+		}
+	}
+}
+
+impl NetworkBehaviour for RequestResponsesBehaviour {
+	type ProtocolsHandler = MultiHandler<
+		String,
+		<RequestResponse<GenericCodec> as NetworkBehaviour>::ProtocolsHandler,
+	>;
+	type OutEvent = Event;
+
+	fn new_handler(&mut self) -> Self::ProtocolsHandler {
+		let iter = self.protocols.iter_mut()
+			.map(|(p, (r, _))| (p.to_string(), NetworkBehaviour::new_handler(r)));
+
+		MultiHandler::try_from_iter(iter)
+			.expect("Protocols are in a HashMap and there can be at most one handler per \
+						  protocol name, which is the only possible error; qed")
+	}
+
+	fn addresses_of_peer(&mut self, _: &PeerId) -> Vec<Multiaddr> {
+		Vec::new()
+	}
+
+	fn inject_connection_established(
+		&mut self,
+		peer_id: &PeerId,
+		conn: &ConnectionId,
+		endpoint: &ConnectedPoint,
+	) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_connection_established(p, peer_id, conn, endpoint)
+		}
+	}
+
+	fn inject_connected(&mut self, peer_id: &PeerId) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_connected(p, peer_id)
+		}
+	}
+
+	fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_connection_closed(p, peer_id, conn, endpoint)
+		}
+	}
+
+	fn inject_disconnected(&mut self, peer_id: &PeerId) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_disconnected(p, peer_id)
+		}
+	}
+
+	fn inject_addr_reach_failure(
+		&mut self,
+		peer_id: Option<&PeerId>,
+		addr: &Multiaddr,
+		error: &dyn std::error::Error
+	) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_addr_reach_failure(p, peer_id, addr, error)
+		}
+	}
+
+	fn inject_event(
+		&mut self,
+		peer_id: PeerId,
+		connection: ConnectionId,
+		(p_name, event): <Self::ProtocolsHandler as ProtocolsHandler>::OutEvent,
+	) {
+		if let Some((proto, _)) = self.protocols.get_mut(&*p_name) {
+			return proto.inject_event(peer_id, connection, event)
+		}
+
+		log::warn!(target: "sub-libp2p",
+			"inject_node_event: no request-response instance registered for protocol {:?}",
+			p_name)
+	}
+
+	fn inject_new_external_addr(&mut self, addr: &Multiaddr) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_new_external_addr(p, addr)
+		}
+	}
+
+	fn inject_expired_listen_addr(&mut self, addr: &Multiaddr) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_expired_listen_addr(p, addr)
+		}
+	}
+
+	fn inject_dial_failure(&mut self, peer_id: &PeerId) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_dial_failure(p, peer_id)
+		}
+	}
+
+	fn inject_new_listen_addr(&mut self, addr: &Multiaddr) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_new_listen_addr(p, addr)
+		}
+	}
+
+	fn inject_listener_error(&mut self, id: ListenerId, err: &(dyn std::error::Error + 'static)) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_listener_error(p, id, err)
+		}
+	}
+
+	fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) {
+		for (p, _) in self.protocols.values_mut() {
+			NetworkBehaviour::inject_listener_closed(p, id, reason)
+		}
+	}
+
+	fn poll(
+		&mut self,
+		cx: &mut Context,
+		params: &mut impl PollParameters,
+	) -> Poll<
+		NetworkBehaviourAction<
+			<Self::ProtocolsHandler as ProtocolsHandler>::InEvent,
+			Self::OutEvent,
+		>,
+	> {
+		'poll_all: loop {
+			// Poll to see if any response is ready to be sent back.
+			while let Poll::Ready(Some(result)) = self.pending_responses.poll_next_unpin(cx) {
+				match result {
+					RequestProcessingOutcome::Response {
+						protocol, inner_channel, response
+					} => {
+						if let Some((protocol, _)) = self.protocols.get_mut(&*protocol) {
+							protocol.send_response(inner_channel, Ok(response));
+						}
+					}
+					RequestProcessingOutcome::Busy { peer, protocol } => {
+						let out = Event::InboundRequest {
+							peer,
+							protocol,
+							result: Err(ResponseFailure::Busy),
+						};
+						return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
+					}
+				}
+			}
+
+			// Poll request-responses protocols.
+			for (protocol, (behaviour, resp_builder)) in &mut self.protocols {
+				while let Poll::Ready(ev) = behaviour.poll(cx, params) {
+					let ev = match ev {
+						// Main events we are interested in.
+						NetworkBehaviourAction::GenerateEvent(ev) => ev,
+
+						// Other events generated by the underlying behaviour are transparently
+						// passed through.
+						NetworkBehaviourAction::DialAddress { address } => {
+							log::error!("The request-response isn't supposed to start dialing peers");
+							return Poll::Ready(NetworkBehaviourAction::DialAddress { address })
+						}
+						NetworkBehaviourAction::DialPeer { peer_id, condition } => {
+							log::error!("The request-response isn't supposed to start dialing peers");
+							return Poll::Ready(NetworkBehaviourAction::DialPeer {
+								peer_id,
+								condition,
+							})
+						}
+						NetworkBehaviourAction::NotifyHandler {
+							peer_id,
+							handler,
+							event,
+						} => {
+							return Poll::Ready(NetworkBehaviourAction::NotifyHandler {
+								peer_id,
+								handler,
+								event: ((*protocol).to_string(), event),
+							})
+						}
+						NetworkBehaviourAction::ReportObservedAddr { address } => {
+							return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
+								address,
+							})
+						}
+					};
+
+					match ev {
+						// Received a request from a remote.
+						RequestResponseEvent::Message {
+							peer,
+							message: RequestResponseMessage::Request { request, channel },
+						} => {
+							let (tx, rx) = oneshot::channel();
+
+							// Submit the request to the "response builder" passed by the user at
+							// initialization.
+							if let Some(resp_builder) = resp_builder {
+								// If the response builder is too busy, silently drop `tx`.
+								// This will be reported as a `Busy` error.
+								let _ = resp_builder.try_send(IncomingRequest {
+									peer: peer.clone(),
+									payload: request,
+									pending_response: tx,
+								});
+							}
+
+							let protocol = protocol.clone();
+							self.pending_responses.push(Box::pin(async move {
+								// The `tx` created above can be dropped if we are not capable of
+								// processing this request, which is reflected as a "Busy" error.
+								if let Ok(response) = rx.await {
+									RequestProcessingOutcome::Response {
+										protocol, inner_channel: channel, response
+									}
+								} else {
+									RequestProcessingOutcome::Busy { peer, protocol }
+								}
+							}));
+
+							// This `continue` makes sure that `pending_responses` gets polled
+							// after we have added the new element.
+							continue 'poll_all;
+						}
+
+						// Received a response from a remote to one of our requests.
+						RequestResponseEvent::Message {
+							message:
+								RequestResponseMessage::Response {
+									request_id,
+									response,
+								},
+							..
+						} => {
+							let out = Event::RequestFinished {
+								request_id,
+								result: response.map_err(|()| RequestFailure::Refused),
+							};
+							return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
+						}
+
+						// One of our requests has failed.
+						RequestResponseEvent::OutboundFailure {
+							request_id,
+							error,
+							..
+						} => {
+							let out = Event::RequestFinished {
+								request_id,
+								result: Err(RequestFailure::Network(error)),
+							};
+							return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
+						}
+
+						// Remote has tried to send a request but failed.
+						RequestResponseEvent::InboundFailure { peer, error } => {
+							let out = Event::InboundRequest {
+								peer,
+								protocol: protocol.clone(),
+								result: Err(ResponseFailure::Network(error)),
+							};
+							return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
+						}
+					};
+				}
+			}
+
+			break Poll::Pending;
+		}
+	}
+}
+
+/// Error when registering a protocol.
+#[derive(Debug, derive_more::Display, derive_more::Error)]
+pub enum RegisterError {
+	/// A protocol has been specified multiple times.
+	DuplicateProtocol(#[error(ignore)] Cow<'static, str>),
+}
+
+/// Error when sending a request.
+#[derive(Debug, derive_more::Display, derive_more::Error)]
+pub enum SendRequestError {
+	/// We are not currently connected to the requested peer.
+	NotConnected,
+	/// Given protocol hasn't been registered.
+	UnknownProtocol,
+}
+
+/// Error in a request.
+#[derive(Debug, derive_more::Display, derive_more::Error)]
+pub enum RequestFailure {
+	/// Remote has closed the substream before answering, thereby signaling that it considers the
+	/// request as valid, but refused to answer it.
+	Refused,
+	/// Problem on the network.
+	#[display(fmt = "Problem on the network")]
+	Network(#[error(ignore)] OutboundFailure),
+}
+
+/// Error when processing a request sent by a remote.
+#[derive(Debug, derive_more::Display, derive_more::Error)]
+pub enum ResponseFailure {
+	/// Internal response builder is too busy to process this request.
+	Busy,
+	/// Problem on the network.
+	#[display(fmt = "Problem on the network")]
+	Network(#[error(ignore)] InboundFailure),
+}
+
+/// Implements the libp2p [`RequestResponseCodec`] trait. Defines how streams of bytes are turned
+/// into requests and responses and vice-versa.
+#[derive(Debug, Clone)]
+#[doc(hidden)]  // Needs to be public in order to satisfy the Rust compiler.
+pub struct GenericCodec {
+	max_request_size: u64,
+	max_response_size: u64,
+}
+
+#[async_trait::async_trait]
+impl RequestResponseCodec for GenericCodec {
+	type Protocol = Vec<u8>;
+	type Request = Vec<u8>;
+	type Response = Result<Vec<u8>, ()>;
+
+	async fn read_request<T>(
+		&mut self,
+		_: &Self::Protocol,
+		mut io: &mut T,
+	) -> io::Result<Self::Request>
+	where
+		T: AsyncRead + Unpin + Send,
+	{
+		// Read the length.
+		let length = unsigned_varint::aio::read_usize(&mut io).await
+			.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
+		if length > usize::try_from(self.max_request_size).unwrap_or(usize::max_value()) {
+			return Err(io::Error::new(
+				io::ErrorKind::InvalidInput,
+				format!("Request size exceeds limit: {} > {}", length, self.max_request_size)
+			));
+		}
+
+		// Read the payload.
+		let mut buffer = vec![0; length];
+		io.read_exact(&mut buffer).await?;
+		Ok(buffer)
+	}
+
+	async fn read_response<T>(
+		&mut self,
+		_: &Self::Protocol,
+		mut io: &mut T,
+	) -> io::Result<Self::Response>
+	where
+		T: AsyncRead + Unpin + Send,
+	{
+		// Note that this function returns a `Result<Result<...>>`. Returning an `Err` is
+		// considered as a protocol error and will result in the entire connection being closed.
+		// Returning `Ok(Err(_))` signifies that a response has successfully been fetched, and
+		// that this response is an error.
+
+		// Read the length.
+		let length = match unsigned_varint::aio::read_usize(&mut io).await {
+			Ok(l) => l,
+			Err(unsigned_varint::io::ReadError::Io(err))
+				if matches!(err.kind(), io::ErrorKind::UnexpectedEof) =>
+			{
+				return Ok(Err(()));
+			}
+			Err(err) => return Err(io::Error::new(io::ErrorKind::InvalidInput, err)),
+		};
+
+		if length > usize::try_from(self.max_response_size).unwrap_or(usize::max_value()) {
+			return Err(io::Error::new(
+				io::ErrorKind::InvalidInput,
+				format!("Response size exceeds limit: {} > {}", length, self.max_response_size)
+			));
+		}
+
+		// Read the payload.
+		let mut buffer = vec![0; length];
+		io.read_exact(&mut buffer).await?;
+		Ok(Ok(buffer))
+	}
+
+	async fn write_request<T>(
+		&mut self,
+		_: &Self::Protocol,
+		io: &mut T,
+		req: Self::Request,
+	) -> io::Result<()>
+	where
+		T: AsyncWrite + Unpin + Send,
+	{
+		// TODO: check the length?
+		// Write the length.
+		{
+			let mut buffer = unsigned_varint::encode::usize_buffer();
+			io.write_all(unsigned_varint::encode::usize(req.len(), &mut buffer)).await?;
+		}
+
+		// Write the payload.
+		io.write_all(&req).await?;
+
+		io.close().await?;
+		Ok(())
+	}
+
+	async fn write_response<T>(
+		&mut self,
+		_: &Self::Protocol,
+		io: &mut T,
+		res: Self::Response,
+	) -> io::Result<()>
+	where
+		T: AsyncWrite + Unpin + Send,
+	{
+		// If `res` is an `Err`, we jump to closing the substream without writing anything on it.
+		if let Ok(res) = res {
+			// TODO: check the length?
+			// Write the length.
+			{
+				let mut buffer = unsigned_varint::encode::usize_buffer();
+				io.write_all(unsigned_varint::encode::usize(res.len(), &mut buffer)).await?;
+			}
+
+			// Write the payload.
+			io.write_all(&res).await?;
+		}
+
+		io.close().await?;
+		Ok(())
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use futures::{channel::mpsc, prelude::*};
+	use libp2p::identity::Keypair;
+	use libp2p::Multiaddr;
+	use libp2p::core::upgrade;
+	use libp2p::core::transport::{Transport, MemoryTransport};
+	use libp2p::core::upgrade::{InboundUpgradeExt, OutboundUpgradeExt};
+	use libp2p::swarm::{Swarm, SwarmEvent};
+	use std::{iter, time::Duration};
+
+	#[test]
+	fn basic_request_response_works() {
+		let protocol_name = "/test/req-rep/1";
+
+		// Build swarms whose behaviour is `RequestResponsesBehaviour`.
+		let mut swarms = (0..2)
+			.map(|_| {
+				let keypair = Keypair::generate_ed25519();
+				let keypair2 = keypair.clone();
+
+				let transport = MemoryTransport
+					.and_then(move |out, endpoint| {
+						let secio = libp2p::secio::SecioConfig::new(keypair2);
+						libp2p::core::upgrade::apply(
+							out,
+							secio,
+							endpoint,
+							upgrade::Version::V1
+						)
+					})
+					.and_then(move |(peer_id, stream), endpoint| {
+						let peer_id2 = peer_id.clone();
+						let upgrade = libp2p::yamux::Config::default()
+							.map_inbound(move |muxer| (peer_id, muxer))
+							.map_outbound(move |muxer| (peer_id2, muxer));
+						upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1)
+					});
+
+				let behaviour = {
+					let (tx, mut rx) = mpsc::channel(64);
+
+					let b = super::RequestResponsesBehaviour::new(iter::once(super::ProtocolConfig {
+						name: From::from(protocol_name),
+						max_request_size: 1024,
+						max_response_size: 1024 * 1024,
+						request_timeout: Duration::from_secs(30),
+						inbound_queue: Some(tx),
+					})).unwrap();
+
+					async_std::task::spawn(async move {
+						while let Some(rq) = rx.next().await {
+							assert_eq!(rq.payload, b"this is a request");
+							let _ = rq.pending_response.send(b"this is a response".to_vec());
+						}
+					});
+
+					b
+				};
+
+				let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id());
+				let listen_addr: Multiaddr = format!("/memory/{}", rand::random::<u64>()).parse().unwrap();
+
+				Swarm::listen_on(&mut swarm, listen_addr.clone()).unwrap();
+				(swarm, listen_addr)
+			})
+			.collect::<Vec<_>>();
+
+		// Ask `swarm[0]` to dial `swarm[1]`. There isn't any discovery mechanism in place in
+		// this test, so they wouldn't connect to each other.
+		{
+			let dial_addr = swarms[1].1.clone();
+			Swarm::dial_addr(&mut swarms[0].0, dial_addr).unwrap();
+		}
+
+		// Running `swarm[0]` in the background until a `InboundRequest` event happens,
+		// which is a hint about the test having ended.
+		async_std::task::spawn({
+			let (mut swarm, _) = swarms.remove(0);
+			async move {
+				loop {
+					match swarm.next_event().await {
+						SwarmEvent::Behaviour(super::Event::InboundRequest { result, .. }) => {
+							assert!(result.is_ok());
+							break
+						},
+						_ => {}
+					}
+				}
+			}
+		});
+
+		// Remove and run the remaining swarm.
+		let (mut swarm, _) = swarms.remove(0);
+		async_std::task::block_on(async move {
+			let mut sent_request_id = None;
+
+			loop {
+				match swarm.next_event().await {
+					SwarmEvent::ConnectionEstablished { peer_id, .. } => {
+						let id = swarm.send_request(
+							&peer_id,
+							protocol_name,
+							b"this is a request".to_vec()
+						).unwrap();
+						assert!(sent_request_id.is_none());
+						sent_request_id = Some(id);
+					}
+					SwarmEvent::Behaviour(super::Event::RequestFinished {
+						request_id,
+						result,
+					}) => {
+						assert_eq!(Some(request_id), sent_request_id);
+						let result = result.unwrap();
+						assert_eq!(result, b"this is a response");
+						break;
+					}
+					_ => {}
+				}
+			}
+		});
+	}
+
+	#[test]
+	fn max_response_size_exceeded() {
+		let protocol_name = "/test/req-rep/1";
+
+		// Build swarms whose behaviour is `RequestResponsesBehaviour`.
+		let mut swarms = (0..2)
+			.map(|_| {
+				let keypair = Keypair::generate_ed25519();
+				let keypair2 = keypair.clone();
+
+				let transport = MemoryTransport
+					.and_then(move |out, endpoint| {
+						let secio = libp2p::secio::SecioConfig::new(keypair2);
+						libp2p::core::upgrade::apply(
+							out,
+							secio,
+							endpoint,
+							upgrade::Version::V1
+						)
+					})
+					.and_then(move |(peer_id, stream), endpoint| {
+						let peer_id2 = peer_id.clone();
+						let upgrade = libp2p::yamux::Config::default()
+							.map_inbound(move |muxer| (peer_id, muxer))
+							.map_outbound(move |muxer| (peer_id2, muxer));
+						upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1)
+					});
+
+				let behaviour = {
+					let (tx, mut rx) = mpsc::channel(64);
+
+					let b = super::RequestResponsesBehaviour::new(iter::once(super::ProtocolConfig {
+						name: From::from(protocol_name),
+						max_request_size: 1024,
+						max_response_size: 8,  // <-- important for the test
+						request_timeout: Duration::from_secs(30),
+						inbound_queue: Some(tx),
+					})).unwrap();
+
+					async_std::task::spawn(async move {
+						while let Some(rq) = rx.next().await {
+							assert_eq!(rq.payload, b"this is a request");
+							let _ = rq.pending_response.send(b"this response exceeds the limit".to_vec());
+						}
+					});
+
+					b
+				};
+
+				let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id());
+				let listen_addr: Multiaddr = format!("/memory/{}", rand::random::<u64>()).parse().unwrap();
+
+				Swarm::listen_on(&mut swarm, listen_addr.clone()).unwrap();
+				(swarm, listen_addr)
+			})
+			.collect::<Vec<_>>();
+
+		// Ask `swarm[0]` to dial `swarm[1]`. There isn't any discovery mechanism in place in
+		// this test, so they wouldn't connect to each other.
+		{
+			let dial_addr = swarms[1].1.clone();
+			Swarm::dial_addr(&mut swarms[0].0, dial_addr).unwrap();
+		}
+
+		// Running `swarm[0]` in the background until a `InboundRequest` event happens,
+		// which is a hint about the test having ended.
+		async_std::task::spawn({
+			let (mut swarm, _) = swarms.remove(0);
+			async move {
+				loop {
+					match swarm.next_event().await {
+						SwarmEvent::Behaviour(super::Event::InboundRequest { result, .. }) => {
+							assert!(result.is_ok());
+							break
+						},
+						_ => {}
+					}
+				}
+			}
+		});
+
+		// Remove and run the remaining swarm.
+		let (mut swarm, _) = swarms.remove(0);
+		async_std::task::block_on(async move {
+			let mut sent_request_id = None;
+
+			loop {
+				match swarm.next_event().await {
+					SwarmEvent::ConnectionEstablished { peer_id, .. } => {
+						let id = swarm.send_request(
+							&peer_id,
+							protocol_name,
+							b"this is a request".to_vec()
+						).unwrap();
+						assert!(sent_request_id.is_none());
+						sent_request_id = Some(id);
+					}
+					SwarmEvent::Behaviour(super::Event::RequestFinished {
+						request_id,
+						result,
+					}) => {
+						assert_eq!(Some(request_id), sent_request_id);
+						match result {
+							Err(super::RequestFailure::Network(super::OutboundFailure::ConnectionClosed)) => {},
+							_ => panic!()
+						}
+						break;
+					}
+					_ => {}
+				}
+			}
+		});
+	}
+}
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index 6f7751f430751..754b5b184c096 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -29,7 +29,7 @@
 
 use crate::{
 	ExHashT, NetworkStateInfo,
-	behaviour::{Behaviour, BehaviourOut},
+	behaviour::{self, Behaviour, BehaviourOut},
 	config::{parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig},
 	DhtEvent,
 	discovery::DiscoveryConfig,
@@ -42,7 +42,7 @@ use crate::{
 	protocol::{self, event::Event, NotifsHandlerError, LegacyConnectionKillError, NotificationsSink, Ready, sync::SyncState, PeerInfo, Protocol},
 	transport, ReputationChange,
 };
-use futures::prelude::*;
+use futures::{channel::oneshot, prelude::*};
 use libp2p::{PeerId, multiaddr, Multiaddr};
 use libp2p::core::{ConnectedPoint, Executor, connection::{ConnectionError, PendingConnectionError}, either::EitherError};
 use libp2p::kad::record;
@@ -76,6 +76,9 @@ use std::{
 	},
 	task::Poll,
 };
+use wasm_timer::Instant;
+
+pub use behaviour::{ResponseFailure, InboundFailure, RequestFailure, OutboundFailure};
 
 mod out_events;
 #[cfg(test)]
@@ -309,16 +312,28 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 				config
 			};
 
-			let mut behaviour = Behaviour::new(
-				protocol,
-				params.role,
-				user_agent,
-				local_public,
-				block_requests,
-				finality_proof_requests,
-				light_client_handler,
-				discovery_config
-			);
+			let mut behaviour = {
+				let result = Behaviour::new(
+					protocol,
+					params.role,
+					user_agent,
+					local_public,
+					block_requests,
+					finality_proof_requests,
+					light_client_handler,
+					discovery_config,
+					params.network_config.request_response_protocols,
+				);
+
+				match result {
+					Ok(b) => b,
+					Err(crate::request_responses::RegisterError::DuplicateProtocol(proto)) => {
+						return Err(Error::DuplicateRequestResponseProtocol {
+							protocol: proto,
+						})
+					},
+				}
+			};
 
 			for (engine_id, protocol_name) in &params.network_config.notifications_protocols {
 				behaviour.register_notifications_protocol(*engine_id, protocol_name.clone());
@@ -404,6 +419,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 			peers_notifications_sinks,
 			metrics,
 			boot_node_ids,
+			pending_requests: HashMap::with_capacity(128),
 		})
 	}
 
@@ -752,12 +768,50 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// parameter is a `&'static str`, and not a `String`, in order to avoid accidentally having
 	/// an unbounded set of Prometheus metrics, which would be quite bad in terms of memory
 	pub fn event_stream(&self, name: &'static str) -> impl Stream<Item = Event> {
-		// Note: when transitioning to stable futures, remove the `Error` entirely
 		let (tx, rx) = out_events::channel(name);
 		let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::EventStream(tx));
 		rx
 	}
 
+	/// Sends a single targeted request to a specific peer. On success, returns the response of
+	/// the peer.
+	///
+	/// Request-response protocols are a way to complement notifications protocols, but
+	/// notifications should remain the default ways of communicating information. For example, a
+	/// peer can announce something through a notification, after which the recipient can obtain
+	/// more information by performing a request.
+	/// As such, this function is meant to be called only with peers we are already connected to.
+	/// Calling this method with a `target` we are not connected to will *not* attempt to connect
+	/// to said peer.
+	///
+	/// No limit or throttling of concurrent outbound requests per peer and protocol are enforced.
+	/// Such restrictions, if desired, need to be enforced at the call site(s).
+	///
+	/// The protocol must have been registered through
+	/// [`NetworkConfiguration::request_response_protocols`].
+	pub async fn request(
+		&self,
+		target: PeerId,
+		protocol: impl Into<Cow<'static, str>>,
+		request: Vec<u8>
+	) -> Result<Vec<u8>, RequestFailure> {
+		let (tx, rx) = oneshot::channel();
+		let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::Request {
+			target,
+			protocol: protocol.into(),
+			request,
+			pending_response: tx
+		});
+
+		match rx.await {
+			Ok(v) => v,
+			// The channel can only be closed if the network worker no longer exists. If the
+			// network worker no longer exists, then all connections to `target` are necessarily
+			// closed, and we legitimately report this situation as a "ConnectionClosed".
+			Err(_) => Err(RequestFailure::Network(OutboundFailure::ConnectionClosed)),
+		}
+	}
+
 	/// Registers a new notifications protocol.
 	///
 	/// After a protocol has been registered, you can call `write_notifications`.
@@ -1096,6 +1150,12 @@ enum ServiceToWorkerMsg<B: BlockT, H: ExHashT> {
 	AddKnownAddress(PeerId, Multiaddr),
 	SyncFork(Vec<PeerId>, B::Hash, NumberFor<B>),
 	EventStream(out_events::Sender),
+	Request {
+		target: PeerId,
+		protocol: Cow<'static, str>,
+		request: Vec<u8>,
+		pending_response: oneshot::Sender<Result<Vec<u8>, RequestFailure>>,
+	},
 	RegisterNotifProtocol {
 		engine_id: ConsensusEngineId,
 		protocol_name: Cow<'static, [u8]>,
@@ -1132,6 +1192,13 @@ pub struct NetworkWorker<B: BlockT + 'static, H: ExHashT> {
 	metrics: Option<Metrics>,
 	/// The `PeerId`'s of all boot nodes.
 	boot_node_ids: Arc<HashSet<PeerId>>,
+	/// Requests started using [`NetworkService::request`]. Includes the channel to send back the
+	/// response, when the request has started, and the name of the protocol for diagnostic
+	/// purposes.
+	pending_requests: HashMap<
+		behaviour::RequestId,
+		(oneshot::Sender<Result<Vec<u8>, RequestFailure>>, Instant, String)
+	>,
 	/// For each peer and protocol combination, an object that allows sending notifications to
 	/// that peer. Shared with the [`NetworkService`].
 	peers_notifications_sinks: Arc<Mutex<HashMap<(PeerId, ConsensusEngineId), NotificationsSink>>>,
@@ -1165,8 +1232,10 @@ struct Metrics {
 	peerset_num_requested: Gauge<U64>,
 	pending_connections: Gauge<U64>,
 	pending_connections_errors_total: CounterVec<U64>,
-	requests_in_total: HistogramVec,
-	requests_out_finished: HistogramVec,
+	requests_in_failure_total: CounterVec<U64>,
+	requests_in_success_total: HistogramVec,
+	requests_out_failure_total: CounterVec<U64>,
+	requests_out_success_total: HistogramVec,
 	requests_out_started_total: CounterVec<U64>,
 }
 
@@ -1347,10 +1416,17 @@ impl Metrics {
 				),
 				&["reason"]
 			)?, registry)?,
-			requests_in_total: register(HistogramVec::new(
+			requests_in_failure_total: register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_requests_in_failure_total",
+					"Total number of incoming requests that the node has failed to answer"
+				),
+				&["protocol", "reason"]
+			)?, registry)?,
+			requests_in_success_total: register(HistogramVec::new(
 				HistogramOpts {
 					common_opts: Opts::new(
-						"sub_libp2p_requests_in_total",
+						"sub_libp2p_requests_in_success_total",
 						"Total number of requests received and answered"
 					),
 					buckets: prometheus_endpoint::exponential_buckets(0.001, 2.0, 16)
@@ -1358,11 +1434,18 @@ impl Metrics {
 				},
 				&["protocol"]
 			)?, registry)?,
-			requests_out_finished: register(HistogramVec::new(
+			requests_out_failure_total: register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_requests_out_failure_total",
+					"Total number of requests that have failed"
+				),
+				&["protocol", "reason"]
+			)?, registry)?,
+			requests_out_success_total: register(HistogramVec::new(
 				HistogramOpts {
 					common_opts: Opts::new(
-						"sub_libp2p_requests_out_finished",
-						"Time between a request's start and finish (successful or not)"
+						"sub_libp2p_requests_out_success_total",
+						"For successful requests, time between a request's start and finish"
 					),
 					buckets: prometheus_endpoint::exponential_buckets(0.001, 2.0, 16)
 						.expect("parameters are always valid values; qed"),
@@ -1446,6 +1529,31 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 					this.network_service.user_protocol_mut().set_sync_fork_request(peer_ids, &hash, number),
 				ServiceToWorkerMsg::EventStream(sender) =>
 					this.event_streams.push(sender),
+				ServiceToWorkerMsg::Request { target, protocol, request, pending_response } => {
+					// Calling `send_request` can fail immediately in some circumstances.
+					// This is handled by sending back an error on the channel.
+					match this.network_service.send_request(&target, &protocol, request) {
+						Ok(request_id) => {
+							if let Some(metrics) = this.metrics.as_ref() {
+								metrics.requests_out_started_total
+									.with_label_values(&[&protocol])
+									.inc();
+							}
+							this.pending_requests.insert(
+								request_id,
+								(pending_response, Instant::now(), protocol.to_string())
+							);
+						},
+						Err(behaviour::SendRequestError::NotConnected) => {
+							let err = RequestFailure::Network(OutboundFailure::ConnectionClosed);
+							let _ = pending_response.send(Err(err));
+						},
+						Err(behaviour::SendRequestError::UnknownProtocol) => {
+							let err = RequestFailure::Network(OutboundFailure::UnsupportedProtocols);
+							let _ = pending_response.send(Err(err));
+						},
+					}
+				},
 				ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, protocol_name } => {
 					this.network_service
 						.register_notifications_protocol(engine_id, protocol_name);
@@ -1494,23 +1602,72 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 					}
 					this.import_queue.import_finality_proof(origin, hash, nb, proof);
 				},
-				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::AnsweredRequest { protocol, build_time, .. })) => {
+				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::InboundRequest { protocol, result, .. })) => {
 					if let Some(metrics) = this.metrics.as_ref() {
-						metrics.requests_in_total
-							.with_label_values(&[&protocol])
-							.observe(build_time.as_secs_f64());
+						match result {
+							Ok(serve_time) => {
+								metrics.requests_in_success_total
+									.with_label_values(&[&protocol])
+									.observe(serve_time.as_secs_f64());
+							}
+							Err(err) => {
+								let reason = match err {
+									ResponseFailure::Busy => "busy",
+									ResponseFailure::Network(InboundFailure::Timeout) => "timeout",
+									ResponseFailure::Network(InboundFailure::UnsupportedProtocols) =>
+										"unsupported",
+								};
+
+								metrics.requests_in_failure_total
+									.with_label_values(&[&protocol, reason])
+									.inc();
+							}
+						}
+					}
+				},
+				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { request_id, result })) => {
+					if let Some((send_back, started, protocol)) = this.pending_requests.remove(&request_id) {
+						if let Some(metrics) = this.metrics.as_ref() {
+							match &result {
+								Ok(_) => {
+									metrics.requests_out_success_total
+										.with_label_values(&[&protocol])
+										.observe(started.elapsed().as_secs_f64());
+								}
+								Err(err) => {
+									let reason = match err {
+										RequestFailure::Refused => "refused",
+										RequestFailure::Network(OutboundFailure::DialFailure) =>
+											"dial-failure",
+										RequestFailure::Network(OutboundFailure::Timeout) =>
+											"timeout",
+										RequestFailure::Network(OutboundFailure::ConnectionClosed) =>
+											"connection-closed",
+										RequestFailure::Network(OutboundFailure::UnsupportedProtocols) =>
+											"unsupported",
+									};
+
+									metrics.requests_out_failure_total
+										.with_label_values(&[&protocol, reason])
+										.inc();
+								}
+							}
+						}
+						let _ = send_back.send(result);
+					} else {
+						error!("Request not in pending_requests");
 					}
 				},
-				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestStarted { protocol, .. })) => {
+				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestStarted { protocol, .. })) => {
 					if let Some(metrics) = this.metrics.as_ref() {
 						metrics.requests_out_started_total
 							.with_label_values(&[&protocol])
 							.inc();
 					}
 				},
-				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { protocol, request_duration, .. })) => {
+				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestFinished { protocol, request_duration, .. })) => {
 					if let Some(metrics) = this.metrics.as_ref() {
-						metrics.requests_out_finished
+						metrics.requests_out_success_total
 							.with_label_values(&[&protocol])
 							.observe(request_duration.as_secs_f64());
 					}
@@ -1635,14 +1792,14 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 						let reason = match cause {
 							Some(ConnectionError::IO(_)) => "transport-error",
 							Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A(
-								EitherError::A(EitherError::A(EitherError::B(
-								EitherError::A(PingFailure::Timeout))))))))) => "ping-timeout",
+								EitherError::A(EitherError::A(EitherError::A(EitherError::B(
+								EitherError::A(PingFailure::Timeout)))))))))) => "ping-timeout",
 							Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A(
-								EitherError::A(EitherError::A(EitherError::A(
-								NotifsHandlerError::Legacy(LegacyConnectionKillError))))))))) =>	"force-closed",
+								EitherError::A(EitherError::A(EitherError::A(EitherError::A(
+								NotifsHandlerError::Legacy(LegacyConnectionKillError)))))))))) => "force-closed",
 							Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A(
-								EitherError::A(EitherError::A(EitherError::A(
-								NotifsHandlerError::SyncNotificationsClogged)))))))) => "sync-notifications-clogged",
+								EitherError::A(EitherError::A(EitherError::A(EitherError::A(
+								NotifsHandlerError::SyncNotificationsClogged))))))))) => "sync-notifications-clogged",
 							Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(_))) => "protocol-error",
 							Some(ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout)) => "keep-alive-timeout",
 							None => "actively-closed",
@@ -1800,7 +1957,7 @@ impl<B: BlockT + 'static, H: ExHashT> Unpin for NetworkWorker<B, H> {
 /// Turns bytes that are potentially UTF-8 into a reasonable representable string.
 ///
 /// Meant to be used only for debugging or metrics-reporting purposes.
-fn maybe_utf8_bytes_to_string(id: &[u8]) -> Cow<str> {
+pub(crate) fn maybe_utf8_bytes_to_string(id: &[u8]) -> Cow<str> {
 	if let Ok(s) = std::str::from_utf8(&id[..]) {
 		Cow::Borrowed(s)
 	} else {

From e4178cc4850b1fd109a251f304e815b100c29a71 Mon Sep 17 00:00:00 2001
From: Alexander Popiak <alexander.popiak@parity.io>
Date: Fri, 28 Aug 2020 10:46:43 +0200
Subject: [PATCH 024/122] add generated weight info for pallet-collective
 (#6789)

* add benchmark for disapprove_proposal

* use generated WeightInfo for pallet-collective weights

* order collective benchmark params alphabetically to get a consistent ordering

* address review comments

* remove default impl of WeightInfo for ()

* remove comments about weight changes

* add default weights

* Apply suggestions from code review

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>

* whitelist voter account in benchmark

* update weights

* MaxMembers configurable

* remove base weight comment

* add weight to technical collective

* another DB whitelist optimization

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>
---
 bin/node/runtime/src/lib.rs                   |  12 +-
 bin/node/runtime/src/weights/mod.rs           |   1 +
 .../runtime/src/weights/pallet_collective.rs  |  97 +++++
 frame/collective/src/benchmarking.rs          | 119 ++++--
 frame/collective/src/default_weight.rs        |  97 +++++
 frame/collective/src/lib.rs                   | 349 ++++++------------
 6 files changed, 400 insertions(+), 275 deletions(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_collective.rs
 create mode 100644 frame/collective/src/default_weight.rs

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index 17c02eca17b4a..54dea704bd7f6 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -509,6 +509,7 @@ impl pallet_democracy::Trait for Runtime {
 parameter_types! {
 	pub const CouncilMotionDuration: BlockNumber = 5 * DAYS;
 	pub const CouncilMaxProposals: u32 = 100;
+	pub const CouncilMaxMembers: u32 = 100;
 }
 
 type CouncilCollective = pallet_collective::Instance1;
@@ -518,7 +519,8 @@ impl pallet_collective::Trait<CouncilCollective> for Runtime {
 	type Event = Event;
 	type MotionDuration = CouncilMotionDuration;
 	type MaxProposals = CouncilMaxProposals;
-	type WeightInfo = ();
+	type MaxMembers = CouncilMaxMembers;
+	type WeightInfo = weights::pallet_collective::WeightInfo;
 }
 
 parameter_types! {
@@ -530,8 +532,8 @@ parameter_types! {
 	pub const ElectionsPhragmenModuleId: LockIdentifier = *b"phrelect";
 }
 
-// Make sure that there are no more than `MAX_MEMBERS` members elected via elections-phragmen.
-const_assert!(DesiredMembers::get() <= pallet_collective::MAX_MEMBERS);
+// Make sure that there are no more than `MaxMembers` members elected via elections-phragmen.
+const_assert!(DesiredMembers::get() <= CouncilMaxMembers::get());
 
 impl pallet_elections_phragmen::Trait for Runtime {
 	type Event = Event;
@@ -556,6 +558,7 @@ impl pallet_elections_phragmen::Trait for Runtime {
 parameter_types! {
 	pub const TechnicalMotionDuration: BlockNumber = 5 * DAYS;
 	pub const TechnicalMaxProposals: u32 = 100;
+	pub const TechnicalMaxMembers: u32 = 100;
 }
 
 type TechnicalCollective = pallet_collective::Instance2;
@@ -565,7 +568,8 @@ impl pallet_collective::Trait<TechnicalCollective> for Runtime {
 	type Event = Event;
 	type MotionDuration = TechnicalMotionDuration;
 	type MaxProposals = TechnicalMaxProposals;
-	type WeightInfo = ();
+	type MaxMembers = TechnicalMaxMembers;
+	type WeightInfo = weights::pallet_collective::WeightInfo;
 }
 
 type EnsureRootOrHalfCouncil = EnsureOneOf<
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index 322fb8886c0f4..372b13a093e27 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -17,6 +17,7 @@
 
 pub mod frame_system;
 pub mod pallet_balances;
+pub mod pallet_collective;
 pub mod pallet_democracy;
 pub mod pallet_proxy;
 pub mod pallet_timestamp;
diff --git a/bin/node/runtime/src/weights/pallet_collective.rs b/bin/node/runtime/src/weights/pallet_collective.rs
new file mode 100644
index 0000000000000..32b4ad02d7aa9
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_collective.rs
@@ -0,0 +1,97 @@
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_collective::WeightInfo for WeightInfo {
+	fn set_members(m: u32, n: u32, p: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((21040000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((173000 as Weight).saturating_mul(n as Weight))
+			.saturating_add((31595000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(p as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight)))
+	}
+	fn execute(b: u32, m: u32, ) -> Weight {
+		(43359000 as Weight)
+			.saturating_add((4000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((123000 as Weight).saturating_mul(m as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+	}
+	fn propose_execute(b: u32, m: u32, ) -> Weight {
+		(54134000 as Weight)
+			.saturating_add((4000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((239000 as Weight).saturating_mul(m as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+	}
+	fn propose_proposed(b: u32, m: u32, p: u32, ) -> Weight {
+		(90650000 as Weight)
+			.saturating_add((5000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((152000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((970000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn vote(m: u32, ) -> Weight {
+		(74460000 as Weight)
+			.saturating_add((290000 as Weight).saturating_mul(m as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn close_early_disapproved(m: u32, p: u32, ) -> Weight {
+		(86360000 as Weight)
+			.saturating_add((232000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((954000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_early_approved(b: u32, m: u32, p: u32, ) -> Weight {
+		(123653000 as Weight)
+			.saturating_add((1000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((287000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((920000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_disapproved(m: u32, p: u32, ) -> Weight {
+		(95395000 as Weight)
+			.saturating_add((236000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((965000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_approved(b: u32, m: u32, p: u32, ) -> Weight {
+		(135284000 as Weight)
+			.saturating_add((4000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((218000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((951000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn disapprove_proposal(p: u32, ) -> Weight {
+		(50500000 as Weight)
+			.saturating_add((966000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+}
diff --git a/frame/collective/src/benchmarking.rs b/frame/collective/src/benchmarking.rs
index 2c777fadc4cc4..d4e80d515941f 100644
--- a/frame/collective/src/benchmarking.rs
+++ b/frame/collective/src/benchmarking.rs
@@ -45,8 +45,8 @@ benchmarks_instance! {
 	_{ }
 
 	set_members {
-		let m in 1 .. MAX_MEMBERS;
-		let n in 1 .. MAX_MEMBERS;
+		let m in 1 .. T::MaxMembers::get();
+		let n in 1 .. T::MaxMembers::get();
 		let p in 1 .. T::MaxProposals::get();
 
 		// Set old members.
@@ -63,7 +63,7 @@ benchmarks_instance! {
 			SystemOrigin::Root.into(),
 			old_members.clone(),
 			Some(last_old_member.clone()),
-			MAX_MEMBERS,
+			T::MaxMembers::get(),
 		)?;
 
 		// Set a high threshold for proposals passing so that they stay around.
@@ -104,15 +104,15 @@ benchmarks_instance! {
 			new_members.push(last_member.clone());
 		}
 
-	}: _(SystemOrigin::Root, new_members.clone(), Some(last_member), MAX_MEMBERS)
+	}: _(SystemOrigin::Root, new_members.clone(), Some(last_member), T::MaxMembers::get())
 	verify {
 		new_members.sort();
 		assert_eq!(Collective::<T, _>::members(), new_members);
 	}
 
 	execute {
-		let m in 1 .. MAX_MEMBERS;
 		let b in 1 .. MAX_BYTES;
+		let m in 1 .. T::MaxMembers::get();
 
 		let bytes_in_storage = b + size_of::<u32>() as u32;
 
@@ -126,7 +126,7 @@ benchmarks_instance! {
 		let caller: T::AccountId = whitelisted_caller();
 		members.push(caller.clone());
 
-		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members, None, MAX_MEMBERS)?;
+		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members, None, T::MaxMembers::get())?;
 
 		let proposal: T::Proposal = SystemCall::<T>::remark(vec![1; b as usize]).into();
 
@@ -141,8 +141,8 @@ benchmarks_instance! {
 
 	// This tests when execution would happen immediately after proposal
 	propose_execute {
-		let m in 1 .. MAX_MEMBERS;
 		let b in 1 .. MAX_BYTES;
+		let m in 1 .. T::MaxMembers::get();
 
 		let bytes_in_storage = b + size_of::<u32>() as u32;
 
@@ -156,7 +156,7 @@ benchmarks_instance! {
 		let caller: T::AccountId = whitelisted_caller();
 		members.push(caller.clone());
 
-		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members, None, MAX_MEMBERS)?;
+		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members, None, T::MaxMembers::get())?;
 
 		let proposal: T::Proposal = SystemCall::<T>::remark(vec![1; b as usize]).into();
 		let threshold = 1;
@@ -172,9 +172,9 @@ benchmarks_instance! {
 
 	// This tests when proposal is created and queued as "proposed"
 	propose_proposed {
-		let m in 2 .. MAX_MEMBERS;
-		let p in 1 .. T::MaxProposals::get();
 		let b in 1 .. MAX_BYTES;
+		let m in 2 .. T::MaxMembers::get();
+		let p in 1 .. T::MaxProposals::get();
 
 		let bytes_in_storage = b + size_of::<u32>() as u32;
 
@@ -186,7 +186,7 @@ benchmarks_instance! {
 		}
 		let caller: T::AccountId = whitelisted_caller();
 		members.push(caller.clone());
-		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members, None, MAX_MEMBERS)?;
+		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members, None, T::MaxMembers::get())?;
 
 		let threshold = m;
 		// Add previous proposals.
@@ -215,7 +215,7 @@ benchmarks_instance! {
 
 	vote {
 		// We choose 5 as a minimum so we always trigger a vote in the voting loop (`for j in ...`)
-		let m in 5 .. MAX_MEMBERS;
+		let m in 5 .. T::MaxMembers::get();
 
 		let p = T::MaxProposals::get();
 		let b = MAX_BYTES;
@@ -231,7 +231,7 @@ benchmarks_instance! {
 		}
 		let voter: T::AccountId = account("voter", 0, SEED);
 		members.push(voter.clone());
-		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members.clone(), None, MAX_MEMBERS)?;
+		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members.clone(), None, T::MaxMembers::get())?;
 
 		// Threshold is 1 less than the number of members so that one person can vote nay
 		let threshold = m - 1;
@@ -277,6 +277,9 @@ benchmarks_instance! {
 		// Voter switches vote to nay, but does not kill the vote, just updates + inserts
 		let approve = false;
 
+		// Whitelist voter account from further DB operations.
+		let voter_key = frame_system::Account::<T>::hashed_key_for(&voter);
+		frame_benchmarking::benchmarking::add_to_whitelist(voter_key.into());
 	}: _(SystemOrigin::Signed(voter), last_hash.clone(), index, approve)
 	verify {
 		// All proposals exist and the last proposal has just been updated.
@@ -288,11 +291,11 @@ benchmarks_instance! {
 
 	close_early_disapproved {
 		// We choose 4 as a minimum so we always trigger a vote in the voting loop (`for j in ...`)
-		let m in 4 .. MAX_MEMBERS;
+		let m in 4 .. T::MaxMembers::get();
 		let p in 1 .. T::MaxProposals::get();
-		let b in 1 .. MAX_BYTES;
 
-		let bytes_in_storage = b + size_of::<u32>() as u32;
+		let bytes = 100;
+		let bytes_in_storage = bytes + size_of::<u32>() as u32;
 
 		// Construct `members`.
 		let mut members = vec![];
@@ -304,7 +307,7 @@ benchmarks_instance! {
 		}
 		let voter: T::AccountId = account("voter", 0, SEED);
 		members.push(voter.clone());
-		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members.clone(), None, MAX_MEMBERS)?;
+		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members.clone(), None, T::MaxMembers::get())?;
 
 		// Threshold is total members so that one nay will disapprove the vote
 		let threshold = m;
@@ -313,7 +316,7 @@ benchmarks_instance! {
 		let mut last_hash = T::Hash::default();
 		for i in 0 .. p {
 			// Proposals should be different so that different proposal hashes are generated
-			let proposal: T::Proposal = SystemCall::<T>::remark(vec![i as u8; b as usize]).into();
+			let proposal: T::Proposal = SystemCall::<T>::remark(vec![i as u8; bytes as usize]).into();
 			Collective::<T, _>::propose(
 				SystemOrigin::Signed(proposer.clone()).into(),
 				threshold,
@@ -356,6 +359,9 @@ benchmarks_instance! {
 			approve,
 		)?;
 
+		// Whitelist voter account from further DB operations.
+		let voter_key = frame_system::Account::<T>::hashed_key_for(&voter);
+		frame_benchmarking::benchmarking::add_to_whitelist(voter_key.into());
 	}: close(SystemOrigin::Signed(voter), last_hash.clone(), index, Weight::max_value(), bytes_in_storage)
 	verify {
 		// The last proposal is removed.
@@ -364,10 +370,10 @@ benchmarks_instance! {
 	}
 
 	close_early_approved {
+		let b in 1 .. MAX_BYTES;
 		// We choose 4 as a minimum so we always trigger a vote in the voting loop (`for j in ...`)
-		let m in 4 .. MAX_MEMBERS;
+		let m in 4 .. T::MaxMembers::get();
 		let p in 1 .. T::MaxProposals::get();
-		let b in 1 .. MAX_BYTES;
 
 		let bytes_in_storage = b + size_of::<u32>() as u32;
 
@@ -379,7 +385,7 @@ benchmarks_instance! {
 		}
 		let caller: T::AccountId = whitelisted_caller();
 		members.push(caller.clone());
-		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members.clone(), None, MAX_MEMBERS)?;
+		Collective::<T, _>::set_members(SystemOrigin::Root.into(), members.clone(), None, T::MaxMembers::get())?;
 
 		// Threshold is 2 so any two ayes will approve the vote
 		let threshold = 2;
@@ -446,11 +452,11 @@ benchmarks_instance! {
 
 	close_disapproved {
 		// We choose 4 as a minimum so we always trigger a vote in the voting loop (`for j in ...`)
-		let m in 4 .. MAX_MEMBERS;
+		let m in 4 .. T::MaxMembers::get();
 		let p in 1 .. T::MaxProposals::get();
-		let b in 1 .. MAX_BYTES;
 
-		let bytes_in_storage = b + size_of::<u32>() as u32;
+		let bytes = 100;
+		let bytes_in_storage = bytes + size_of::<u32>() as u32;
 
 		// Construct `members`.
 		let mut members = vec![];
@@ -464,7 +470,7 @@ benchmarks_instance! {
 			SystemOrigin::Root.into(),
 			members.clone(),
 			Some(caller.clone()),
-			MAX_MEMBERS,
+			T::MaxMembers::get(),
 		)?;
 
 		// Threshold is one less than total members so that two nays will disapprove the vote
@@ -474,7 +480,7 @@ benchmarks_instance! {
 		let mut last_hash = T::Hash::default();
 		for i in 0 .. p {
 			// Proposals should be different so that different proposal hashes are generated
-			let proposal: T::Proposal = SystemCall::<T>::remark(vec![i as u8; b as usize]).into();
+			let proposal: T::Proposal = SystemCall::<T>::remark(vec![i as u8; bytes as usize]).into();
 			Collective::<T, _>::propose(
 				SystemOrigin::Signed(caller.clone()).into(),
 				threshold,
@@ -517,10 +523,10 @@ benchmarks_instance! {
 	}
 
 	close_approved {
+		let b in 1 .. MAX_BYTES;
 		// We choose 4 as a minimum so we always trigger a vote in the voting loop (`for j in ...`)
-		let m in 4 .. MAX_MEMBERS;
+		let m in 4 .. T::MaxMembers::get();
 		let p in 1 .. T::MaxProposals::get();
-		let b in 1 .. MAX_BYTES;
 
 		let bytes_in_storage = b + size_of::<u32>() as u32;
 
@@ -536,7 +542,7 @@ benchmarks_instance! {
 			SystemOrigin::Root.into(),
 			members.clone(),
 			Some(caller.clone()),
-			MAX_MEMBERS,
+			T::MaxMembers::get(),
 		)?;
 
 		// Threshold is two, so any two ayes will pass the vote
@@ -579,6 +585,54 @@ benchmarks_instance! {
 		assert_eq!(Collective::<T, _>::proposals().len(), (p - 1) as usize);
 		assert_last_event::<T, I>(RawEvent::Executed(last_hash, Err(DispatchError::BadOrigin)).into());
 	}
+
+	disapprove_proposal {
+		let p in 1 .. T::MaxProposals::get();
+
+		let m = 3;
+		let b = MAX_BYTES;
+		let bytes_in_storage = b + size_of::<u32>() as u32;
+
+		// Construct `members`.
+		let mut members = vec![];
+		for i in 0 .. m - 1 {
+			let member = account("member", i, SEED);
+			members.push(member);
+		}
+		let caller: T::AccountId = account("caller", 0, SEED);
+		members.push(caller.clone());
+		Collective::<T, _>::set_members(
+			SystemOrigin::Root.into(),
+			members.clone(),
+			Some(caller.clone()),
+			T::MaxMembers::get(),
+		)?;
+
+		// Threshold is one less than total members so that two nays will disapprove the vote
+		let threshold = m - 1;
+
+		// Add proposals
+		let mut last_hash = T::Hash::default();
+		for i in 0 .. p {
+			// Proposals should be different so that different proposal hashes are generated
+			let proposal: T::Proposal = SystemCall::<T>::remark(vec![i as u8; b as usize]).into();
+			Collective::<T, _>::propose(
+				SystemOrigin::Signed(caller.clone()).into(),
+				threshold,
+				Box::new(proposal.clone()),
+				bytes_in_storage,
+			)?;
+			last_hash = T::Hashing::hash_of(&proposal);
+		}
+
+		System::<T>::set_block_number(T::BlockNumber::max_value());
+		assert_eq!(Collective::<T, _>::proposals().len(), p as usize);
+
+	}: _(SystemOrigin::Root, last_hash)
+	verify {
+		assert_eq!(Collective::<T, _>::proposals().len(), (p - 1) as usize);
+		assert_last_event::<T, I>(RawEvent::Disapproved(last_hash).into());
+	}
 }
 
 #[cfg(test)]
@@ -649,4 +703,11 @@ mod tests {
 			assert_ok!(test_benchmark_close_approved::<Test>());
 		});
 	}
+
+	#[test]
+	fn disapprove_proposal() {
+		new_test_ext().execute_with(|| {
+			assert_ok!(test_benchmark_disapprove_proposal::<Test>());
+		});
+	}
 }
diff --git a/frame/collective/src/default_weight.rs b/frame/collective/src/default_weight.rs
new file mode 100644
index 0000000000000..bb6fe0ea25312
--- /dev/null
+++ b/frame/collective/src/default_weight.rs
@@ -0,0 +1,97 @@
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Default weights for the Collective Pallet
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn set_members(m: u32, n: u32, p: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((21040000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((173000 as Weight).saturating_mul(n as Weight))
+			.saturating_add((31595000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(p as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight)))
+	}
+	fn execute(b: u32, m: u32, ) -> Weight {
+		(43359000 as Weight)
+			.saturating_add((4000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((123000 as Weight).saturating_mul(m as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+	}
+	fn propose_execute(b: u32, m: u32, ) -> Weight {
+		(54134000 as Weight)
+			.saturating_add((4000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((239000 as Weight).saturating_mul(m as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+	}
+	fn propose_proposed(b: u32, m: u32, p: u32, ) -> Weight {
+		(90650000 as Weight)
+			.saturating_add((5000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((152000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((970000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn vote(m: u32, ) -> Weight {
+		(74460000 as Weight)
+			.saturating_add((290000 as Weight).saturating_mul(m as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn close_early_disapproved(m: u32, p: u32, ) -> Weight {
+		(86360000 as Weight)
+			.saturating_add((232000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((954000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_early_approved(b: u32, m: u32, p: u32, ) -> Weight {
+		(123653000 as Weight)
+			.saturating_add((1000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((287000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((920000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_disapproved(m: u32, p: u32, ) -> Weight {
+		(95395000 as Weight)
+			.saturating_add((236000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((965000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_approved(b: u32, m: u32, p: u32, ) -> Weight {
+		(135284000 as Weight)
+			.saturating_add((4000 as Weight).saturating_mul(b as Weight))
+			.saturating_add((218000 as Weight).saturating_mul(m as Weight))
+			.saturating_add((951000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn disapprove_proposal(p: u32, ) -> Weight {
+		(50500000 as Weight)
+			.saturating_add((966000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+}
diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs
index 1edd8c75b90b0..949484a5957b4 100644
--- a/frame/collective/src/lib.rs
+++ b/frame/collective/src/lib.rs
@@ -20,7 +20,7 @@
 //!
 //! The membership can be provided in one of two ways: either directly, using the Root-dispatchable
 //! function `set_members`, or indirectly, through implementing the `ChangeMembers`.
-//! The pallet assumes that the amount of members stays at or below `MAX_MEMBERS` for its weight
+//! The pallet assumes that the amount of members stays at or below `MaxMembers` for its weight
 //! calculations, but enforces this neither in `set_members` nor in `change_members_sorted`.
 //!
 //! A "prime" member may be set allowing their vote to act as the default vote in case of any
@@ -60,6 +60,8 @@ use frame_system::{self as system, ensure_signed, ensure_root};
 #[cfg(feature = "runtime-benchmarks")]
 mod benchmarking;
 
+mod default_weight;
+
 /// Simple index type for proposal counting.
 pub type ProposalIndex = u32;
 
@@ -69,35 +71,17 @@ pub type ProposalIndex = u32;
 /// vote exactly once, therefore also the number of votes for any given motion.
 pub type MemberCount = u32;
 
-/// The maximum number of members supported by the pallet. Used for weight estimation.
-///
-/// NOTE:
-/// + Benchmarks will need to be re-run and weights adjusted if this changes.
-/// + This pallet assumes that dependents keep to the limit without enforcing it.
-pub const MAX_MEMBERS: MemberCount = 100;
-
 pub trait WeightInfo {
 	fn set_members(m: u32, n: u32, p: u32, ) -> Weight;
-	fn execute(m: u32, b: u32, ) -> Weight;
-	fn propose_execute(m: u32, b: u32, ) -> Weight;
-	fn propose_proposed(m: u32, p: u32, b: u32, ) -> Weight;
+	fn execute(b: u32, m: u32, ) -> Weight;
+	fn propose_execute(b: u32, m: u32, ) -> Weight;
+	fn propose_proposed(b: u32, m: u32, p: u32, ) -> Weight;
 	fn vote(m: u32, ) -> Weight;
-	fn close_early_disapproved(m: u32, p: u32, b: u32, ) -> Weight;
-	fn close_early_approved(m: u32, p: u32, b: u32, ) -> Weight;
-	fn close_disapproved(m: u32, p: u32, b: u32, ) -> Weight;
-	fn close_approved(m: u32, p: u32, b: u32, ) -> Weight;
-}
-
-impl WeightInfo for () {
-	fn set_members(_m: u32, _n: u32, _p: u32, ) -> Weight { 1_000_000_000 }
-	fn execute(_m: u32, _b: u32, ) -> Weight { 1_000_000_000 }
-	fn propose_execute(_m: u32, _b: u32, ) -> Weight { 1_000_000_000 }
-	fn propose_proposed(_m: u32, _p: u32, _b: u32, ) -> Weight { 1_000_000_000 }
-	fn vote(_m: u32, ) -> Weight { 1_000_000_000 }
-	fn close_early_disapproved(_m: u32, _p: u32, _b: u32, ) -> Weight { 1_000_000_000 }
-	fn close_early_approved(_m: u32, _p: u32, _b: u32, ) -> Weight { 1_000_000_000 }
-	fn close_disapproved(_m: u32, _p: u32, _b: u32, ) -> Weight { 1_000_000_000 }
-	fn close_approved(_m: u32, _p: u32, _b: u32, ) -> Weight { 1_000_000_000 }
+	fn close_early_disapproved(m: u32, p: u32, ) -> Weight;
+	fn close_early_approved(b: u32, m: u32, p: u32, ) -> Weight;
+	fn close_disapproved(m: u32, p: u32, ) -> Weight;
+	fn close_approved(b: u32, m: u32, p: u32, ) -> Weight;
+	fn disapprove_proposal(p: u32, ) -> Weight;
 }
 
 pub trait Trait<I: Instance=DefaultInstance>: frame_system::Trait {
@@ -117,7 +101,14 @@ pub trait Trait<I: Instance=DefaultInstance>: frame_system::Trait {
 	type MotionDuration: Get<Self::BlockNumber>;
 
 	/// Maximum number of proposals allowed to be active in parallel.
-	type MaxProposals: Get<u32>;
+	type MaxProposals: Get<ProposalIndex>;
+
+	/// The maximum number of members supported by the pallet. Used for weight estimation.
+	///
+	/// NOTE:
+	/// + Benchmarks will need to be re-run and weights adjusted if this changes.
+	/// + This pallet assumes that dependents keep to the limit without enforcing it.
+	type MaxMembers: Get<MemberCount>;
 
 	/// Weight information for extrinsics in this pallet.
 	type WeightInfo: WeightInfo;
@@ -233,131 +224,6 @@ decl_error! {
 	}
 }
 
-/// Functions for calcuating the weight of dispatchables.
-mod weight_for {
-	use frame_support::{traits::Get, weights::Weight};
-	use super::{Trait, Instance};
-
-	/// Calculate the weight for `set_members`.
-	///
-	/// Based on benchmark:
-	/// 0 + M * 20.47 + N * 0.109 + P * 26.29 µs (min squares analysis)
-	///
-	/// Note: The complexity of `set_members` is quadratic (`O(MP + N)`), so the linear approximation
-	/// of the benchmark is not always permissible. It is here, though, because the linear approximation
-	/// covered the range of possible values and we estimate weight via the worst case (max paramter
-	/// values) before execution so we can be sure that we are only overestimating.
-	pub(crate) fn set_members<T: Trait<I>, I: Instance>(
-		old_count: Weight,
-		new_count: Weight,
-		proposals: Weight,
-	) -> Weight {
-		let db = T::DbWeight::get();
-		db.reads_writes(1, 1) // mutate `Members`
-			.saturating_add(db.writes(1)) // set `Prime`
-			.saturating_add(db.reads(1)) // read `Proposals`
-			.saturating_add(db.reads_writes(proposals, proposals)) // update votes (`Voting`)
-			.saturating_add(old_count.saturating_mul(21_000_000)) // M
-			.saturating_add(new_count.saturating_mul(110_000)) // N
-			.saturating_add(proposals.saturating_mul(27_000_000)) // P
-	}
-
-	/// Calculate the weight for `execute`.
-	///
-	/// Based on benchmark:
-	/// 22.62 + M * 0.115 + B * 0.003 µs (min squares analysis)
-	pub(crate) fn execute<T: Trait<I>, I: Instance>(
-		members: Weight,
-		proposal: Weight,
-		length: Weight,
-	) -> Weight {
-		T::DbWeight::get().reads(1) // read members for `is_member`
-			.saturating_add(23_000_000) // constant
-			.saturating_add(length.saturating_mul(4_000)) // B
-			.saturating_add(members.saturating_mul(120_000)) // M
-			.saturating_add(proposal) // P
-	}
-
-	/// Calculate the weight for `propose` if the proposal is executed straight away (`threshold < 2`).
-	///
-	/// Based on benchmark:
-	/// 28.12 + M * 0.218 + B * 0.003 µs (min squares analysis)
-	pub(crate) fn propose_execute<T: Trait<I>, I: Instance>(
-		members: Weight,
-		proposal: Weight,
-		length: Weight,
-	) -> Weight {
-		T::DbWeight::get().reads(2) // `is_member` + `contains_key`
-			.saturating_add(29_000_000) // constant
-			.saturating_add(length.saturating_mul(3_000)) // B
-			.saturating_add(members.saturating_mul(220_000)) // M
-			.saturating_add(proposal) // P1
-	}
-
-	/// Calculate the weight for `propose` if the proposal is put up for a vote (`threshold >= 2`).
-	///
-	/// Based on benchmark:
-	/// 49.75 + M * 0.105 + P2 0.502 + B * 0.006 µs (min squares analysis)
-	pub(crate) fn propose_proposed<T: Trait<I>, I: Instance>(
-		members: Weight,
-		proposals: Weight,
-		length: Weight,
-	) -> Weight {
-		T::DbWeight::get().reads(2) // `is_member` + `contains_key`
-			.saturating_add(T::DbWeight::get().reads_writes(2, 4)) // `proposal` insertion
-			.saturating_add(50_000_000) // constant
-			.saturating_add(length.saturating_mul(6_000)) // B
-			.saturating_add(members.saturating_mul(110_000)) // M
-			.saturating_add(proposals.saturating_mul(510_000)) // P2
-	}
-
-	/// Calculate the weight for `vote`.
-	///
-	/// Based on benchmark:
-	/// 24.03 + M * 0.349 + P * 0.119 + B * 0.003 µs (min squares analysis)
-	pub(crate) fn vote<T: Trait<I>, I: Instance>(
-		members: Weight,
-	) -> Weight {
-		T::DbWeight::get().reads(1) // read `Members`
-			.saturating_add(T::DbWeight::get().reads_writes(1, 1)) // mutate `Voting`
-			.saturating_add(30_000_000) // constant
-			.saturating_add(members.saturating_mul(500_000)) // M
-	}
-
-	/// Calculate the weight for `close`.
-	///
-	/// Based on benchmarks:
-	/// - early disapproved: 37.21 + M * 0.239 + P2 * 0.466 + B * 0.002 µs (min squares analysis)
-	/// - early approved:    50.82 + M * 0.211 + P2 * 0.478 + B * 0.008 µs (min squares analysis)
-	/// - disapproved:       51.08 + M * 0.224 + P2 * 0.475 + B * 0.003 µs (min squares analysis)
-	/// - approved:          65.95 + M * 0.226 + P2 * 0.487 + B * 0.005 µs (min squares analysis)
-	pub(crate) fn close<T: Trait<I>, I: Instance>(
-		members: Weight,
-		proposal_weight: Weight,
-		proposals: Weight,
-		length: Weight,
-	) -> Weight {
-		let db = T::DbWeight::get();
-		close_without_finalize::<T, I>(members, length)
-			.saturating_add(db.reads(1)) // `Prime`
-			.saturating_add(db.writes(1)) // `Proposals`
-			.saturating_add(db.writes(1)) // `Voting`
-			.saturating_add(proposal_weight) // P1
-			.saturating_add(proposals.saturating_mul(490_000)) // P2
-	}
-
-	/// Calculate the weight for `close` without the call to `approve/disapprove_proposal`.
-	pub(crate) fn close_without_finalize<T: Trait<I>, I: Instance>(
-		members: Weight,
-		length: Weight,
-	) -> Weight {
-		T::DbWeight::get().reads(3) // `Members`, `Voting`, `ProposalOf`
-			.saturating_add(66_000_000) // constant
-			.saturating_add(length.saturating_mul(8_000)) // B
-			.saturating_add(members.saturating_mul(250_000)) // M
-	}
-}
-
 /// Return the weight of a dispatch call result as an `Option`.
 ///
 /// Will return the weight regardless of what the state of the result is.
@@ -385,7 +251,7 @@ decl_module! {
 		///
 		/// Requires root origin.
 		///
-		/// NOTE: Does not enforce the expected `MAX_MEMBERS` limit on the amount of members, but
+		/// NOTE: Does not enforce the expected `MaxMembers` limit on the amount of members, but
 		///       the weight estimations rely on it to estimate dispatchable weight.
 		///
 		/// # <weight>
@@ -401,10 +267,10 @@ decl_module! {
 		///   - 1 storage write (codec `O(1)`) for deleting the old `prime` and setting the new one
 		/// # </weight>
 		#[weight = (
-			weight_for::set_members::<T, I>(
-				(*old_count).into(), // M
-				new_members.len() as Weight, // N
-				T::MaxProposals::get().into(), // P
+			T::WeightInfo::set_members(
+				*old_count, // M
+				new_members.len() as u32, // N
+				T::MaxProposals::get() // P
 			),
 			DispatchClass::Operational
 		)]
@@ -414,10 +280,10 @@ decl_module! {
 			old_count: MemberCount,
 		) -> DispatchResultWithPostInfo {
 			ensure_root(origin)?;
-			if new_members.len() > MAX_MEMBERS as usize {
+			if new_members.len() > T::MaxMembers::get() as usize {
 				debug::error!(
 					"New members count exceeds maximum amount of members expected. (expected: {}, actual: {})",
-					MAX_MEMBERS,
+					T::MaxMembers::get(),
 					new_members.len()
 				);
 			}
@@ -435,10 +301,10 @@ decl_module! {
 			<Self as ChangeMembers<T::AccountId>>::set_members_sorted(&new_members, &old);
 			Prime::<T, I>::set(prime);
 
-			Ok(Some(weight_for::set_members::<T, I>(
-				old.len() as Weight, // M
-				new_members.len() as Weight, // N
-				T::MaxProposals::get().into(), // P
+			Ok(Some(T::WeightInfo::set_members(
+				old.len() as u32, // M
+				new_members.len() as u32, // N
+				T::MaxProposals::get(), // P
 			)).into())
 		}
 
@@ -453,11 +319,10 @@ decl_module! {
 		/// - 1 event
 		/// # </weight>
 		#[weight = (
-			weight_for::execute::<T, I>(
-				MAX_MEMBERS.into(),
-				proposal.get_dispatch_info().weight,
-				*length_bound as Weight,
-			),
+			T::WeightInfo::execute(
+				*length_bound, // B
+				T::MaxMembers::get(), // M
+			).saturating_add(proposal.get_dispatch_info().weight), // P
 			DispatchClass::Operational
 		)]
 		fn execute(origin,
@@ -476,11 +341,12 @@ decl_module! {
 				RawEvent::MemberExecuted(proposal_hash, result.map(|_| ()).map_err(|e| e.error))
 			);
 
-			Ok(get_result_weight(result).map(|w| weight_for::execute::<T, I>(
-				members.len() as Weight,
-				w,
-				proposal_len as Weight
-			)).into())
+			Ok(get_result_weight(result).map(|w| {
+				T::WeightInfo::execute(
+					proposal_len as u32,  // B
+					members.len() as u32, // M
+				).saturating_add(w) // P
+			}).into())
 		}
 
 		/// Add a new proposal to either be voted on or executed directly.
@@ -512,16 +378,15 @@ decl_module! {
 		/// # </weight>
 		#[weight = (
 			if *threshold < 2 {
-				weight_for::propose_execute::<T, I>(
-					MAX_MEMBERS.into(), // M
-					proposal.get_dispatch_info().weight, // P1
-					*length_bound as Weight, // B
-				)
+				T::WeightInfo::propose_execute(
+					*length_bound, // B
+					T::MaxMembers::get(), // M
+				).saturating_add(proposal.get_dispatch_info().weight) // P1
 			} else {
-				weight_for::propose_proposed::<T, I>(
-					MAX_MEMBERS.into(), // M
-					T::MaxProposals::get().into(), // P2
-					*length_bound as Weight, // B
+				T::WeightInfo::propose_proposed(
+					*length_bound, // B
+					T::MaxMembers::get(), // M
+					T::MaxProposals::get(), // P2
 				)
 			},
 			DispatchClass::Operational
@@ -547,11 +412,12 @@ decl_module! {
 					RawEvent::Executed(proposal_hash, result.map(|_| ()).map_err(|e| e.error))
 				);
 
-				Ok(get_result_weight(result).map(|w| weight_for::propose_execute::<T, I>(
-					members.len() as Weight, // M
-					w, // P1
-					proposal_len as Weight, // B
-				)).into())
+				Ok(get_result_weight(result).map(|w| {
+					T::WeightInfo::propose_execute(
+						proposal_len as u32, // B
+						members.len() as u32, // M
+					).saturating_add(w) // P1
+				}).into())
 			} else {
 				let active_proposals =
 					<Proposals<T, I>>::try_mutate(|proposals| -> Result<usize, DispatchError> {
@@ -571,10 +437,10 @@ decl_module! {
 
 				Self::deposit_event(RawEvent::Proposed(who, index, proposal_hash, threshold));
 
-				Ok(Some(weight_for::propose_proposed::<T, I>(
-					members.len() as Weight, // M
-					active_proposals as Weight, // P2
-					proposal_len as Weight, // B
+				Ok(Some(T::WeightInfo::propose_proposed(
+					proposal_len as u32, // B
+					members.len() as u32, // M
+					active_proposals as u32, // P2
 				)).into())
 			}
 		}
@@ -592,7 +458,7 @@ decl_module! {
 		/// - 1 event
 		/// # </weight>
 		#[weight = (
-			weight_for::vote::<T, I>(MAX_MEMBERS.into()),
+			T::WeightInfo::vote(T::MaxMembers::get()),
 			DispatchClass::Operational
 		)]
 		fn vote(origin,
@@ -636,7 +502,7 @@ decl_module! {
 
 			Voting::<T, I>::insert(&proposal, voting);
 
-			Ok(Some(weight_for::vote::<T, I>(members.len() as Weight)).into())
+			Ok(Some(T::WeightInfo::vote(members.len() as u32)).into())
 		}
 
 		/// Close a vote that is either approved, disapproved or whose voting period has ended.
@@ -667,12 +533,17 @@ decl_module! {
 		/// - up to 3 events
 		/// # </weight>
 		#[weight = (
-			weight_for::close::<T, I>(
-				MAX_MEMBERS.into(), // `M`
-				*proposal_weight_bound, // `P1`
-				T::MaxProposals::get().into(), // `P2`
-				*length_bound as Weight, // B
-			),
+			{
+				let b = *length_bound;
+				let m = T::MaxMembers::get();
+				let p1 = *proposal_weight_bound;
+				let p2 = T::MaxProposals::get();
+				T::WeightInfo::close_early_approved(b, m, p2)
+					.max(T::WeightInfo::close_early_disapproved(m, p2))
+					.max(T::WeightInfo::close_approved(b, m, p2))
+					.max(T::WeightInfo::close_disapproved(m, p2))
+					.saturating_add(p1)
+			},
 			DispatchClass::Operational
 		)]
 		fn close(origin,
@@ -699,17 +570,17 @@ decl_module! {
 					proposal_weight_bound
 				)?;
 				Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes));
-				let approve_weight = Self::do_approve_proposal(seats, voting, proposal_hash, proposal);
+				let (proposal_weight, proposal_count) =
+					Self::do_approve_proposal(seats, voting, proposal_hash, proposal);
 				return Ok(Some(
-					weight_for::close_without_finalize::<T, I>(seats.into(), len as Weight)
-						.saturating_add(approve_weight)
+					T::WeightInfo::close_early_approved(len as u32, seats, proposal_count)
+						.saturating_add(proposal_weight)
 				).into());
 			} else if disapproved {
 				Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes));
-				let disapprove_weight = Self::do_disapprove_proposal(proposal_hash);
+				let proposal_count = Self::do_disapprove_proposal(proposal_hash);
 				return Ok(Some(
-					weight_for::close_without_finalize::<T, I>(seats.into(), 0)
-						.saturating_add(disapprove_weight)
+					T::WeightInfo::close_early_disapproved(seats, proposal_count)
 				).into());
 			}
 
@@ -733,19 +604,17 @@ decl_module! {
 					proposal_weight_bound
 				)?;
 				Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes));
-				let approve_weight = Self::do_approve_proposal(seats, voting, proposal_hash, proposal);
+				let (proposal_weight, proposal_count) =
+					Self::do_approve_proposal(seats, voting, proposal_hash, proposal);
 				return Ok(Some(
-					weight_for::close_without_finalize::<T, I>(seats.into(), len as Weight)
-						.saturating_add(T::DbWeight::get().reads(1)) // read `Prime`
-						.saturating_add(approve_weight)
+					T::WeightInfo::close_approved(len as u32, seats, proposal_count)
+						.saturating_add(proposal_weight)
 				).into());
 			} else {
 				Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes));
-				let disapprove_weight = Self::do_disapprove_proposal(proposal_hash);
+				let proposal_count = Self::do_disapprove_proposal(proposal_hash);
 				return Ok(Some(
-					weight_for::close_without_finalize::<T, I>(seats.into(), 0)
-						.saturating_add(T::DbWeight::get().reads(1)) // read `Prime`
-						.saturating_add(disapprove_weight)
+					T::WeightInfo::close_disapproved(seats, proposal_count)
 				).into());
 			}
 		}
@@ -759,18 +628,15 @@ decl_module! {
 		///
 		/// # <weight>
 		/// Complexity: O(P) where P is the number of max proposals
-		/// Base Weight: .49 * P
 		/// DB Weight:
 		/// * Reads: Proposals
 		/// * Writes: Voting, Proposals, ProposalOf
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 3) // `Voting`, `Proposals`, `ProposalOf`
-			.saturating_add(490_000 * Weight::from(T::MaxProposals::get())) // P2
-		]
+		#[weight = T::WeightInfo::disapprove_proposal(T::MaxProposals::get())]
 		fn disapprove_proposal(origin, proposal_hash: T::Hash) -> DispatchResultWithPostInfo {
 			ensure_root(origin)?;
-			let actual_weight = Self::do_disapprove_proposal(proposal_hash);
-			Ok(Some(actual_weight).into())
+			let proposal_count = Self::do_disapprove_proposal(proposal_hash);
+			Ok(Some(T::WeightInfo::disapprove_proposal(proposal_count)).into())
 		}
 	}
 }
@@ -822,8 +688,7 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 		voting: Votes<T::AccountId, T::BlockNumber>,
 		proposal_hash: T::Hash,
 		proposal: <T as Trait<I>>::Proposal,
-	) -> Weight {
-		let mut weight: Weight = 0;
+	) -> (Weight, u32) {
 		Self::deposit_event(RawEvent::Approved(proposal_hash));
 
 		let dispatch_weight = proposal.get_dispatch_info().weight;
@@ -832,23 +697,21 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 		Self::deposit_event(
 			RawEvent::Executed(proposal_hash, result.map(|_| ()).map_err(|e| e.error))
 		);
-		weight = weight.saturating_add(
-			// default to the dispatch info weight for safety
-			get_result_weight(result).unwrap_or(dispatch_weight) // P1
-		);
+		// default to the dispatch info weight for safety
+		let proposal_weight = get_result_weight(result).unwrap_or(dispatch_weight); // P1
 
-		let remove_proposal_weight = Self::remove_proposal(proposal_hash);
-		weight.saturating_add(remove_proposal_weight)
+		let proposal_count = Self::remove_proposal(proposal_hash);
+		(proposal_weight, proposal_count)
 	}
 
-	fn do_disapprove_proposal(proposal_hash: T::Hash) -> Weight {
+	fn do_disapprove_proposal(proposal_hash: T::Hash) -> u32 {
 		// disapproved
 		Self::deposit_event(RawEvent::Disapproved(proposal_hash));
 		Self::remove_proposal(proposal_hash)
 	}
 
 	// Removes a proposal from the pallet, cleaning up votes and the vector of proposals.
-	fn remove_proposal(proposal_hash: T::Hash) -> Weight {
+	fn remove_proposal(proposal_hash: T::Hash) -> u32 {
 		// remove proposal and vote
 		ProposalOf::<T, I>::remove(&proposal_hash);
 		Voting::<T, I>::remove(&proposal_hash);
@@ -856,15 +719,14 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 			proposals.retain(|h| h != &proposal_hash);
 			proposals.len() + 1 // calculate weight based on original length
 		});
-		T::DbWeight::get().reads_writes(1, 3) // `Voting`, `Proposals`, `ProposalOf`
-			.saturating_add(490_000 * num_proposals as Weight) // P2
+		num_proposals as u32
 	}
 }
 
 impl<T: Trait<I>, I: Instance> ChangeMembers<T::AccountId> for Module<T, I> {
 	/// Update the members of the collective. Votes are updated and the prime is reset.
 	///
-	/// NOTE: Does not enforce the expected `MAX_MEMBERS` limit on the amount of members, but
+	/// NOTE: Does not enforce the expected `MaxMembers` limit on the amount of members, but
 	///       the weight estimations rely on it to estimate dispatchable weight.
 	///
 	/// # <weight>
@@ -884,10 +746,10 @@ impl<T: Trait<I>, I: Instance> ChangeMembers<T::AccountId> for Module<T, I> {
 		outgoing: &[T::AccountId],
 		new: &[T::AccountId],
 	) {
-		if new.len() > MAX_MEMBERS as usize {
+		if new.len() > T::MaxMembers::get() as usize {
 			debug::error!(
 				"New members count exceeds maximum amount of members expected. (expected: {}, actual: {})",
-				MAX_MEMBERS,
+				T::MaxMembers::get(),
 				new.len()
 			);
 		}
@@ -1047,6 +909,7 @@ mod tests {
 		pub const AvailableBlockRatio: Perbill = Perbill::one();
 		pub const MotionDuration: u64 = 3;
 		pub const MaxProposals: u32 = 100;
+		pub const MaxMembers: u32 = 100;
 	}
 	impl frame_system::Trait for Test {
 		type BaseCallFilter = ();
@@ -1081,6 +944,7 @@ mod tests {
 		type Event = Event;
 		type MotionDuration = MotionDuration;
 		type MaxProposals = MaxProposals;
+		type MaxMembers = MaxMembers;
 		type WeightInfo = ();
 	}
 	impl Trait for Test {
@@ -1089,6 +953,7 @@ mod tests {
 		type Event = Event;
 		type MotionDuration = MotionDuration;
 		type MaxProposals = MaxProposals;
+		type MaxMembers = MaxMembers;
 		type WeightInfo = ();
 	}
 
@@ -1164,7 +1029,7 @@ mod tests {
 	#[test]
 	fn proposal_weight_limit_works_on_approve() {
 		new_test_ext().execute_with(|| {
-			let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MAX_MEMBERS));
+			let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MaxMembers::get()));
 			let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32);
 			let proposal_weight = proposal.get_dispatch_info().weight;
 			let hash = BlakeTwo256::hash_of(&proposal);
@@ -1184,7 +1049,7 @@ mod tests {
 	#[test]
 	fn proposal_weight_limit_ignored_on_disapprove() {
 		new_test_ext().execute_with(|| {
-			let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MAX_MEMBERS));
+			let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MaxMembers::get()));
 			let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32);
 			let proposal_weight = proposal.get_dispatch_info().weight;
 			let hash = BlakeTwo256::hash_of(&proposal);
@@ -1205,7 +1070,7 @@ mod tests {
 			let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32);
 			let proposal_weight = proposal.get_dispatch_info().weight;
 			let hash = BlakeTwo256::hash_of(&proposal);
-			assert_ok!(Collective::set_members(Origin::root(), vec![1, 2, 3], Some(3), MAX_MEMBERS));
+			assert_ok!(Collective::set_members(Origin::root(), vec![1, 2, 3], Some(3), MaxMembers::get()));
 
 			assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len));
 			assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true));
@@ -1230,7 +1095,7 @@ mod tests {
 			let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32);
 			let proposal_weight = proposal.get_dispatch_info().weight;
 			let hash = BlakeTwo256::hash_of(&proposal);
-			assert_ok!(Collective::set_members(Origin::root(), vec![1, 2, 3], Some(1), MAX_MEMBERS));
+			assert_ok!(Collective::set_members(Origin::root(), vec![1, 2, 3], Some(1), MaxMembers::get()));
 
 			assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len));
 			assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true));
@@ -1298,7 +1163,7 @@ mod tests {
 				Collective::voting(&hash),
 				Some(Votes { index: 0, threshold: 3, ayes: vec![1, 2], nays: vec![], end })
 			);
-			assert_ok!(Collective::set_members(Origin::root(), vec![2, 3, 4], None, MAX_MEMBERS));
+			assert_ok!(Collective::set_members(Origin::root(), vec![2, 3, 4], None, MaxMembers::get()));
 			assert_eq!(
 				Collective::voting(&hash),
 				Some(Votes { index: 0, threshold: 3, ayes: vec![2], nays: vec![], end })
@@ -1313,7 +1178,7 @@ mod tests {
 				Collective::voting(&hash),
 				Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![3], end })
 			);
-			assert_ok!(Collective::set_members(Origin::root(), vec![2, 4], None, MAX_MEMBERS));
+			assert_ok!(Collective::set_members(Origin::root(), vec![2, 4], None, MaxMembers::get()));
 			assert_eq!(
 				Collective::voting(&hash),
 				Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![], end })
@@ -1371,7 +1236,7 @@ mod tests {
 	#[test]
 	fn correct_validate_and_get_proposal() {
 		new_test_ext().execute_with(|| {
-			let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MAX_MEMBERS));
+			let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MaxMembers::get()));
 			let length = proposal.encode().len() as u32;
 			assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), length));
 

From 8f2865566403f574bdff604410ad1b4c08b971ea Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Fri, 28 Aug 2020 17:34:25 +0200
Subject: [PATCH 025/122] client/*: Treat protocol name as str and not [u8]
 (#6967)

* client/*: Treat protocol name as str and not [u8]

Notification protocol names are in practice always valid utf8 strings.
Instead of treating them as such in the type system, thus far they were
casted to a [u8] at creation time.

With this commit protocol names are instead treated as valid utf8
strings throughout the codebase and passed as `Cow<'static, str>`
instead of `Cow<'static, [u8]>`. Among other things this eliminates the
need for string casting when logging.

* client/network: Don't allocate when protocol name is borrowed
---
 .../finality-grandpa/src/communication/mod.rs |  2 +-
 .../src/communication/tests.rs                |  2 +-
 client/network-gossip/src/bridge.rs           | 10 +++---
 client/network-gossip/src/lib.rs              |  4 +--
 client/network-gossip/src/state_machine.rs    |  2 +-
 client/network/src/behaviour.rs               |  2 +-
 client/network/src/config.rs                  |  2 +-
 client/network/src/gossip/tests.rs            |  4 +--
 client/network/src/protocol.rs                | 32 +++++++++---------
 .../src/protocol/generic_proto/behaviour.rs   | 18 +++++-----
 .../protocol/generic_proto/handler/group.rs   | 20 +++++------
 .../generic_proto/handler/notif_in.rs         |  4 +--
 .../generic_proto/handler/notif_out.rs        |  8 ++---
 .../generic_proto/upgrade/notifications.rs    | 33 ++++++++++++-------
 client/network/src/service.rs                 | 12 +++----
 client/network/src/service/tests.rs           |  8 ++---
 16 files changed, 87 insertions(+), 76 deletions(-)

diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs
index a8bfb84416b8f..9509922cf2d3a 100644
--- a/client/finality-grandpa/src/communication/mod.rs
+++ b/client/finality-grandpa/src/communication/mod.rs
@@ -69,7 +69,7 @@ mod periodic;
 pub(crate) mod tests;
 
 pub use sp_finality_grandpa::GRANDPA_ENGINE_ID;
-pub const GRANDPA_PROTOCOL_NAME: &[u8] = b"/paritytech/grandpa/1";
+pub const GRANDPA_PROTOCOL_NAME: &'static str = "/paritytech/grandpa/1";
 
 // cost scalars for reporting peers.
 mod cost {
diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs
index 273804f7a4508..6a1513769aa26 100644
--- a/client/finality-grandpa/src/communication/tests.rs
+++ b/client/finality-grandpa/src/communication/tests.rs
@@ -61,7 +61,7 @@ impl sc_network_gossip::Network<Block> for TestNetwork {
 		let _ = self.sender.unbounded_send(Event::WriteNotification(who, message));
 	}
 
-	fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, [u8]>) {}
+	fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, str>) {}
 
 	fn announce(&self, block: Hash, _associated_data: Vec<u8>) {
 		let _ = self.sender.unbounded_send(Event::Announce(block));
diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs
index df2a5c8e7e94a..70c2942597aa5 100644
--- a/client/network-gossip/src/bridge.rs
+++ b/client/network-gossip/src/bridge.rs
@@ -69,7 +69,7 @@ impl<B: BlockT> GossipEngine<B> {
 	pub fn new<N: Network<B> + Send + Clone + 'static>(
 		network: N,
 		engine_id: ConsensusEngineId,
-		protocol_name: impl Into<Cow<'static, [u8]>>,
+		protocol_name: impl Into<Cow<'static, str>>,
 		validator: Arc<dyn Validator<B>>,
 	) -> Self where B: 'static {
 		// We grab the event stream before registering the notifications protocol, otherwise we
@@ -333,7 +333,7 @@ mod tests {
 			unimplemented!();
 		}
 
-		fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, [u8]>) {}
+		fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, str>) {}
 
 		fn announce(&self, _: B::Hash, _: Vec<u8>) {
 			unimplemented!();
@@ -362,7 +362,7 @@ mod tests {
 		let mut gossip_engine = GossipEngine::<Block>::new(
 			network.clone(),
 			[1, 2, 3, 4],
-			"my_protocol".as_bytes(),
+			"my_protocol",
 			Arc::new(AllowAll{}),
 		);
 
@@ -390,7 +390,7 @@ mod tests {
 		let mut gossip_engine = GossipEngine::<Block>::new(
 			network.clone(),
 			engine_id.clone(),
-			"my_protocol".as_bytes(),
+			"my_protocol",
 			Arc::new(AllowAll{}),
 		);
 
@@ -525,7 +525,7 @@ mod tests {
 			let mut gossip_engine = GossipEngine::<Block>::new(
 				network.clone(),
 				engine_id.clone(),
-				"my_protocol".as_bytes(),
+				"my_protocol",
 				Arc::new(TestValidator{}),
 			);
 
diff --git a/client/network-gossip/src/lib.rs b/client/network-gossip/src/lib.rs
index 42aeca86cb275..1d566ed3cbba2 100644
--- a/client/network-gossip/src/lib.rs
+++ b/client/network-gossip/src/lib.rs
@@ -87,7 +87,7 @@ pub trait Network<B: BlockT> {
 	fn register_notifications_protocol(
 		&self,
 		engine_id: ConsensusEngineId,
-		protocol_name: Cow<'static, [u8]>,
+		protocol_name: Cow<'static, str>,
 	);
 
 	/// Notify everyone we're connected to that we have the given block.
@@ -117,7 +117,7 @@ impl<B: BlockT, H: ExHashT> Network<B> for Arc<NetworkService<B, H>> {
 	fn register_notifications_protocol(
 		&self,
 		engine_id: ConsensusEngineId,
-		protocol_name: Cow<'static, [u8]>,
+		protocol_name: Cow<'static, str>,
 	) {
 		NetworkService::register_notifications_protocol(self, engine_id, protocol_name)
 	}
diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs
index 80a0f9e70bcbf..60c669ecb6680 100644
--- a/client/network-gossip/src/state_machine.rs
+++ b/client/network-gossip/src/state_machine.rs
@@ -489,7 +489,7 @@ mod tests {
 			unimplemented!();
 		}
 
-		fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, [u8]>) {}
+		fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, str>) {}
 
 		fn announce(&self, _: B::Hash, _: Vec<u8>) {
 			unimplemented!();
diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs
index 4a47a26f55c24..2c399cfdf7707 100644
--- a/client/network/src/behaviour.rs
+++ b/client/network/src/behaviour.rs
@@ -255,7 +255,7 @@ impl<B: BlockT, H: ExHashT> Behaviour<B, H> {
 	pub fn register_notifications_protocol(
 		&mut self,
 		engine_id: ConsensusEngineId,
-		protocol_name: impl Into<Cow<'static, [u8]>>,
+		protocol_name: impl Into<Cow<'static, str>>,
 	) {
 		// This is the message that we will send to the remote as part of the initial handshake.
 		// At the moment, we force this to be an encoded `Roles`.
diff --git a/client/network/src/config.rs b/client/network/src/config.rs
index 5185befacf5ae..cf1f8393f380d 100644
--- a/client/network/src/config.rs
+++ b/client/network/src/config.rs
@@ -415,7 +415,7 @@ pub struct NetworkConfiguration {
 	pub node_key: NodeKeyConfig,
 	/// List of notifications protocols that the node supports. Must also include a
 	/// `ConsensusEngineId` for backwards-compatibility.
-	pub notifications_protocols: Vec<(ConsensusEngineId, Cow<'static, [u8]>)>,
+	pub notifications_protocols: Vec<(ConsensusEngineId, Cow<'static, str>)>,
 	/// List of request-response protocols that the node supports.
 	pub request_response_protocols: Vec<RequestResponseConfig>,
 	/// Maximum allowed number of incoming connections.
diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs
index 6c3e26da13c64..9ba44f564e132 100644
--- a/client/network/src/gossip/tests.rs
+++ b/client/network/src/gossip/tests.rs
@@ -130,14 +130,14 @@ fn build_nodes_one_proto()
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 
 	let (node1, events_stream1) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))],
+		notifications_protocols: vec![(ENGINE_ID, From::from("/foo"))],
 		listen_addresses: vec![listen_addr.clone()],
 		transport: config::TransportConfig::MemoryOnly,
 		.. config::NetworkConfiguration::new_local()
 	});
 
 	let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))],
+		notifications_protocols: vec![(ENGINE_ID, From::from("/foo"))],
 		listen_addresses: vec![],
 		reserved_nodes: vec![config::MultiaddrWithPeerId {
 			multiaddr: listen_addr,
diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index a97aeec1ec4b5..ab04aa2cbb0f7 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -245,13 +245,13 @@ pub struct Protocol<B: BlockT, H: ExHashT> {
 	/// Handles opening the unique substream and sending and receiving raw messages.
 	behaviour: GenericProto,
 	/// For each legacy gossiping engine ID, the corresponding new protocol name.
-	protocol_name_by_engine: HashMap<ConsensusEngineId, Cow<'static, [u8]>>,
+	protocol_name_by_engine: HashMap<ConsensusEngineId, Cow<'static, str>>,
 	/// For each protocol name, the legacy equivalent.
-	legacy_equiv_by_name: HashMap<Cow<'static, [u8]>, Fallback>,
+	legacy_equiv_by_name: HashMap<Cow<'static, str>, Fallback>,
 	/// Name of the protocol used for transactions.
-	transactions_protocol: Cow<'static, [u8]>,
+	transactions_protocol: Cow<'static, str>,
 	/// Name of the protocol used for block announces.
-	block_announces_protocol: Cow<'static, [u8]>,
+	block_announces_protocol: Cow<'static, str>,
 	/// Prometheus metrics.
 	metrics: Option<Metrics>,
 	/// The `PeerId`'s of all boot nodes.
@@ -421,19 +421,21 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 
 		let mut legacy_equiv_by_name = HashMap::new();
 
-		let transactions_protocol: Cow<'static, [u8]> = Cow::from({
-			let mut proto = b"/".to_vec();
-			proto.extend(protocol_id.as_ref().as_bytes());
-			proto.extend(b"/transactions/1");
+		let transactions_protocol: Cow<'static, str> = Cow::from({
+			let mut proto = String::new();
+			proto.push_str("/");
+			proto.push_str(protocol_id.as_ref());
+			proto.push_str("/transactions/1");
 			proto
 		});
 		behaviour.register_notif_protocol(transactions_protocol.clone(), Vec::new());
 		legacy_equiv_by_name.insert(transactions_protocol.clone(), Fallback::Transactions);
 
-		let block_announces_protocol: Cow<'static, [u8]> = Cow::from({
-			let mut proto = b"/".to_vec();
-			proto.extend(protocol_id.as_ref().as_bytes());
-			proto.extend(b"/block-announces/1");
+		let block_announces_protocol: Cow<'static, str> = Cow::from({
+			let mut proto = String::new();
+			proto.push_str("/");
+			proto.push_str(protocol_id.as_ref());
+			proto.push_str("/block-announces/1");
 			proto
 		});
 		behaviour.register_notif_protocol(
@@ -689,7 +691,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 	fn send_message(
 		&mut self,
 		who: &PeerId,
-		message: Option<(Cow<'static, [u8]>, Vec<u8>)>,
+		message: Option<(Cow<'static, str>, Vec<u8>)>,
 		legacy: Message<B>,
 	) {
 		send_message::<B>(
@@ -1086,7 +1088,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 	pub fn register_notifications_protocol<'a>(
 		&'a mut self,
 		engine_id: ConsensusEngineId,
-		protocol_name: impl Into<Cow<'static, [u8]>>,
+		protocol_name: impl Into<Cow<'static, str>>,
 		handshake_message: Vec<u8>,
 	) -> impl Iterator<Item = (&'a PeerId, Roles, &'a NotificationsSink)> + 'a {
 		let protocol_name = protocol_name.into();
@@ -1639,7 +1641,7 @@ fn send_message<B: BlockT>(
 	behaviour: &mut GenericProto,
 	stats: &mut HashMap<&'static str, PacketStats>,
 	who: &PeerId,
-	message: Option<(Cow<'static, [u8]>, Vec<u8>)>,
+	message: Option<(Cow<'static, str>, Vec<u8>)>,
 	legacy_message: Message<B>,
 ) {
 	let encoded = legacy_message.encode();
diff --git a/client/network/src/protocol/generic_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs
index f965980640ad6..56a5b3fb0ab2d 100644
--- a/client/network/src/protocol/generic_proto/behaviour.rs
+++ b/client/network/src/protocol/generic_proto/behaviour.rs
@@ -120,7 +120,7 @@ pub struct GenericProto {
 	/// Notification protocols. Entries are only ever added and not removed.
 	/// Contains, for each protocol, the protocol name and the message to send as part of the
 	/// initial handshake.
-	notif_protocols: Vec<(Cow<'static, [u8]>, Arc<RwLock<Vec<u8>>>)>,
+	notif_protocols: Vec<(Cow<'static, str>, Arc<RwLock<Vec<u8>>>)>,
 
 	/// Receiver for instructions about who to connect to or disconnect from.
 	peerset: sc_peerset::Peerset,
@@ -322,7 +322,7 @@ pub enum GenericProtoOut {
 		/// Id of the peer the message came from.
 		peer_id: PeerId,
 		/// Engine corresponding to the message.
-		protocol_name: Cow<'static, [u8]>,
+		protocol_name: Cow<'static, str>,
 		/// Message that has been received.
 		message: BytesMut,
 	},
@@ -360,7 +360,7 @@ impl GenericProto {
 	/// will retain the protocols that were registered then, and not any new one.
 	pub fn register_notif_protocol(
 		&mut self,
-		protocol_name: impl Into<Cow<'static, [u8]>>,
+		protocol_name: impl Into<Cow<'static, str>>,
 		handshake_msg: impl Into<Vec<u8>>
 	) {
 		self.notif_protocols.push((protocol_name.into(), Arc::new(RwLock::new(handshake_msg.into()))));
@@ -371,10 +371,10 @@ impl GenericProto {
 	/// Has no effect if the protocol is unknown.
 	pub fn set_notif_protocol_handshake(
 		&mut self,
-		protocol_name: &[u8],
+		protocol_name: &str,
 		handshake_message: impl Into<Vec<u8>>
 	) {
-		if let Some(protocol) = self.notif_protocols.iter_mut().find(|(name, _)| name == &protocol_name) {
+		if let Some(protocol) = self.notif_protocols.iter_mut().find(|(name, _)| name == protocol_name) {
 			*protocol.1.write() = handshake_message.into();
 		}
 	}
@@ -551,7 +551,7 @@ impl GenericProto {
 	pub fn write_notification(
 		&mut self,
 		target: &PeerId,
-		protocol_name: Cow<'static, [u8]>,
+		protocol_name: Cow<'static, str>,
 		message: impl Into<Vec<u8>>,
 		encoded_fallback_message: Vec<u8>,
 	) {
@@ -569,11 +569,11 @@ impl GenericProto {
 			target: "sub-libp2p",
 			"External API => Notification({:?}, {:?})",
 			target,
-			str::from_utf8(&protocol_name)
+			protocol_name,
 		);
 		trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target);
 		notifs_sink.send_sync_notification(
-			&protocol_name,
+			protocol_name,
 			encoded_fallback_message,
 			message
 		);
@@ -1374,7 +1374,7 @@ impl NetworkBehaviour for GenericProto {
 					target: "sub-libp2p",
 					"Handler({:?}) => Notification({:?})",
 					source,
-					str::from_utf8(&protocol_name)
+					protocol_name,
 				);
 				trace!(target: "sub-libp2p", "External API <= Message({:?}, {:?})", protocol_name, source);
 				let event = GenericProtoOut::Notification {
diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs
index bcdbaf848511f..43627f3d6041c 100644
--- a/client/network/src/protocol/generic_proto/handler/group.rs
+++ b/client/network/src/protocol/generic_proto/handler/group.rs
@@ -224,7 +224,7 @@ pub enum NotifsHandlerOut {
 	/// Received a message on a custom protocol substream.
 	Notification {
 		/// Name of the protocol of the message.
-		protocol_name: Cow<'static, [u8]>,
+		protocol_name: Cow<'static, str>,
 
 		/// Message that has been received.
 		message: BytesMut,
@@ -270,7 +270,7 @@ enum NotificationsSinkMessage {
 	/// Message emitted by [`NotificationsSink::reserve_notification`] and
 	/// [`NotificationsSink::write_notification_now`].
 	Notification {
-		protocol_name: Vec<u8>,
+		protocol_name: Cow<'static, str>,
 		encoded_fallback_message: Vec<u8>,
 		message: Vec<u8>,
 	},
@@ -311,13 +311,13 @@ impl NotificationsSink {
 	/// This method will be removed in a future version.
 	pub fn send_sync_notification<'a>(
 		&'a self,
-		protocol_name: &[u8],
+		protocol_name: Cow<'static, str>,
 		encoded_fallback_message: impl Into<Vec<u8>>,
 		message: impl Into<Vec<u8>>
 	) {
 		let mut lock = self.inner.sync_channel.lock();
 		let result = lock.try_send(NotificationsSinkMessage::Notification {
-			protocol_name: protocol_name.to_owned(),
+			protocol_name: protocol_name,
 			encoded_fallback_message: encoded_fallback_message.into(),
 			message: message.into()
 		});
@@ -336,12 +336,12 @@ impl NotificationsSink {
 	///
 	/// The protocol name is expected to be checked ahead of calling this method. It is a logic
 	/// error to send a notification using an unknown protocol.
-	pub async fn reserve_notification<'a>(&'a self, protocol_name: &[u8]) -> Result<Ready<'a>, ()> {
+	pub async fn reserve_notification<'a>(&'a self, protocol_name: Cow<'static, str>) -> Result<Ready<'a>, ()> {
 		let mut lock = self.inner.async_channel.lock().await;
 
 		let poll_ready = future::poll_fn(|cx| lock.poll_ready(cx)).await;
 		if poll_ready.is_ok() {
-			Ok(Ready { protocol_name: protocol_name.to_owned(), lock })
+			Ok(Ready { protocol_name: protocol_name, lock })
 		} else {
 			Err(())
 		}
@@ -355,7 +355,7 @@ pub struct Ready<'a> {
 	/// Guarded channel. The channel inside is guaranteed to not be full.
 	lock: FuturesMutexGuard<'a, mpsc::Sender<NotificationsSinkMessage>>,
 	/// Name of the protocol. Should match one of the protocols passed at initialization.
-	protocol_name: Vec<u8>,
+	protocol_name: Cow<'static, str>,
 }
 
 impl<'a> Ready<'a> {
@@ -392,7 +392,7 @@ impl NotifsHandlerProto {
 	/// ourselves or respond to handshake from the remote.
 	pub fn new(
 		legacy: RegisteredProtocol,
-		list: impl Into<Vec<(Cow<'static, [u8]>, Arc<RwLock<Vec<u8>>>)>>,
+		list: impl Into<Vec<(Cow<'static, str>, Arc<RwLock<Vec<u8>>>)>>,
 	) -> Self {
 		let list = list.into();
 
@@ -613,7 +613,7 @@ impl ProtocolsHandler for NotifsHandler {
 						message
 					} => {
 						for (handler, _) in &mut self.out_handlers {
-							if handler.protocol_name() == &protocol_name[..] && handler.is_open() {
+							if *handler.protocol_name() == protocol_name && handler.is_open() {
 								handler.send_or_discard(message);
 								continue 'poll_notifs_sink;
 							}
@@ -698,7 +698,7 @@ impl ProtocolsHandler for NotifsHandler {
 							if self.notifications_sink_rx.is_some() {
 								let msg = NotifsHandlerOut::Notification {
 									message,
-									protocol_name: handler.protocol_name().to_owned().into(),
+									protocol_name: handler.protocol_name().clone(),
 								};
 								return Poll::Ready(ProtocolsHandlerEvent::Custom(msg));
 							}
diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs
index ddd78566fcd2a..9eb8ec7471610 100644
--- a/client/network/src/protocol/generic_proto/handler/notif_in.rs
+++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs
@@ -109,7 +109,7 @@ pub enum NotifsInHandlerOut {
 impl NotifsInHandlerProto {
 	/// Builds a new `NotifsInHandlerProto`.
 	pub fn new(
-		protocol_name: impl Into<Cow<'static, [u8]>>
+		protocol_name: impl Into<Cow<'static, str>>
 	) -> Self {
 		NotifsInHandlerProto {
 			in_protocol: NotificationsIn::new(protocol_name),
@@ -136,7 +136,7 @@ impl IntoProtocolsHandler for NotifsInHandlerProto {
 
 impl NotifsInHandler {
 	/// Returns the name of the protocol that we accept.
-	pub fn protocol_name(&self) -> &[u8] {
+	pub fn protocol_name(&self) -> &Cow<'static, str> {
 		self.in_protocol.protocol_name()
 	}
 }
diff --git a/client/network/src/protocol/generic_proto/handler/notif_out.rs b/client/network/src/protocol/generic_proto/handler/notif_out.rs
index 4ba9d9a0b74aa..4079d2fa2a6b2 100644
--- a/client/network/src/protocol/generic_proto/handler/notif_out.rs
+++ b/client/network/src/protocol/generic_proto/handler/notif_out.rs
@@ -57,13 +57,13 @@ const INITIAL_KEEPALIVE_TIME: Duration = Duration::from_secs(5);
 /// See the documentation of [`NotifsOutHandler`] for more information.
 pub struct NotifsOutHandlerProto {
 	/// Name of the protocol to negotiate.
-	protocol_name: Cow<'static, [u8]>,
+	protocol_name: Cow<'static, str>,
 }
 
 impl NotifsOutHandlerProto {
 	/// Builds a new [`NotifsOutHandlerProto`]. Will use the given protocol name for the
 	/// notifications substream.
-	pub fn new(protocol_name: impl Into<Cow<'static, [u8]>>) -> Self {
+	pub fn new(protocol_name: impl Into<Cow<'static, str>>) -> Self {
 		NotifsOutHandlerProto {
 			protocol_name: protocol_name.into(),
 		}
@@ -97,7 +97,7 @@ impl IntoProtocolsHandler for NotifsOutHandlerProto {
 /// the remote for the purpose of sending notifications to it.
 pub struct NotifsOutHandler {
 	/// Name of the protocol to negotiate.
-	protocol_name: Cow<'static, [u8]>,
+	protocol_name: Cow<'static, str>,
 
 	/// Relationship with the node we're connected to.
 	state: State,
@@ -220,7 +220,7 @@ impl NotifsOutHandler {
 	}
 
 	/// Returns the name of the protocol that we negotiate.
-	pub fn protocol_name(&self) -> &[u8] {
+	pub fn protocol_name(&self) -> &Cow<'static, str> {
 		&self.protocol_name
 	}
 
diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs
index 80fd7761f8088..6b636607d8031 100644
--- a/client/network/src/protocol/generic_proto/upgrade/notifications.rs
+++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs
@@ -50,7 +50,7 @@ const MAX_HANDSHAKE_SIZE: usize = 1024;
 #[derive(Debug, Clone)]
 pub struct NotificationsIn {
 	/// Protocol name to use when negotiating the substream.
-	protocol_name: Cow<'static, [u8]>,
+	protocol_name: Cow<'static, str>,
 }
 
 /// Upgrade that opens a substream, waits for the remote to accept by sending back a status
@@ -58,7 +58,7 @@ pub struct NotificationsIn {
 #[derive(Debug, Clone)]
 pub struct NotificationsOut {
 	/// Protocol name to use when negotiating the substream.
-	protocol_name: Cow<'static, [u8]>,
+	protocol_name: Cow<'static, str>,
 	/// Message to send when we start the handshake.
 	initial_message: Vec<u8>,
 }
@@ -100,14 +100,14 @@ pub struct NotificationsOutSubstream<TSubstream> {
 
 impl NotificationsIn {
 	/// Builds a new potential upgrade.
-	pub fn new(protocol_name: impl Into<Cow<'static, [u8]>>) -> Self {
+	pub fn new(protocol_name: impl Into<Cow<'static, str>>) -> Self {
 		NotificationsIn {
 			protocol_name: protocol_name.into(),
 		}
 	}
 
 	/// Returns the name of the protocol that we accept.
-	pub fn protocol_name(&self) -> &[u8] {
+	pub fn protocol_name(&self) -> &Cow<'static, str> {
 		&self.protocol_name
 	}
 }
@@ -117,7 +117,11 @@ impl UpgradeInfo for NotificationsIn {
 	type InfoIter = iter::Once<Self::Info>;
 
 	fn protocol_info(&self) -> Self::InfoIter {
-		iter::once(self.protocol_name.clone())
+		let bytes: Cow<'static, [u8]> = match &self.protocol_name {
+			Cow::Borrowed(s) => Cow::Borrowed(s.as_bytes()),
+			Cow::Owned(s) => Cow::Owned(s.as_bytes().to_vec())
+		};
+		iter::once(bytes)
 	}
 }
 
@@ -244,7 +248,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
 
 impl NotificationsOut {
 	/// Builds a new potential upgrade.
-	pub fn new(protocol_name: impl Into<Cow<'static, [u8]>>, initial_message: impl Into<Vec<u8>>) -> Self {
+	pub fn new(protocol_name: impl Into<Cow<'static, str>>, initial_message: impl Into<Vec<u8>>) -> Self {
 		let initial_message = initial_message.into();
 		if initial_message.len() > MAX_HANDSHAKE_SIZE {
 			error!(target: "sub-libp2p", "Outbound networking handshake is above allowed protocol limit");
@@ -262,7 +266,11 @@ impl UpgradeInfo for NotificationsOut {
 	type InfoIter = iter::Once<Self::Info>;
 
 	fn protocol_info(&self) -> Self::InfoIter {
-		iter::once(self.protocol_name.clone())
+		let bytes: Cow<'static, [u8]> = match &self.protocol_name {
+			Cow::Borrowed(s) => Cow::Borrowed(s.as_bytes()),
+			Cow::Owned(s) => Cow::Owned(s.as_bytes().to_vec())
+		};
+		iter::once(bytes)
 	}
 }
 
@@ -378,10 +386,11 @@ mod tests {
 	use async_std::net::{TcpListener, TcpStream};
 	use futures::{prelude::*, channel::oneshot};
 	use libp2p::core::upgrade;
+	use std::borrow::Cow;
 
 	#[test]
 	fn basic_works() {
-		const PROTO_NAME: &'static [u8] = b"/test/proto/1";
+		const PROTO_NAME: Cow<'static, str> = Cow::Borrowed("/test/proto/1");
 		let (listener_addr_tx, listener_addr_rx) = oneshot::channel();
 
 		let client = async_std::task::spawn(async move {
@@ -420,7 +429,7 @@ mod tests {
 	fn empty_handshake() {
 		// Check that everything still works when the handshake messages are empty.
 
-		const PROTO_NAME: &'static [u8] = b"/test/proto/1";
+		const PROTO_NAME: Cow<'static, str> = Cow::Borrowed("/test/proto/1");
 		let (listener_addr_tx, listener_addr_rx) = oneshot::channel();
 
 		let client = async_std::task::spawn(async move {
@@ -457,7 +466,7 @@ mod tests {
 
 	#[test]
 	fn refused() {
-		const PROTO_NAME: &'static [u8] = b"/test/proto/1";
+		const PROTO_NAME: Cow<'static, str> = Cow::Borrowed("/test/proto/1");
 		let (listener_addr_tx, listener_addr_rx) = oneshot::channel();
 
 		let client = async_std::task::spawn(async move {
@@ -495,7 +504,7 @@ mod tests {
 
 	#[test]
 	fn large_initial_message_refused() {
-		const PROTO_NAME: &'static [u8] = b"/test/proto/1";
+		const PROTO_NAME: Cow<'static, str> = Cow::Borrowed("/test/proto/1");
 		let (listener_addr_tx, listener_addr_rx) = oneshot::channel();
 
 		let client = async_std::task::spawn(async move {
@@ -526,7 +535,7 @@ mod tests {
 
 	#[test]
 	fn large_handshake_refused() {
-		const PROTO_NAME: &'static [u8] = b"/test/proto/1";
+		const PROTO_NAME: Cow<'static, str> = Cow::Borrowed("/test/proto/1");
 		let (listener_addr_tx, listener_addr_rx) = oneshot::channel();
 
 		let client = async_std::task::spawn(async move {
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index 754b5b184c096..a3ac8371dc739 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -105,7 +105,7 @@ pub struct NetworkService<B: BlockT + 'static, H: ExHashT> {
 	/// that peer. Updated by the [`NetworkWorker`].
 	peers_notifications_sinks: Arc<Mutex<HashMap<(PeerId, ConsensusEngineId), NotificationsSink>>>,
 	/// For each legacy gossiping engine ID, the corresponding new protocol name.
-	protocol_name_by_engine: Mutex<HashMap<ConsensusEngineId, Cow<'static, [u8]>>>,
+	protocol_name_by_engine: Mutex<HashMap<ConsensusEngineId, Cow<'static, str>>>,
 	/// Field extracted from the [`Metrics`] struct and necessary to report the
 	/// notifications-related metrics.
 	notifications_sizes_metric: Option<HistogramVec>,
@@ -646,7 +646,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 				})
 			});
 
-			sink.send_sync_notification(&protocol_name, fallback, message);
+			sink.send_sync_notification(protocol_name, fallback, message);
 		} else {
 			return;
 		}
@@ -828,7 +828,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	pub fn register_notifications_protocol(
 		&self,
 		engine_id: ConsensusEngineId,
-		protocol_name: impl Into<Cow<'static, [u8]>>,
+		protocol_name: impl Into<Cow<'static, str>>,
 	) {
 		let protocol_name = protocol_name.into();
 		self.protocol_name_by_engine.lock().insert(engine_id, protocol_name.clone());
@@ -1062,7 +1062,7 @@ pub struct NotificationSender {
 	sink: NotificationsSink,
 
 	/// Name of the protocol on the wire.
-	protocol_name: Cow<'static, [u8]>,
+	protocol_name: Cow<'static, str>,
 
 	/// Engine ID used for the fallback message.
 	engine_id: ConsensusEngineId,
@@ -1076,7 +1076,7 @@ impl NotificationSender {
 	/// Returns a future that resolves when the `NotificationSender` is ready to send a notification.
 	pub async fn ready<'a>(&'a self) -> Result<NotificationSenderReady<'a>, NotificationSenderError> {
 		Ok(NotificationSenderReady {
-			ready: match self.sink.reserve_notification(&self.protocol_name).await {
+			ready: match self.sink.reserve_notification(self.protocol_name.clone()).await {
 				Ok(r) => r,
 				Err(()) => return Err(NotificationSenderError::Closed),
 			},
@@ -1158,7 +1158,7 @@ enum ServiceToWorkerMsg<B: BlockT, H: ExHashT> {
 	},
 	RegisterNotifProtocol {
 		engine_id: ConsensusEngineId,
-		protocol_name: Cow<'static, [u8]>,
+		protocol_name: Cow<'static, str>,
 	},
 	DisconnectPeer(PeerId),
 	UpdateChain,
diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs
index 5090362e37606..4b6f9dd156482 100644
--- a/client/network/src/service/tests.rs
+++ b/client/network/src/service/tests.rs
@@ -131,14 +131,14 @@ fn build_nodes_one_proto()
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 
 	let (node1, events_stream1) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))],
+		notifications_protocols: vec![(ENGINE_ID, From::from("/foo"))],
 		listen_addresses: vec![listen_addr.clone()],
 		transport: config::TransportConfig::MemoryOnly,
 		.. config::NetworkConfiguration::new_local()
 	});
 
 	let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))],
+		notifications_protocols: vec![(ENGINE_ID, From::from("/foo"))],
 		listen_addresses: vec![],
 		reserved_nodes: vec![config::MultiaddrWithPeerId {
 			multiaddr: listen_addr,
@@ -281,7 +281,7 @@ fn lots_of_incoming_peers_works() {
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 
 	let (main_node, _) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))],
+		notifications_protocols: vec![(ENGINE_ID, From::from("/foo"))],
 		listen_addresses: vec![listen_addr.clone()],
 		in_peers: u32::max_value(),
 		transport: config::TransportConfig::MemoryOnly,
@@ -298,7 +298,7 @@ fn lots_of_incoming_peers_works() {
 		let main_node_peer_id = main_node_peer_id.clone();
 
 		let (_dialing_node, event_stream) = build_test_full_node(config::NetworkConfiguration {
-			notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))],
+			notifications_protocols: vec![(ENGINE_ID, From::from("/foo"))],
 			listen_addresses: vec![],
 			reserved_nodes: vec![config::MultiaddrWithPeerId {
 				multiaddr: listen_addr.clone(),

From 6b2f7eaa4778edb5aa1f6ee673db12ac116234c3 Mon Sep 17 00:00:00 2001
From: Andronik Ordian <write@reusable.software>
Date: Sat, 29 Aug 2020 05:30:13 +0200
Subject: [PATCH 026/122] update kvdb-rocksdb to 0.9.1 and rocksdb to 6.11.4
 (#6963)

---
 Cargo.lock                | 18 +++++++++---------
 bin/node/bench/Cargo.toml |  2 +-
 client/db/Cargo.toml      |  4 ++--
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 402dfb6ec9e1f..65ddc7e2c1202 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -395,9 +395,9 @@ dependencies = [
 
 [[package]]
 name = "bindgen"
-version = "0.53.3"
+version = "0.54.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c72a978d268b1d70b0e963217e60fdabd9523a941457a6c42a7315d15c7e89e5"
+checksum = "66c0bb6167449588ff70803f4127f0684f9063097eca5016f37eb52b92c2cf36"
 dependencies = [
  "bitflags",
  "cexpr",
@@ -2735,9 +2735,9 @@ dependencies = [
 
 [[package]]
 name = "kvdb-rocksdb"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c341ef15cfb1f923fa3b5138bfbd2d4813a2c1640b473727a53351c7f0b0fa2"
+checksum = "44947dd392f09475af614d740fe0320b66d01cb5b977f664bbbb5e45a70ea4c1"
 dependencies = [
  "fs-swap",
  "kvdb",
@@ -3246,9 +3246,9 @@ dependencies = [
 
 [[package]]
 name = "librocksdb-sys"
-version = "6.7.4"
+version = "6.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "883213ae3d09bfc3d104aefe94b25ebb183b6f4d3a515b23b14817e1f4854005"
+checksum = "eb5b56f651c204634b936be2f92dbb42c36867e00ff7fe2405591f3b9fa66f09"
 dependencies = [
  "bindgen",
  "cc",
@@ -6096,9 +6096,9 @@ dependencies = [
 
 [[package]]
 name = "rocksdb"
-version = "0.14.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61aa17a99a2413cd71c1106691bf59dad7de0cd5099127f90e9d99c429c40d4a"
+checksum = "23d83c02c429044d58474eaf5ae31e062d0de894e21125b47437ec0edc1397e6"
 dependencies = [
  "libc",
  "librocksdb-sys",
@@ -9547,7 +9547,7 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56"
 dependencies = [
- "rand 0.5.6",
+ "rand 0.7.3",
 ]
 
 [[package]]
diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml
index adefbd07082a3..1914f460be0f1 100644
--- a/bin/node/bench/Cargo.toml
+++ b/bin/node/bench/Cargo.toml
@@ -22,7 +22,7 @@ serde_json = "1.0.41"
 structopt = "0.3"
 derive_more = "0.99.2"
 kvdb = "0.7"
-kvdb-rocksdb = "0.9"
+kvdb-rocksdb = "0.9.1"
 sp-trie = { version = "2.0.0-rc6", path = "../../../primitives/trie" }
 sp-core = { version = "2.0.0-rc6", path = "../../../primitives/core" }
 sp-consensus = { version = "0.8.0-rc6", path = "../../../primitives/consensus/common" }
diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml
index 28ef90cf231e2..004a7753e42d3 100644
--- a/client/db/Cargo.toml
+++ b/client/db/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 parking_lot = "0.10.0"
 log = "0.4.8"
 kvdb = "0.7.0"
-kvdb-rocksdb = { version = "0.9", optional = true }
+kvdb-rocksdb = { version = "0.9.1", optional = true }
 kvdb-memorydb = "0.7.0"
 linked-hash-map = "0.5.2"
 hash-db = "0.15.2"
@@ -42,7 +42,7 @@ sp-keyring = { version = "2.0.0-rc6", path = "../../primitives/keyring" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../test-utils/runtime/client" }
 env_logger = "0.7.0"
 quickcheck = "0.9"
-kvdb-rocksdb = "0.9"
+kvdb-rocksdb = "0.9.1"
 tempfile = "3"
 
 [features]

From 7d53c94d57c7600efa23d0efc9699af0677b0184 Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Mon, 31 Aug 2020 10:55:43 +0200
Subject: [PATCH 027/122] Use AsyncReadExt::read_exact, not just read (#6977)

---
 .../src/protocol/generic_proto/upgrade/notifications.rs       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs
index 6b636607d8031..51fbc8d9c60ba 100644
--- a/client/network/src/protocol/generic_proto/upgrade/notifications.rs
+++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs
@@ -148,7 +148,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
 
 			let mut initial_message = vec![0u8; initial_message_len];
 			if !initial_message.is_empty() {
-				socket.read(&mut initial_message).await?;
+				socket.read_exact(&mut initial_message).await?;
 			}
 
 			let substream = NotificationsInSubstream {
@@ -300,7 +300,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
 
 			let mut handshake = vec![0u8; handshake_len];
 			if !handshake.is_empty() {
-				socket.read(&mut handshake).await?;
+				socket.read_exact(&mut handshake).await?;
 			}
 
 			Ok((handshake, NotificationsOutSubstream {

From 539f4d25241fc7dd815ab9ef4bd6422d193b599a Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Mon, 31 Aug 2020 14:37:06 +0200
Subject: [PATCH 028/122] client/cli/src/config: Warn on low file descriptor
 limit (#6956)

* client/cli/src/config: Warn on low file descriptor limit

Substrate sets the soft file descriptor limit to the hard limit at
startup. In the case of the latter being low already (< 10_000) a
Substrate node under high demand might run into issues e.g. when opening
up new TCP connections or persisting data to the database.

With this commit a warn message is printed to stderr.

* client/cli/Cargo.toml: Update to fdlimit 0.2.0
---
 Cargo.lock                     |  4 ++--
 client/cli/Cargo.toml          |  2 +-
 client/cli/src/config.rs       | 23 ++++++++++++++++++-----
 client/service/test/Cargo.toml |  2 +-
 4 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 65ddc7e2c1202..7cd624f7835ba 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1486,9 +1486,9 @@ checksum = "36a9cb09840f81cd211e435d00a4e487edd263dc3c8ff815c32dd76ad668ebed"
 
 [[package]]
 name = "fdlimit"
-version = "0.1.4"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0da54a593b34c71b889ee45f5b5bb900c74148c5f7f8c6a9479ee7899f69603c"
+checksum = "47bc6e222b8349b2bd0acb85a1d16d22852376b3ceed2a7f09c2692c3d8a78d0"
 dependencies = [
  "libc",
 ]
diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml
index 2643376f84131..8b634d687c717 100644
--- a/client/cli/Cargo.toml
+++ b/client/cli/Cargo.toml
@@ -22,7 +22,7 @@ ansi_term = "0.12.1"
 lazy_static = "1.4.0"
 tokio = { version = "0.2.21", features = [ "signal", "rt-core", "rt-threaded", "blocking" ] }
 futures = "0.3.4"
-fdlimit = "0.1.4"
+fdlimit = "0.2.0"
 libp2p = "0.24.0"
 parity-scale-codec = "1.3.0"
 hex = "0.4.2"
diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs
index ff0222216ce15..5da49fefd7acf 100644
--- a/client/cli/src/config.rs
+++ b/client/cli/src/config.rs
@@ -24,6 +24,7 @@ use crate::{
 	init_logger, DatabaseParams, ImportParams, KeystoreParams, NetworkParams, NodeKeyParams,
 	OffchainWorkerParams, PruningParams, SharedParams, SubstrateCli,
 };
+use log::warn;
 use names::{Generator, Name};
 use sc_client_api::execution_extensions::ExecutionStrategies;
 use sc_service::config::{
@@ -38,9 +39,12 @@ use std::path::PathBuf;
 /// The maximum number of characters for a node name.
 pub(crate) const NODE_NAME_MAX_LENGTH: usize = 64;
 
-/// default sub directory to store network config
+/// Default sub directory to store network config.
 pub(crate) const DEFAULT_NETWORK_CONFIG_PATH: &'static str = "network";
 
+/// The recommended open file descriptor limit to be configured for the process.
+const RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT: u64 = 10_000;
+
 /// Default configuration values used by Substrate
 ///
 /// These values will be used by [`CliConfiguritation`] to set
@@ -531,17 +535,26 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 	///
 	/// This method:
 	///
-	/// 1. Set the panic handler
-	/// 2. Raise the FD limit
-	/// 3. Initialize the logger
+	/// 1. Sets the panic handler
+	/// 2. Initializes the logger
+	/// 3. Raises the FD limit
 	fn init<C: SubstrateCli>(&self) -> Result<()> {
 		let logger_pattern = self.log_filters()?;
 
 		sp_panic_handler::set(&C::support_url(), &C::impl_version());
 
-		fdlimit::raise_fd_limit();
 		init_logger(&logger_pattern);
 
+		if let Some(new_limit) = fdlimit::raise_fd_limit() {
+			if new_limit < RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT {
+				warn!(
+					"Low open file descriptor limit configured for the process. \
+					 Current value: {:?}, recommended value: {:?}.",
+					new_limit, RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT,
+				);
+			}
+		}
+
 		Ok(())
 	}
 }
diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml
index 501843dc5b6ca..03d5e264c85da 100644
--- a/client/service/test/Cargo.toml
+++ b/client/service/test/Cargo.toml
@@ -18,7 +18,7 @@ tokio = "0.1.22"
 futures01 = { package = "futures", version = "0.1.29" }
 log = "0.4.8"
 env_logger = "0.7.0"
-fdlimit = "0.1.4"
+fdlimit = "0.2.0"
 parking_lot = "0.10.0"
 sc-light = { version = "2.0.0-rc6", path = "../../light" }
 sp-blockchain = { version = "2.0.0-rc6", path = "../../../primitives/blockchain" }

From 8182870e7db5b4684d93cd1b21cd83b6a6a3246d Mon Sep 17 00:00:00 2001
From: cheme <emericchevalier.pro@gmail.com>
Date: Mon, 31 Aug 2020 16:16:30 +0200
Subject: [PATCH 029/122] Update substrate bip39 version. (#6955)

* update bip39 version

* and lock
---
 Cargo.lock                 | 5 +++--
 primitives/core/Cargo.toml | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7cd624f7835ba..136b2c81191e0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8620,14 +8620,15 @@ dependencies = [
 
 [[package]]
 name = "substrate-bip39"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c004e8166d6e0aa3a9d5fa673e5b7098ff25f930de1013a21341988151e681bb"
+checksum = "bed6646a0159b9935b5d045611560eeef842b78d7adc3ba36f5ca325a13a0236"
 dependencies = [
  "hmac",
  "pbkdf2",
  "schnorrkel",
  "sha2 0.8.2",
+ "zeroize",
 ]
 
 [[package]]
diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml
index 1375fa228bfd1..518c35eae4d4f 100644
--- a/primitives/core/Cargo.toml
+++ b/primitives/core/Cargo.toml
@@ -26,7 +26,7 @@ hash-db = { version = "0.15.2", default-features = false }
 hash256-std-hasher = { version = "0.15.2", default-features = false }
 base58 = { version = "0.1.0", optional = true }
 rand = { version = "0.7.3", optional = true, features = ["small_rng"] }
-substrate-bip39 = { version = "0.4.1", optional = true }
+substrate-bip39 = { version = "0.4.2", optional = true }
 tiny-bip39 = { version = "0.7", optional = true }
 regex = { version = "1.3.1", optional = true }
 num-traits = { version = "0.2.8", default-features = false }

From b6220158a50f0403e47cba85e92a1daaed5a2671 Mon Sep 17 00:00:00 2001
From: Gerben van de Wiel <clshannon@protonmail.com>
Date: Mon, 31 Aug 2020 16:20:41 +0200
Subject: [PATCH 030/122] Inverting events set and changed in nicks pallet
 (#6989)

Fixing #6988
---
 frame/nicks/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs
index 56262819c9654..ce0d65d881669 100644
--- a/frame/nicks/src/lib.rs
+++ b/frame/nicks/src/lib.rs
@@ -149,12 +149,12 @@ decl_module! {
 			ensure!(name.len() <= T::MaxLength::get(), Error::<T>::TooLong);
 
 			let deposit = if let Some((_, deposit)) = <NameOf<T>>::get(&sender) {
-				Self::deposit_event(RawEvent::NameSet(sender.clone()));
+				Self::deposit_event(RawEvent::NameChanged(sender.clone()));
 				deposit
 			} else {
 				let deposit = T::ReservationFee::get();
 				T::Currency::reserve(&sender, deposit.clone())?;
-				Self::deposit_event(RawEvent::NameChanged(sender.clone()));
+				Self::deposit_event(RawEvent::NameSet(sender.clone()));
 				deposit
 			};
 

From dbbaef19d593b56693339baaf9f6a92c7e58bc6d Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Mon, 31 Aug 2020 17:41:17 +0200
Subject: [PATCH 031/122] Silence the error about non-registered protocols
 (#6987)

* Silence the error about non-registered protocols

* Silence the other two locations as well
---
 client/network/src/protocol.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index ab04aa2cbb0f7..4d651bb64e9a9 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -658,7 +658,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 						messages: vec![(msg.engine_id, From::from(msg.data))],
 					}
 				} else {
-					warn!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id);
+					debug!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id);
 					CustomMessageOutcome::None
 				},
 			GenericMessage::ConsensusBatch(messages) => {
@@ -668,7 +668,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 						if self.protocol_name_by_engine.contains_key(&msg.engine_id) {
 							Some((msg.engine_id, From::from(msg.data)))
 						} else {
-							warn!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id);
+							debug!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id);
 							None
 						}
 					})
@@ -1828,7 +1828,7 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 						CustomMessageOutcome::None
 					}
 					None => {
-						error!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name);
+						debug!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name);
 						CustomMessageOutcome::None
 					}
 				}

From 79726738bb9ea94ca00844cd44ccfc95bf38c007 Mon Sep 17 00:00:00 2001
From: Ashley <ashley.ruglys@gmail.com>
Date: Mon, 31 Aug 2020 19:05:29 +0200
Subject: [PATCH 032/122] Change browser-demo build.sh to use python 3 again
 (#6992)

---
 bin/node/cli/browser-demo/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/node/cli/browser-demo/build.sh b/bin/node/cli/browser-demo/build.sh
index ea0380b760e31..be52b7a523f01 100755
--- a/bin/node/cli/browser-demo/build.sh
+++ b/bin/node/cli/browser-demo/build.sh
@@ -1,4 +1,4 @@
 #!/usr/bin/env sh
 cargo +nightly build --release -p node-cli --target wasm32-unknown-unknown --no-default-features --features browser -Z features=itarget
 wasm-bindgen ../../../../target/wasm32-unknown-unknown/release/node_cli.wasm --out-dir pkg --target web
-python -m SimpleHTTPServer 8000
+python -m http.server 8000

From 92311666a6aaee4aa6556d6b810bfdf0e6ab3a97 Mon Sep 17 00:00:00 2001
From: Xiliang Chen <xlchen1291@gmail.com>
Date: Tue, 1 Sep 2020 15:26:25 +1200
Subject: [PATCH 033/122] fix pallet-evm features (#6995)

---
 frame/evm/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frame/evm/Cargo.toml b/frame/evm/Cargo.toml
index 0f14f3afe4862..739a13a165816 100644
--- a/frame/evm/Cargo.toml
+++ b/frame/evm/Cargo.toml
@@ -22,7 +22,7 @@ sp-core = { version = "2.0.0-rc6", default-features = false, path = "../../primi
 sp-runtime = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/runtime" }
 sp-std = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/std" }
 sp-io = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/io" }
-primitive-types = { version = "0.7.0", default-features = false, features = ["rlp"] }
+primitive-types = { version = "0.7.0", default-features = false, features = ["rlp", "byteorder"] }
 rlp = { version = "0.4", default-features = false }
 evm = { version = "0.17", default-features = false }
 sha3 = { version = "0.8", default-features = false }

From 9007d2860e75cf8113c597f5e440542c560ac498 Mon Sep 17 00:00:00 2001
From: Ashley <ashley.ruglys@gmail.com>
Date: Tue, 1 Sep 2020 11:06:22 +0200
Subject: [PATCH 034/122] Move subcommands from sc-cli to nodes (#6948)

---
 bin/node-template/node/src/cli.rs     |  26 +-
 bin/node-template/node/src/command.rs |  56 +++-
 bin/node/cli/src/cli.rs               |  25 +-
 bin/node/cli/src/command.rs           |  52 +++-
 client/cli/src/commands/mod.rs        | 403 --------------------------
 client/cli/src/runner.rs              |  51 +---
 6 files changed, 141 insertions(+), 472 deletions(-)

diff --git a/bin/node-template/node/src/cli.rs b/bin/node-template/node/src/cli.rs
index 46ab9bc3dafac..f3667fa79d19e 100644
--- a/bin/node-template/node/src/cli.rs
+++ b/bin/node-template/node/src/cli.rs
@@ -1,5 +1,5 @@
 use structopt::StructOpt;
-use sc_cli::{RunCmd, Subcommand};
+use sc_cli::RunCmd;
 
 #[derive(Debug, StructOpt)]
 pub struct Cli {
@@ -9,3 +9,27 @@ pub struct Cli {
 	#[structopt(flatten)]
 	pub run: RunCmd,
 }
+
+#[derive(Debug, StructOpt)]
+pub enum Subcommand {
+	/// Build a chain specification.
+	BuildSpec(sc_cli::BuildSpecCmd),
+
+	/// Validate blocks.
+	CheckBlock(sc_cli::CheckBlockCmd),
+
+	/// Export blocks.
+	ExportBlocks(sc_cli::ExportBlocksCmd),
+
+	/// Export the state of a given block into a chain spec.
+	ExportState(sc_cli::ExportStateCmd),
+
+	/// Import blocks.
+	ImportBlocks(sc_cli::ImportBlocksCmd),
+
+	/// Remove the whole chain.
+	PurgeChain(sc_cli::PurgeChainCmd),
+
+	/// Revert the chain to a previous state.
+	Revert(sc_cli::RevertCmd),
+}
diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs
index 9cd2248d6547a..98c56e948300e 100644
--- a/bin/node-template/node/src/command.rs
+++ b/bin/node-template/node/src/command.rs
@@ -16,7 +16,7 @@
 // limitations under the License.
 
 use crate::chain_spec;
-use crate::cli::Cli;
+use crate::cli::{Cli, Subcommand};
 use crate::service;
 use sc_cli::{SubstrateCli, RuntimeVersion, Role, ChainSpec};
 use sc_service::PartialComponents;
@@ -66,15 +66,55 @@ impl SubstrateCli for Cli {
 pub fn run() -> sc_cli::Result<()> {
 	let cli = Cli::from_args();
 
-	match cli.subcommand {
-		Some(ref subcommand) => {
-			let runner = cli.create_runner(subcommand)?;
-			runner.run_subcommand(subcommand, |config| {
-				let PartialComponents { client, backend, task_manager, import_queue, .. }
+	match &cli.subcommand {
+		Some(Subcommand::BuildSpec(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.sync_run(|config| cmd.run(config.chain_spec, config.network))
+		},
+		Some(Subcommand::CheckBlock(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, import_queue, ..}
 					= new_partial(&config)?;
-				Ok((client, backend, import_queue, task_manager))
+				Ok((cmd.run(client, import_queue), task_manager))
 			})
-		}
+		},
+		Some(Subcommand::ExportBlocks(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, config.database), task_manager))
+			})
+		},
+		Some(Subcommand::ExportState(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, config.chain_spec), task_manager))
+			})
+		},
+		Some(Subcommand::ImportBlocks(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, import_queue, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, import_queue), task_manager))
+			})
+		},
+		Some(Subcommand::PurgeChain(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.sync_run(|config| cmd.run(config.database))
+		},
+		Some(Subcommand::Revert(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, backend, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, backend), task_manager))
+			})
+		},
 		None => {
 			let runner = cli.create_runner(&cli.run)?;
 			runner.run_node_until_exit(|config| match config.role {
diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs
index 42a13fcb39070..2130ff1e4b106 100644
--- a/bin/node/cli/src/cli.rs
+++ b/bin/node/cli/src/cli.rs
@@ -33,10 +33,6 @@ pub struct Cli {
 /// Possible subcommands of the main binary.
 #[derive(Debug, StructOpt)]
 pub enum Subcommand {
-	/// A set of base subcommands handled by `sc_cli`.
-	#[structopt(flatten)]
-	Base(sc_cli::Subcommand),
-
 	/// Key management cli utilities
 	Key(KeySubcommand),
 
@@ -59,4 +55,25 @@ pub enum Subcommand {
 
 	/// Sign a message, with a given (secret) key.
 	Sign(SignCmd),
+
+	/// Build a chain specification.
+	BuildSpec(sc_cli::BuildSpecCmd),
+
+	/// Validate blocks.
+	CheckBlock(sc_cli::CheckBlockCmd),
+
+	/// Export blocks.
+	ExportBlocks(sc_cli::ExportBlocksCmd),
+
+	/// Export the state of a given block into a chain spec.
+	ExportState(sc_cli::ExportStateCmd),
+
+	/// Import blocks.
+	ImportBlocks(sc_cli::ImportBlocksCmd),
+
+	/// Remove the whole chain.
+	PurgeChain(sc_cli::PurgeChainCmd),
+
+	/// Revert the chain to a previous state.
+	Revert(sc_cli::RevertCmd),
 }
diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs
index 10e9413702b81..a715b2ecaa091 100644
--- a/bin/node/cli/src/command.rs
+++ b/bin/node/cli/src/command.rs
@@ -97,13 +97,53 @@ pub fn run() -> Result<()> {
 		Some(Subcommand::Sign(cmd)) => cmd.run(),
 		Some(Subcommand::Verify(cmd)) => cmd.run(),
 		Some(Subcommand::Vanity(cmd)) => cmd.run(),
-		Some(Subcommand::Base(subcommand)) => {
-			let runner = cli.create_runner(subcommand)?;
-			runner.run_subcommand(subcommand, |config| {
-				let PartialComponents { client, backend, task_manager, import_queue, ..}
+		Some(Subcommand::BuildSpec(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.sync_run(|config| cmd.run(config.chain_spec, config.network))
+		},
+		Some(Subcommand::CheckBlock(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, import_queue, ..}
 					= new_partial(&config)?;
-				Ok((client, backend, import_queue, task_manager))
+				Ok((cmd.run(client, import_queue), task_manager))
 			})
-		}
+		},
+		Some(Subcommand::ExportBlocks(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, config.database), task_manager))
+			})
+		},
+		Some(Subcommand::ExportState(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, config.chain_spec), task_manager))
+			})
+		},
+		Some(Subcommand::ImportBlocks(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, import_queue, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, import_queue), task_manager))
+			})
+		},
+		Some(Subcommand::PurgeChain(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.sync_run(|config| cmd.run(config.database))
+		},
+		Some(Subcommand::Revert(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { client, task_manager, backend, ..}
+					= new_partial(&config)?;
+				Ok((cmd.run(client, backend), task_manager))
+			})
+		},
 	}
 }
diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs
index 108c38b19db30..7b740d1003238 100644
--- a/client/cli/src/commands/mod.rs
+++ b/client/cli/src/commands/mod.rs
@@ -34,9 +34,6 @@ mod inspect;
 mod key;
 pub mod utils;
 
-use std::fmt::Debug;
-use structopt::StructOpt;
-
 pub use self::{
 	build_spec_cmd::BuildSpecCmd,
 	check_block_cmd::CheckBlockCmd,
@@ -56,403 +53,3 @@ pub use self::{
 	revert_cmd::RevertCmd,
 	run_cmd::RunCmd,
 };
-
-/// All core commands that are provided by default.
-///
-/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From
-/// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of
-/// `Run` are exported as main executable parameters.
-#[derive(Debug, StructOpt)]
-pub enum Subcommand {
-	/// Build a spec.json file, outputs to stdout.
-	BuildSpec(BuildSpecCmd),
-
-	/// Export blocks to a file.
-	ExportBlocks(ExportBlocksCmd),
-
-	/// Import blocks from file.
-	ImportBlocks(ImportBlocksCmd),
-
-	/// Validate a single block.
-	CheckBlock(CheckBlockCmd),
-
-	/// Export state as raw chain spec.
-	ExportState(ExportStateCmd),
-
-	/// Revert chain to the previous state.
-	Revert(RevertCmd),
-
-	/// Remove the whole chain data.
-	PurgeChain(PurgeChainCmd),
-}
-
-/// Macro that helps implement CliConfiguration on an enum of subcommand automatically
-///
-/// # Example
-///
-/// ```
-/// # #[macro_use] extern crate sc_cli;
-///
-/// # struct EmptyVariant {}
-///
-///	# impl sc_cli::CliConfiguration for EmptyVariant {
-///	#     fn shared_params(&self) -> &sc_cli::SharedParams { unimplemented!() }
-///	#     fn chain_id(&self, _: bool) -> sc_cli::Result<String> { Ok("test-chain-id".to_string()) }
-///	# }
-///
-/// # fn main() {
-/// enum Subcommand {
-///	    Variant1(EmptyVariant),
-///	    Variant2(EmptyVariant),
-///	}
-///
-/// substrate_cli_subcommands!(
-///     Subcommand => Variant1, Variant2
-/// );
-///
-/// # use sc_cli::CliConfiguration;
-/// # assert_eq!(Subcommand::Variant1(EmptyVariant {}).chain_id(false).unwrap(), "test-chain-id");
-///
-/// # }
-/// ```
-///
-/// Which will expand to:
-///
-/// ```ignore
-/// impl CliConfiguration for Subcommand {
-///	    fn base_path(&self) -> Result<Option<PathBuf>> {
-///	        match self {
-///	            Subcommand::Variant1(cmd) => cmd.base_path(),
-///	            Subcommand::Variant2(cmd) => cmd.base_path(),
-///	        }
-///	    }
-///
-///	    fn is_dev(&self) -> Result<bool> {
-///	        match self {
-///	            Subcommand::Variant1(cmd) => cmd.is_dev(),
-///	            Subcommand::Variant2(cmd) => cmd.is_dev(),
-///	        }
-///	    }
-///
-///     // ...
-/// }
-/// ```
-#[macro_export]
-macro_rules! substrate_cli_subcommands {
-	($enum:ident => $($variant:ident),*) => {
-		impl $crate::CliConfiguration for $enum {
-			fn shared_params(&self) -> &$crate::SharedParams {
-				match self {
-					$($enum::$variant(cmd) => cmd.shared_params()),*
-				}
-			}
-
-			fn import_params(&self) -> Option<&$crate::ImportParams> {
-				match self {
-					$($enum::$variant(cmd) => cmd.import_params()),*
-				}
-			}
-
-			fn pruning_params(&self) -> Option<&$crate::PruningParams> {
-				match self {
-					$($enum::$variant(cmd) => cmd.pruning_params()),*
-				}
-			}
-
-			fn keystore_params(&self) -> Option<&$crate::KeystoreParams> {
-				match self {
-					$($enum::$variant(cmd) => cmd.keystore_params()),*
-				}
-			}
-
-			fn network_params(&self) -> Option<&$crate::NetworkParams> {
-				match self {
-					$($enum::$variant(cmd) => cmd.network_params()),*
-				}
-			}
-
-			fn offchain_worker_params(&self) -> Option<&$crate::OffchainWorkerParams> {
-				match self {
-					$($enum::$variant(cmd) => cmd.offchain_worker_params()),*
-				}
-			}
-
-			fn database_params(&self) -> Option<&$crate::DatabaseParams> {
-				match self {
-					$($enum::$variant(cmd) => cmd.database_params()),*
-				}
-			}
-
-			fn base_path(&self) -> $crate::Result<::std::option::Option<sc_service::config::BasePath>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.base_path()),*
-				}
-			}
-
-			fn is_dev(&self) -> $crate::Result<bool> {
-				match self {
-					$($enum::$variant(cmd) => cmd.is_dev()),*
-				}
-			}
-
-			fn role(&self, is_dev: bool) -> $crate::Result<::sc_service::Role> {
-				match self {
-					$($enum::$variant(cmd) => cmd.role(is_dev)),*
-				}
-			}
-
-			fn transaction_pool(&self)
-			-> $crate::Result<::sc_service::config::TransactionPoolOptions> {
-				match self {
-					$($enum::$variant(cmd) => cmd.transaction_pool()),*
-				}
-			}
-
-			fn network_config(
-				&self,
-				chain_spec: &std::boxed::Box<dyn sc_service::ChainSpec>,
-				is_dev: bool,
-				net_config_dir: std::path::PathBuf,
-				client_id: &str,
-				node_name: &str,
-				node_key: sc_service::config::NodeKeyConfig,
-				default_listen_port: u16,
-			) -> $crate::Result<::sc_service::config::NetworkConfiguration> {
-				match self {
-					$(
-						$enum::$variant(cmd) => cmd.network_config(
-							chain_spec,
-							is_dev,
-							net_config_dir,
-							client_id,
-							node_name,
-							node_key,
-							default_listen_port,
-						)
-					),*
-				}
-			}
-
-			fn keystore_config(&self, base_path: &::std::path::PathBuf)
-			-> $crate::Result<::sc_service::config::KeystoreConfig> {
-				match self {
-					$($enum::$variant(cmd) => cmd.keystore_config(base_path)),*
-				}
-			}
-
-			fn database_cache_size(&self) -> $crate::Result<::std::option::Option<usize>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.database_cache_size()),*
-				}
-			}
-
-			fn database_config(
-				&self,
-				base_path: &::std::path::PathBuf,
-				cache_size: usize,
-				database: $crate::Database,
-			) -> $crate::Result<::sc_service::config::DatabaseConfig> {
-				match self {
-					$($enum::$variant(cmd) => cmd.database_config(base_path, cache_size, database)),*
-				}
-			}
-
-			fn database(&self) -> $crate::Result<::std::option::Option<$crate::Database>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.database()),*
-				}
-			}
-
-			fn state_cache_size(&self) -> $crate::Result<usize> {
-				match self {
-					$($enum::$variant(cmd) => cmd.state_cache_size()),*
-				}
-			}
-
-			fn state_cache_child_ratio(&self) -> $crate::Result<::std::option::Option<usize>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.state_cache_child_ratio()),*
-				}
-			}
-
-			fn pruning(&self, unsafe_pruning: bool, role: &::sc_service::Role)
-			-> $crate::Result<::sc_service::config::PruningMode> {
-				match self {
-					$($enum::$variant(cmd) => cmd.pruning(unsafe_pruning, role)),*
-				}
-			}
-
-			fn chain_id(&self, is_dev: bool) -> $crate::Result<String> {
-				match self {
-					$($enum::$variant(cmd) => cmd.chain_id(is_dev)),*
-				}
-			}
-
-			fn init<C: $crate::SubstrateCli>(&self) -> $crate::Result<()> {
-				match self {
-					$($enum::$variant(cmd) => cmd.init::<C>()),*
-				}
-			}
-
-			fn node_name(&self) -> $crate::Result<String> {
-				match self {
-					$($enum::$variant(cmd) => cmd.node_name()),*
-				}
-			}
-
-			fn wasm_method(&self) -> $crate::Result<::sc_service::config::WasmExecutionMethod> {
-				match self {
-					$($enum::$variant(cmd) => cmd.wasm_method()),*
-				}
-			}
-
-			fn execution_strategies(&self, is_dev: bool, is_validator: bool)
-			-> $crate::Result<::sc_client_api::execution_extensions::ExecutionStrategies> {
-				match self {
-					$($enum::$variant(cmd) => cmd.execution_strategies(is_dev, is_validator)),*
-				}
-			}
-
-			fn rpc_ipc(&self) -> $crate::Result<::std::option::Option<::std::string::String>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.rpc_ipc()),*
-				}
-			}
-
-			fn rpc_http(
-				&self,
-				default_listen_port: u16,
-			) -> $crate::Result<std::option::Option<std::net::SocketAddr>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.rpc_http(default_listen_port)),*
-				}
-			}
-
-			fn rpc_ws(
-				&self,
-				default_listen_port: u16,
-			) -> $crate::Result<std::option::Option<std::net::SocketAddr>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.rpc_ws(default_listen_port)),*
-				}
-			}
-
-			fn rpc_methods(&self) -> $crate::Result<sc_service::config::RpcMethods> {
-				match self {
-					$($enum::$variant(cmd) => cmd.rpc_methods()),*
-				}
-			}
-
-			fn rpc_ws_max_connections(&self) -> $crate::Result<::std::option::Option<usize>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.rpc_ws_max_connections()),*
-				}
-			}
-
-			fn rpc_cors(&self, is_dev: bool)
-			-> $crate::Result<std::option::Option<std::vec::Vec<String>>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.rpc_cors(is_dev)),*
-				}
-			}
-
-			fn prometheus_config(&self, default_listen_port: u16)
-			-> $crate::Result<std::option::Option<sc_service::config::PrometheusConfig>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.prometheus_config(default_listen_port)),*
-				}
-			}
-
-			fn telemetry_endpoints(
-				&self,
-				chain_spec: &Box<dyn sc_service::ChainSpec>,
-			) -> $crate::Result<std::option::Option<sc_service::config::TelemetryEndpoints>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.telemetry_endpoints(chain_spec)),*
-				}
-			}
-
-			fn telemetry_external_transport(&self)
-			-> $crate::Result<::std::option::Option<::sc_service::config::ExtTransport>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.telemetry_external_transport()),*
-				}
-			}
-
-			fn default_heap_pages(&self) -> $crate::Result<::std::option::Option<u64>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.default_heap_pages()),*
-				}
-			}
-
-			fn offchain_worker(
-				&self,
-				role: &::sc_service::Role,
-			) -> $crate::Result<::sc_service::config::OffchainWorkerConfig> {
-				match self {
-					$($enum::$variant(cmd) => cmd.offchain_worker(role)),*
-				}
-			}
-
-			fn force_authoring(&self) -> $crate::Result<bool> {
-				match self {
-					$($enum::$variant(cmd) => cmd.force_authoring()),*
-				}
-			}
-
-			fn disable_grandpa(&self) -> $crate::Result<bool> {
-				match self {
-					$($enum::$variant(cmd) => cmd.disable_grandpa()),*
-				}
-			}
-
-			fn dev_key_seed(&self, is_dev: bool) -> $crate::Result<::std::option::Option<String>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.dev_key_seed(is_dev)),*
-				}
-			}
-
-			fn tracing_targets(&self) -> $crate::Result<::std::option::Option<String>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.tracing_targets()),*
-				}
-			}
-
-			fn tracing_receiver(&self) -> $crate::Result<::sc_service::TracingReceiver> {
-				match self {
-					$($enum::$variant(cmd) => cmd.tracing_receiver()),*
-				}
-			}
-
-			fn node_key(&self, net_config_dir: &::std::path::PathBuf)
-			-> $crate::Result<::sc_service::config::NodeKeyConfig> {
-				match self {
-					$($enum::$variant(cmd) => cmd.node_key(net_config_dir)),*
-				}
-			}
-
-			fn max_runtime_instances(&self) -> $crate::Result<::std::option::Option<usize>> {
-				match self {
-					$($enum::$variant(cmd) => cmd.max_runtime_instances()),*
-				}
-			}
-
-			fn log_filters(&self) -> $crate::Result<String> {
-				match self {
-					$($enum::$variant(cmd) => cmd.log_filters()),*
-				}
-			}
-		}
-	}
-}
-
-substrate_cli_subcommands!(
-	Subcommand =>
-		BuildSpec,
-		ExportBlocks,
-		ExportState,
-		ImportBlocks,
-		CheckBlock,
-		Revert,
-		PurgeChain
-);
diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs
index f2558b1bb6070..64bd88d63130b 100644
--- a/client/cli/src/runner.rs
+++ b/client/cli/src/runner.rs
@@ -18,7 +18,6 @@
 
 use crate::CliConfiguration;
 use crate::Result;
-use crate::Subcommand;
 use crate::SubstrateCli;
 use chrono::prelude::*;
 use futures::pin_mut;
@@ -26,10 +25,8 @@ use futures::select;
 use futures::{future, future::FutureExt, Future};
 use log::info;
 use sc_service::{Configuration, TaskType, TaskManager};
-use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL};
-use std::{fmt::Debug, marker::PhantomData, str::FromStr, sync::Arc};
-use sc_client_api::{UsageProvider, BlockBackend, StorageProvider};
+use std::marker::PhantomData;
 
 #[cfg(target_family = "unix")]
 async fn main<F, E>(func: F) -> std::result::Result<(), Box<dyn std::error::Error>>
@@ -173,52 +170,6 @@ impl<C: SubstrateCli> Runner<C> {
 		info!("⛓  Native runtime: {}", C::native_runtime_version(&self.config.chain_spec));
 	}
 
-	/// A helper function that runs a future with tokio and stops if the process receives the signal
-	/// `SIGTERM` or `SIGINT`.
-	pub fn run_subcommand<BU, B, BA, IQ, CL>(self, subcommand: &Subcommand, builder: BU)
-		-> Result<()>
-	where
-		BU: FnOnce(Configuration)
-			-> sc_service::error::Result<(Arc<CL>, Arc<BA>, IQ, TaskManager)>,
-		B: BlockT + for<'de> serde::Deserialize<'de>,
-		BA: sc_client_api::backend::Backend<B> + 'static,
-		IQ: sc_service::ImportQueue<B> + 'static,
-		<B as BlockT>::Hash: FromStr,
-		<<B as BlockT>::Hash as FromStr>::Err: Debug,
-		<<<B as BlockT>::Header as HeaderT>::Number as FromStr>::Err: Debug,
-		CL: UsageProvider<B> + BlockBackend<B> + StorageProvider<B, BA> + Send + Sync +
-		'static,
-	{
-		let chain_spec = self.config.chain_spec.cloned_box();
-		let network_config = self.config.network.clone();
-		let db_config = self.config.database.clone();
-
-		match subcommand {
-			Subcommand::BuildSpec(cmd) => cmd.run(chain_spec, network_config),
-			Subcommand::ExportBlocks(cmd) => {
-				let (client, _, _, task_manager) = builder(self.config)?;
-				run_until_exit(self.tokio_runtime, cmd.run(client, db_config), task_manager)
-			}
-			Subcommand::ImportBlocks(cmd) => {
-				let (client, _, import_queue, task_manager) = builder(self.config)?;
-				run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager)
-			}
-			Subcommand::CheckBlock(cmd) => {
-				let (client, _, import_queue, task_manager) = builder(self.config)?;
-				run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager)
-			}
-			Subcommand::Revert(cmd) => {
-				let (client, backend, _, task_manager) = builder(self.config)?;
-				run_until_exit(self.tokio_runtime, cmd.run(client, backend), task_manager)
-			},
-			Subcommand::PurgeChain(cmd) => cmd.run(db_config),
-			Subcommand::ExportState(cmd) => {
-				let (client, _, _, task_manager) = builder(self.config)?;
-				run_until_exit(self.tokio_runtime, cmd.run(client, chain_spec), task_manager)
-			},
-		}
-	}
-
 	/// A helper function that runs a node with tokio and stops if the process receives the signal
 	/// `SIGTERM` or `SIGINT`.
 	pub fn run_node_until_exit(

From 8502300e246e386b1c2e58a725a0e37e3f75df3f Mon Sep 17 00:00:00 2001
From: gabriel klawitter <gabreal@users.noreply.github.com>
Date: Tue, 1 Sep 2020 12:48:03 +0200
Subject: [PATCH 035/122] ci: deploy alerting rules: fix run on changes (#6998)

* ci: deploy alerting rules: fix run on changes

Co-authored-by: Max Inden <mail@max-inden.de>
---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c97d68bab0030..d20a65b4df54d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -734,7 +734,8 @@ deploy-kubernetes-alerting-rules:
     refs:
       - master
     changes:
-      - "${RULES}"
+      - .gitlab-ci.yml
+      - .maintain/monitoring/
 
 
 

From f7234584544e90ce0fa04f9b75808b99e829684d Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Tue, 1 Sep 2020 13:23:10 +0200
Subject: [PATCH 036/122] *: Update to Prometheus v0.10.0 (#6964)

* *: Update to Prometheus v0.10.0-rc.1

* *: Update to Prometheus v0.10.0
---
 Cargo.lock                  | 65 +++++++++++++++++++++++++++++++++----
 primitives/utils/Cargo.toml |  2 +-
 utils/prometheus/Cargo.toml |  2 +-
 3 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 136b2c81191e0..fbf7f307962b5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -776,6 +776,15 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "cloudabi"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.44"
@@ -2474,6 +2483,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485"
+
 [[package]]
 name = "integer-sqrt"
 version = "0.1.3"
@@ -3346,6 +3361,15 @@ dependencies = [
  "scopeguard 1.1.0",
 ]
 
+[[package]]
+name = "lock_api"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
+dependencies = [
+ "scopeguard 1.1.0",
+]
+
 [[package]]
 name = "log"
 version = "0.4.11"
@@ -5277,6 +5301,17 @@ dependencies = [
  "parking_lot_core 0.7.2",
 ]
 
+[[package]]
+name = "parking_lot"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733"
+dependencies = [
+ "instant",
+ "lock_api 0.4.1",
+ "parking_lot_core 0.8.0",
+]
+
 [[package]]
 name = "parking_lot_core"
 version = "0.4.0"
@@ -5297,7 +5332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
 dependencies = [
  "cfg-if",
- "cloudabi",
+ "cloudabi 0.0.3",
  "libc",
  "redox_syscall",
  "rustc_version",
@@ -5312,7 +5347,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
 dependencies = [
  "cfg-if",
- "cloudabi",
+ "cloudabi 0.0.3",
+ "libc",
+ "redox_syscall",
+ "smallvec 1.4.1",
+ "winapi 0.3.9",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
+dependencies = [
+ "cfg-if",
+ "cloudabi 0.1.0",
+ "instant",
  "libc",
  "redox_syscall",
  "smallvec 1.4.1",
@@ -5573,14 +5623,15 @@ dependencies = [
 
 [[package]]
 name = "prometheus"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd0ced56dee39a6e960c15c74dc48849d614586db2eaada6497477af7c7811cd"
+checksum = "30d70cf4412832bcac9cffe27906f4a66e450d323525e977168c70d1b36120ae"
 dependencies = [
  "cfg-if",
  "fnv",
  "lazy_static",
- "spin",
+ "parking_lot 0.11.0",
+ "regex",
  "thiserror",
 ]
 
@@ -5719,7 +5770,7 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9"
 dependencies = [
- "cloudabi",
+ "cloudabi 0.0.3",
  "fuchsia-cprng",
  "libc",
  "rand_core 0.3.1",
@@ -5847,7 +5898,7 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
 dependencies = [
- "cloudabi",
+ "cloudabi 0.0.3",
  "fuchsia-cprng",
  "libc",
  "rand_core 0.4.2",
diff --git a/primitives/utils/Cargo.toml b/primitives/utils/Cargo.toml
index b21dba40a9d9b..e19350a9eb827 100644
--- a/primitives/utils/Cargo.toml
+++ b/primitives/utils/Cargo.toml
@@ -12,7 +12,7 @@ description = "I/O for Substrate runtimes"
 futures = "0.3.4"
 futures-core = "0.3.4"
 lazy_static = "1.4.0"
-prometheus = { version = "0.9.0", default-features = false }
+prometheus = { version = "0.10.0", default-features = false }
 futures-timer = "3.0.2"
 
 [features]
diff --git a/utils/prometheus/Cargo.toml b/utils/prometheus/Cargo.toml
index 4ed4575ccf709..712aaa68dfed3 100644
--- a/utils/prometheus/Cargo.toml
+++ b/utils/prometheus/Cargo.toml
@@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 log = "0.4.8"
-prometheus = { version = "0.9", default-features = false }
+prometheus = { version = "0.10.0", default-features = false }
 futures-util = { version = "0.3.1", default-features = false, features = ["io"] }
 derive_more = "0.99"
 

From 0f302b97ca50c4a415d72504e7142769ecaa14a1 Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Wed, 2 Sep 2020 16:30:41 +0200
Subject: [PATCH 037/122] Ensure that handshake is sent back even in case of
 back-pressure (#6979)

* Ensure that handshake is sent back even in case of back-pressure

* Update client/network/src/protocol/generic_proto/handler/group.rs

Co-authored-by: Max Inden <mail@max-inden.de>

* Also process OpenRequest and Closed

* Fix bad merge

* God I'm so lost with all these merges

* Immediately return Closed

Co-authored-by: Max Inden <mail@max-inden.de>
---
 .../protocol/generic_proto/handler/group.rs   | 66 +++++++++++--------
 .../generic_proto/handler/notif_in.rs         | 27 ++++++++
 .../generic_proto/upgrade/notifications.rs    | 46 ++++++++++++-
 3 files changed, 110 insertions(+), 29 deletions(-)

diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs
index 43627f3d6041c..6804dd3c789da 100644
--- a/client/network/src/protocol/generic_proto/handler/group.rs
+++ b/client/network/src/protocol/generic_proto/handler/group.rs
@@ -674,36 +674,48 @@ impl ProtocolsHandler for NotifsHandler {
 						return Poll::Ready(ProtocolsHandlerEvent::Close(NotifsHandlerError::Legacy(err))),
 				}
 			}
+		}
+
+		for (handler_num, (handler, handshake_message)) in self.in_handlers.iter_mut().enumerate() {
+			loop {
+				let poll = if self.pending_legacy_handshake.is_none() {
+					handler.poll(cx)
+				} else {
+					handler.poll_process(cx)
+				};
 
-			for (handler_num, (handler, handshake_message)) in self.in_handlers.iter_mut().enumerate() {
-				while let Poll::Ready(ev) = handler.poll(cx) {
-					match ev {
-						ProtocolsHandlerEvent::OutboundSubstreamRequest { .. } =>
-							error!("Incoming substream handler tried to open a substream"),
-						ProtocolsHandlerEvent::Close(err) => void::unreachable(err),
-						ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(_)) =>
-							match self.enabled {
-								EnabledState::Initial => self.pending_in.push(handler_num),
-								EnabledState::Enabled => {
-									// We create `handshake_message` on a separate line to be sure
-									// that the lock is released as soon as possible.
-									let handshake_message = handshake_message.read().clone();
-									handler.inject_event(NotifsInHandlerIn::Accept(handshake_message))
-								},
-								EnabledState::Disabled =>
-									handler.inject_event(NotifsInHandlerIn::Refuse),
+				let ev = match poll {
+					Poll::Ready(e) => e,
+					Poll::Pending => break,
+				};
+
+				match ev {
+					ProtocolsHandlerEvent::OutboundSubstreamRequest { .. } =>
+						error!("Incoming substream handler tried to open a substream"),
+					ProtocolsHandlerEvent::Close(err) => void::unreachable(err),
+					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(_)) =>
+						match self.enabled {
+							EnabledState::Initial => self.pending_in.push(handler_num),
+							EnabledState::Enabled => {
+								// We create `handshake_message` on a separate line to be sure
+								// that the lock is released as soon as possible.
+								let handshake_message = handshake_message.read().clone();
+								handler.inject_event(NotifsInHandlerIn::Accept(handshake_message))
 							},
-						ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed) => {},
-						ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(message)) => {
-							if self.notifications_sink_rx.is_some() {
-								let msg = NotifsHandlerOut::Notification {
-									message,
-									protocol_name: handler.protocol_name().clone(),
-								};
-								return Poll::Ready(ProtocolsHandlerEvent::Custom(msg));
-							}
+							EnabledState::Disabled =>
+								handler.inject_event(NotifsInHandlerIn::Refuse),
 						},
-					}
+					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed) => {},
+					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(message)) => {
+						debug_assert!(self.pending_legacy_handshake.is_none());
+						if self.notifications_sink_rx.is_some() {
+							let msg = NotifsHandlerOut::Notification {
+								message,
+								protocol_name: handler.protocol_name().clone(),
+							};
+							return Poll::Ready(ProtocolsHandlerEvent::Custom(msg));
+						}
+					},
 				}
 			}
 		}
diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs
index 9eb8ec7471610..5a50cce268117 100644
--- a/client/network/src/protocol/generic_proto/handler/notif_in.rs
+++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs
@@ -139,6 +139,33 @@ impl NotifsInHandler {
 	pub fn protocol_name(&self) -> &Cow<'static, str> {
 		self.in_protocol.protocol_name()
 	}
+
+	/// Equivalent to the `poll` method of `ProtocolsHandler`, except that it is guaranteed to
+	/// never generate [`NotifsInHandlerOut::Notif`].
+	///
+	/// Use this method in situations where it is not desirable to receive events but still
+	/// necessary to drive any potential incoming handshake or request.
+	pub fn poll_process(
+		&mut self,
+		cx: &mut Context
+	) -> Poll<
+		ProtocolsHandlerEvent<DeniedUpgrade, (), NotifsInHandlerOut, void::Void>
+	> {
+		if let Some(event) = self.events_queue.pop_front() {
+			return Poll::Ready(event)
+		}
+
+		match self.substream.as_mut().map(|s| NotificationsInSubstream::poll_process(Pin::new(s), cx)) {
+			None | Some(Poll::Pending) => {},
+			Some(Poll::Ready(Ok(v))) => match v {},
+			Some(Poll::Ready(Err(_))) => {
+				self.substream = None;
+				return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed));
+			},
+		}
+
+		Poll::Pending
+	}
 }
 
 impl ProtocolsHandler for NotifsInHandler {
diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs
index 51fbc8d9c60ba..64b4b980da002 100644
--- a/client/network/src/protocol/generic_proto/upgrade/notifications.rs
+++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs
@@ -39,7 +39,7 @@ use futures::prelude::*;
 use futures_codec::Framed;
 use libp2p::core::{UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade};
 use log::error;
-use std::{borrow::Cow, io, iter, mem, pin::Pin, task::{Context, Poll}};
+use std::{borrow::Cow, convert::Infallible, io, iter, mem, pin::Pin, task::{Context, Poll}};
 use unsigned_varint::codec::UviBytes;
 
 /// Maximum allowed size of the two handshake messages, in bytes.
@@ -162,7 +162,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
 }
 
 impl<TSubstream> NotificationsInSubstream<TSubstream>
-where TSubstream: AsyncRead + AsyncWrite,
+where TSubstream: AsyncRead + AsyncWrite + Unpin,
 {
 	/// Sends the handshake in order to inform the remote that we accept the substream.
 	pub fn send_handshake(&mut self, message: impl Into<Vec<u8>>) {
@@ -173,6 +173,48 @@ where TSubstream: AsyncRead + AsyncWrite,
 
 		self.handshake = NotificationsInSubstreamHandshake::PendingSend(message.into());
 	}
+
+	/// Equivalent to `Stream::poll_next`, except that it only drives the handshake and is
+	/// guaranteed to not generate any notification.
+	pub fn poll_process(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<Infallible, io::Error>> {
+		let mut this = self.project();
+
+		loop {
+			match mem::replace(this.handshake, NotificationsInSubstreamHandshake::Sent) {
+				NotificationsInSubstreamHandshake::PendingSend(msg) =>
+					match Sink::poll_ready(this.socket.as_mut(), cx) {
+						Poll::Ready(_) => {
+							*this.handshake = NotificationsInSubstreamHandshake::Flush;
+							match Sink::start_send(this.socket.as_mut(), io::Cursor::new(msg)) {
+								Ok(()) => {},
+								Err(err) => return Poll::Ready(Err(err)),
+							}
+						},
+						Poll::Pending => {
+							*this.handshake = NotificationsInSubstreamHandshake::PendingSend(msg);
+							return Poll::Pending
+						}
+					},
+				NotificationsInSubstreamHandshake::Flush =>
+					match Sink::poll_flush(this.socket.as_mut(), cx)? {
+						Poll::Ready(()) =>
+							*this.handshake = NotificationsInSubstreamHandshake::Sent,
+						Poll::Pending => {
+							*this.handshake = NotificationsInSubstreamHandshake::Flush;
+							return Poll::Pending
+						}
+					},
+
+				st @ NotificationsInSubstreamHandshake::NotSent |
+				st @ NotificationsInSubstreamHandshake::Sent |
+				st @ NotificationsInSubstreamHandshake::ClosingInResponseToRemote |
+				st @ NotificationsInSubstreamHandshake::BothSidesClosed => {
+					*this.handshake = st;
+					return Poll::Pending;
+				}
+			}
+		}
+	}
 }
 
 impl<TSubstream> Stream for NotificationsInSubstream<TSubstream>

From 3541fa8064fac984df7d225425e86e42fd3c238d Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Wed, 2 Sep 2020 17:20:51 +0200
Subject: [PATCH 038/122] frame/authority-discovery: Have authorities() return
 both current and next (#6788)

* frame/authority-discovery: Have authorities() return both current and next

Authority address lookups on the DHT happen periodically (every 10
mintues) and are rather slow (~10 seconds).

In order to smooth the transition period between two sessions, have the
runtime module return both the current as well as the next authority
set. Thereby the client authority module will:

1. Publish its addresses one session in advance.

2. Prefetch the addresses of authorities of the next session in advance.

* frame/authority-discovery: Deduplicate authority ids

* frame/authority-discovery: Don't dedup on_genesis authorities

* frame/authority-discovery: Remove mut and sort on comparison in tests

* frame/authority-discovery: Use BTreeSet for deduplication
---
 client/authority-discovery/src/worker.rs  | 16 ++---
 frame/authority-discovery/src/lib.rs      | 80 +++++++++++++++++------
 primitives/authority-discovery/src/lib.rs |  4 +-
 3 files changed, 71 insertions(+), 29 deletions(-)

diff --git a/client/authority-discovery/src/worker.rs b/client/authority-discovery/src/worker.rs
index 232e59d08dd78..629ea4fb2f423 100644
--- a/client/authority-discovery/src/worker.rs
+++ b/client/authority-discovery/src/worker.rs
@@ -99,7 +99,7 @@ pub enum Role {
 ///
 /// 2. **Discovers other authorities**
 ///
-///    1. Retrieves the current set of authorities.
+///    1. Retrieves the current and next set of authorities.
 ///
 ///    2. Starts DHT queries for the ids of the authorities.
 ///
@@ -447,7 +447,7 @@ where
 				.collect::<HashMap<_, _>>()
 		};
 
-		// Check if the event origins from an authority in the current authority set.
+		// Check if the event origins from an authority in the current or next authority set.
 		let authority_id: &AuthorityId = authorities
 			.get(&remote_key)
 			.ok_or(Error::MatchingHashedAuthorityIdWithAuthorityId)?;
@@ -514,12 +514,12 @@ where
 		Ok(())
 	}
 
-	/// Retrieve our public keys within the current authority set.
+	/// Retrieve our public keys within the current and next authority set.
 	//
 	// A node might have multiple authority discovery keys within its keystore, e.g. an old one and
-	// one for the upcoming session. In addition it could be participating in the current authority
-	// set with two keys. The function does not return all of the local authority discovery public
-	// keys, but only the ones intersecting with the current authority set.
+	// one for the upcoming session. In addition it could be participating in the current and (/ or)
+	// next authority set with two keys. The function does not return all of the local authority
+	// discovery public keys, but only the ones intersecting with the current or next authority set.
 	fn get_own_public_keys_within_authority_set(
 		key_store: &BareCryptoStorePtr,
 		client: &Client,
@@ -530,14 +530,14 @@ where
 			.collect::<HashSet<_>>();
 
 		let id = BlockId::hash(client.info().best_hash);
-		let current_authorities = client.runtime_api()
+		let authorities = client.runtime_api()
 			.authorities(&id)
 			.map_err(Error::CallingRuntime)?
 			.into_iter()
 			.map(std::convert::Into::into)
 			.collect::<HashSet<_>>();
 
-		let intersection = local_pub_keys.intersection(&current_authorities)
+		let intersection = local_pub_keys.intersection(&authorities)
 			.cloned()
 			.map(std::convert::Into::into)
 			.collect();
diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs
index 55e32b21dcb94..d584838ecbedd 100644
--- a/frame/authority-discovery/src/lib.rs
+++ b/frame/authority-discovery/src/lib.rs
@@ -23,7 +23,7 @@
 // Ensure we're `no_std` when compiling for Wasm.
 #![cfg_attr(not(feature = "std"), no_std)]
 
-use sp_std::prelude::*;
+use sp_std::{collections::btree_set::BTreeSet, prelude::*};
 use frame_support::{decl_module, decl_storage};
 use sp_authority_discovery::AuthorityId;
 
@@ -32,7 +32,7 @@ pub trait Trait: frame_system::Trait + pallet_session::Trait {}
 
 decl_storage! {
 	trait Store for Module<T: Trait> as AuthorityDiscovery {
-		/// Keys of the current authority set.
+		/// Keys of the current and next authority set.
 		Keys get(fn keys): Vec<AuthorityId>;
 	}
 	add_extra_genesis {
@@ -47,7 +47,7 @@ decl_module! {
 }
 
 impl<T: Trait> Module<T> {
-	/// Retrieve authority identifiers of the current authority set.
+	/// Retrieve authority identifiers of the current and next authority set.
 	pub fn authorities() -> Vec<AuthorityId> {
 		Keys::get()
 	}
@@ -71,17 +71,17 @@ impl<T: Trait> pallet_session::OneSessionHandler<T::AccountId> for Module<T> {
 	where
 		I: Iterator<Item = (&'a T::AccountId, Self::Key)>,
 	{
-		let keys = authorities.map(|x| x.1).collect::<Vec<_>>();
-		Self::initialize_keys(&keys);
+		Self::initialize_keys(&authorities.map(|x| x.1).collect::<Vec<_>>());
 	}
 
-	fn on_new_session<'a, I: 'a>(changed: bool, validators: I, _queued_validators: I)
+	fn on_new_session<'a, I: 'a>(changed: bool, validators: I, queued_validators: I)
 	where
 		I: Iterator<Item = (&'a T::AccountId, Self::Key)>,
 	{
-		// Remember who the authorities are for the new session.
+		// Remember who the authorities are for the new and next session.
 		if changed {
-			Keys::put(validators.map(|x| x.1).collect::<Vec<_>>());
+			let keys = validators.chain(queued_validators).map(|x| x.1).collect::<BTreeSet<_>>();
+			Keys::put(keys.into_iter().collect::<Vec<_>>());
 		}
 	}
 
@@ -192,12 +192,13 @@ mod tests {
 	}
 
 	#[test]
-	fn authorities_returns_current_authority_set() {
-		// The whole authority discovery module ignores account ids, but we still need it for
-		// `pallet_session::OneSessionHandler::on_new_session`, thus its safe to use the same value everywhere.
+	fn authorities_returns_current_and_next_authority_set() {
+		// The whole authority discovery module ignores account ids, but we still need them for
+		// `pallet_session::OneSessionHandler::on_new_session`, thus its safe to use the same value
+		// everywhere.
 		let account_id = AuthorityPair::from_seed_slice(vec![10; 32].as_ref()).unwrap().public();
 
-		let first_authorities: Vec<AuthorityId> = vec![0, 1].into_iter()
+		let mut first_authorities: Vec<AuthorityId> = vec![0, 1].into_iter()
 			.map(|i| AuthorityPair::from_seed_slice(vec![i; 32].as_ref()).unwrap().public())
 			.map(AuthorityId::from)
 			.collect();
@@ -206,12 +207,21 @@ mod tests {
 			.map(|i| AuthorityPair::from_seed_slice(vec![i; 32].as_ref()).unwrap().public())
 			.map(AuthorityId::from)
 			.collect();
-
 		// Needed for `pallet_session::OneSessionHandler::on_new_session`.
-		let second_authorities_and_account_ids: Vec<(&AuthorityId, AuthorityId)> = second_authorities.clone()
+		let second_authorities_and_account_ids = second_authorities.clone()
 			.into_iter()
 			.map(|id| (&account_id, id))
+			.collect::<Vec<(&AuthorityId, AuthorityId)> >();
+
+		let mut third_authorities: Vec<AuthorityId> = vec![4, 5].into_iter()
+			.map(|i| AuthorityPair::from_seed_slice(vec![i; 32].as_ref()).unwrap().public())
+			.map(AuthorityId::from)
 			.collect();
+		// Needed for `pallet_session::OneSessionHandler::on_new_session`.
+		let third_authorities_and_account_ids = third_authorities.clone()
+			.into_iter()
+			.map(|id| (&account_id, id))
+			.collect::<Vec<(&AuthorityId, AuthorityId)> >();
 
 		// Build genesis.
 		let mut t = frame_system::GenesisConfig::default()
@@ -233,23 +243,55 @@ mod tests {
 			AuthorityDiscovery::on_genesis_session(
 				first_authorities.iter().map(|id| (id, id.clone()))
 			);
-			assert_eq!(first_authorities, AuthorityDiscovery::authorities());
+			first_authorities.sort();
+			let mut authorities_returned = AuthorityDiscovery::authorities();
+			authorities_returned.sort();
+			assert_eq!(first_authorities, authorities_returned);
 
 			// When `changed` set to false, the authority set should not be updated.
 			AuthorityDiscovery::on_new_session(
 				false,
 				second_authorities_and_account_ids.clone().into_iter(),
-				vec![].into_iter(),
+				third_authorities_and_account_ids.clone().into_iter(),
+			);
+			let mut authorities_returned = AuthorityDiscovery::authorities();
+			authorities_returned.sort();
+			assert_eq!(
+				first_authorities,
+				authorities_returned,
+				"Expected authority set not to change as `changed` was set to false.",
 			);
-			assert_eq!(first_authorities, AuthorityDiscovery::authorities());
 
 			// When `changed` set to true, the authority set should be updated.
 			AuthorityDiscovery::on_new_session(
 				true,
 				second_authorities_and_account_ids.into_iter(),
-				vec![].into_iter(),
+				third_authorities_and_account_ids.clone().into_iter(),
+			);
+			let mut second_and_third_authorities = second_authorities.iter()
+				.chain(third_authorities.iter())
+				.cloned()
+				.collect::<Vec<AuthorityId>>();
+			second_and_third_authorities.sort();
+			assert_eq!(
+				second_and_third_authorities,
+				AuthorityDiscovery::authorities(),
+				"Expected authority set to contain both the authorities of the new as well as the \
+				 next session."
+			);
+
+			// With overlapping authority sets, `authorities()` should return a deduplicated set.
+			AuthorityDiscovery::on_new_session(
+				true,
+				third_authorities_and_account_ids.clone().into_iter(),
+				third_authorities_and_account_ids.clone().into_iter(),
+			);
+			third_authorities.sort();
+			assert_eq!(
+				third_authorities,
+				AuthorityDiscovery::authorities(),
+				"Expected authority set to be deduplicated."
 			);
-			assert_eq!(second_authorities, AuthorityDiscovery::authorities());
 		});
 	}
 }
diff --git a/primitives/authority-discovery/src/lib.rs b/primitives/authority-discovery/src/lib.rs
index 8903a7f383755..0ae47c9758ee6 100644
--- a/primitives/authority-discovery/src/lib.rs
+++ b/primitives/authority-discovery/src/lib.rs
@@ -45,9 +45,9 @@ sp_api::decl_runtime_apis! {
 	/// The authority discovery api.
 	///
 	/// This api is used by the `client/authority-discovery` module to retrieve identifiers
-	/// of the current authority set.
+	/// of the current and next authority set.
 	pub trait AuthorityDiscoveryApi {
-		/// Retrieve authority identifiers of the current authority set.
+		/// Retrieve authority identifiers of the current and next authority set.
 		fn authorities() -> Vec<AuthorityId>;
 	}
 }

From f8189fcd0b0aac63a8f54177c92d0b0e6ad7b252 Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Wed, 2 Sep 2020 17:28:03 +0200
Subject: [PATCH 039/122] Stop sending messages on legacy substream altogether
 (#6975)

* Stop sending messages on legacy substream altogether

* Ensure that handshake is sent back even in case of back-pressure

* Update client/network/src/protocol/generic_proto/handler/group.rs

Co-authored-by: Max Inden <mail@max-inden.de>

* Also process OpenRequest and Closed

* Also process OpenRequest and Closed

* Fix bad merge

* God I'm so lost with all these merges

* Immediately return Closed

* Add warning for sending on non-registered protocol

* Register GrandPa protocol in tests

* Update client/network/src/protocol/generic_proto/handler/group.rs

Co-authored-by: Max Inden <mail@max-inden.de>

Co-authored-by: Max Inden <mail@max-inden.de>
---
 client/finality-grandpa/src/tests.rs          |  11 +-
 client/network/src/protocol.rs                | 176 ++----------------
 .../src/protocol/generic_proto/behaviour.rs   |  24 ---
 .../protocol/generic_proto/handler/group.rs   |  68 +++----
 .../protocol/generic_proto/handler/legacy.rs  |  20 --
 .../src/protocol/generic_proto/tests.rs       | 150 +--------------
 .../protocol/generic_proto/upgrade/legacy.rs  |   9 -
 client/network/src/service.rs                 |  35 +---
 client/network/test/src/lib.rs                |  10 +-
 9 files changed, 70 insertions(+), 433 deletions(-)

diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs
index d2905e4da4453..6e8def57f50ea 100644
--- a/client/finality-grandpa/src/tests.rs
+++ b/client/finality-grandpa/src/tests.rs
@@ -23,7 +23,7 @@ use assert_matches::assert_matches;
 use environment::HasVoted;
 use sc_network_test::{
 	Block, BlockImportAdapter, Hash, PassThroughVerifier, Peer, PeersClient, PeersFullClient,
-	TestClient, TestNetFactory,
+	TestClient, TestNetFactory, FullPeerConfig,
 };
 use sc_network::config::{ProtocolConfig, BoxFinalityProofRequestBuilder};
 use parking_lot::Mutex;
@@ -94,6 +94,15 @@ impl TestNetFactory for GrandpaTestNet {
 		ProtocolConfig::default()
 	}
 
+	fn add_full_peer(&mut self) {
+		self.add_full_peer_with_config(FullPeerConfig {
+			notifications_protocols: vec![
+				(communication::GRANDPA_ENGINE_ID, communication::GRANDPA_PROTOCOL_NAME.into())
+			],
+			..Default::default()
+		})
+	}
+
 	fn make_verifier(
 		&self,
 		_client: PeersClient,
diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 4d651bb64e9a9..08de0691b3c94 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -39,13 +39,13 @@ use sp_consensus::{
 use codec::{Decode, Encode};
 use sp_runtime::{generic::BlockId, ConsensusEngineId, Justification};
 use sp_runtime::traits::{
-	Block as BlockT, Header as HeaderT, NumberFor, One, Zero, CheckedSub
+	Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub
 };
 use sp_arithmetic::traits::SaturatedConversion;
 use message::{BlockAnnounce, Message};
 use message::generic::{Message as GenericMessage, Roles};
 use prometheus_endpoint::{
-	Registry, Gauge, Counter, CounterVec, GaugeVec,
+	Registry, Gauge, Counter, GaugeVec,
 	PrometheusError, Opts, register, U64
 };
 use sync::{ChainSync, SyncState};
@@ -53,7 +53,7 @@ use std::borrow::Cow;
 use std::collections::{HashMap, HashSet, VecDeque, hash_map::Entry};
 use std::sync::Arc;
 use std::fmt::Write;
-use std::{cmp, io, num::NonZeroUsize, pin::Pin, task::Poll, time};
+use std::{io, num::NonZeroUsize, pin::Pin, task::Poll, time};
 use log::{log, Level, trace, debug, warn, error};
 use wasm_timer::Instant;
 
@@ -86,11 +86,6 @@ pub(crate) const CURRENT_VERSION: u32 = 6;
 /// Lowest version we support
 pub(crate) const MIN_VERSION: u32 = 3;
 
-// Maximum allowed entries in `BlockResponse`
-const MAX_BLOCK_DATA_RESPONSE: u32 = 128;
-// Maximum total bytes allowed for block bodies in `BlockResponse`
-const MAX_BODIES_BYTES: usize = 8 * 1024 * 1024;
-
 /// When light node connects to the full node and the full node is behind light node
 /// for at least `LIGHT_MAXIMAL_BLOCKS_DIFFERENCE` blocks, we consider it not useful
 /// and disconnect to free connection slot.
@@ -119,8 +114,6 @@ mod rep {
 	pub const UNEXPECTED_RESPONSE: Rep = Rep::new_fatal("Unexpected response packet");
 	/// We received an unexpected transaction packet.
 	pub const UNEXPECTED_TRANSACTIONS: Rep = Rep::new_fatal("Unexpected transactions packet");
-	/// We received an unexpected light node request.
-	pub const UNEXPECTED_REQUEST: Rep = Rep::new_fatal("Unexpected block request packet");
 	/// Peer has different genesis.
 	pub const GENESIS_MISMATCH: Rep = Rep::new_fatal("Genesis mismatch");
 	/// Peer is on unsupported protocol version.
@@ -139,7 +132,6 @@ struct Metrics {
 	finality_proofs: GaugeVec<U64>,
 	justifications: GaugeVec<U64>,
 	propagated_transactions: Counter<U64>,
-	legacy_requests_received: CounterVec<U64>,
 }
 
 impl Metrics {
@@ -185,13 +177,6 @@ impl Metrics {
 				"sync_propagated_transactions",
 				"Number of transactions propagated to at least one peer",
 			)?, r)?,
-			legacy_requests_received: register(CounterVec::new(
-				Opts::new(
-					"sync_legacy_requests_received",
-					"Number of block/finality/light-client requests received on the legacy substream",
-				),
-				&["kind"]
-			)?, r)?,
 		})
 	}
 }
@@ -616,17 +601,13 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		match message {
 			GenericMessage::Status(_) =>
 				debug!(target: "sub-libp2p", "Received unexpected Status"),
-			GenericMessage::BlockRequest(r) => self.on_block_request(who, r),
-			GenericMessage::BlockResponse(r) => {
-				let outcome = self.on_block_response(who.clone(), r);
-				self.update_peer_info(&who);
-				return outcome
-			},
 			GenericMessage::BlockAnnounce(announce) => {
 				self.pre_validate_block_announce(who.clone(), announce);
 			},
 			GenericMessage::Transactions(m) =>
 				self.on_transactions(who, m),
+			GenericMessage::BlockResponse(_) =>
+				warn!(target: "sub-libp2p", "Received unexpected BlockResponse"),
 			GenericMessage::RemoteCallResponse(_) =>
 				warn!(target: "sub-libp2p", "Received unexpected RemoteCallResponse"),
 			GenericMessage::RemoteReadResponse(_) =>
@@ -637,6 +618,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 				warn!(target: "sub-libp2p", "Received unexpected RemoteChangesResponse"),
 			GenericMessage::FinalityProofResponse(_) =>
 				warn!(target: "sub-libp2p", "Received unexpected FinalityProofResponse"),
+			GenericMessage::BlockRequest(_) |
 			GenericMessage::FinalityProofRequest(_) |
 			GenericMessage::RemoteReadChildRequest(_) |
 			GenericMessage::RemoteCallRequest(_) |
@@ -688,21 +670,6 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		CustomMessageOutcome::None
 	}
 
-	fn send_message(
-		&mut self,
-		who: &PeerId,
-		message: Option<(Cow<'static, str>, Vec<u8>)>,
-		legacy: Message<B>,
-	) {
-		send_message::<B>(
-			&mut self.behaviour,
-			&mut self.context_data.stats,
-			who,
-			message,
-			legacy,
-		);
-	}
-
 	fn update_peer_request(&mut self, who: &PeerId, request: &mut message::BlockRequest<B>) {
 		update_peer_request::<B, H>(&mut self.context_data.peers, who, request)
 	}
@@ -728,92 +695,6 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		}
 	}
 
-	fn on_block_request(&mut self, peer: PeerId, request: message::BlockRequest<B>) {
-		if let Some(metrics) = &self.metrics {
-			metrics.legacy_requests_received.with_label_values(&["block-request"]).inc();
-		}
-
-		trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?} for {:?}",
-			request.id,
-			peer,
-			request.from,
-			request.to,
-			request.max,
-			request.fields,
-		);
-
-		// sending block requests to the node that is unable to serve it is considered a bad behavior
-		if !self.config.roles.is_full() {
-			trace!(target: "sync", "Peer {} is trying to sync from the light node", peer);
-			self.behaviour.disconnect_peer(&peer);
-			self.peerset_handle.report_peer(peer, rep::UNEXPECTED_REQUEST);
-			return;
-		}
-
-		let mut blocks = Vec::new();
-		let mut id = match request.from {
-			message::FromBlock::Hash(h) => BlockId::Hash(h),
-			message::FromBlock::Number(n) => BlockId::Number(n),
-		};
-		let max = cmp::min(request.max.unwrap_or(u32::max_value()), MAX_BLOCK_DATA_RESPONSE) as usize;
-		let get_header = request.fields.contains(message::BlockAttributes::HEADER);
-		let get_body = request.fields.contains(message::BlockAttributes::BODY);
-		let get_justification = request
-			.fields
-			.contains(message::BlockAttributes::JUSTIFICATION);
-		let mut total_size = 0;
-		while let Some(header) = self.context_data.chain.header(id).unwrap_or(None) {
-			if blocks.len() >= max || (blocks.len() >= 1 && total_size > MAX_BODIES_BYTES) {
-				break;
-			}
-			let number = *header.number();
-			let hash = header.hash();
-			let parent_hash = *header.parent_hash();
-			let justification = if get_justification {
-				self.context_data.chain.justification(&BlockId::Hash(hash)).unwrap_or(None)
-			} else {
-				None
-			};
-			let block_data = message::generic::BlockData {
-				hash,
-				header: if get_header { Some(header) } else { None },
-				body: if get_body {
-					self.context_data
-						.chain
-						.block_body(&BlockId::Hash(hash))
-						.unwrap_or(None)
-				} else {
-					None
-				},
-				receipt: None,
-				message_queue: None,
-				justification,
-			};
-			// Stop if we don't have requested block body
-			if get_body && block_data.body.is_none() {
-				trace!(target: "sync", "Missing data for block request.");
-				break;
-			}
-			total_size += block_data.body.as_ref().map_or(0, |b| b.len());
-			blocks.push(block_data);
-			match request.direction {
-				message::Direction::Ascending => id = BlockId::Number(number + One::one()),
-				message::Direction::Descending => {
-					if number.is_zero() {
-						break;
-					}
-					id = BlockId::Hash(parent_hash)
-				}
-			}
-		}
-		let response = message::generic::BlockResponse {
-			id: request.id,
-			blocks,
-		};
-		trace!(target: "sync", "Sending BlockResponse with {} blocks", response.blocks.len());
-		self.send_message(&peer, None, GenericMessage::BlockResponse(response))
-	}
-
 	/// Adjusts the reputation of a node.
 	pub fn report_peer(&self, who: PeerId, reputation: sc_peerset::ReputationChange) {
 		self.peerset_handle.report_peer(who, reputation)
@@ -1217,14 +1098,11 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 						.push(who.to_base58());
 				}
 				trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who);
-				let encoded = to_send.encode();
-				send_message::<B> (
-					&mut self.behaviour,
-					&mut self.context_data.stats,
-					&who,
-					Some((self.transactions_protocol.clone(), encoded)),
-					GenericMessage::Transactions(to_send)
-				)
+				self.behaviour.write_notification(
+					who,
+					self.transactions_protocol.clone(),
+					to_send.encode()
+				);
 			}
 		}
 
@@ -1299,15 +1177,11 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 					},
 				};
 
-				let encoded = message.encode();
-
-				send_message::<B> (
-					&mut self.behaviour,
-					&mut self.context_data.stats,
-					&who,
-					Some((self.block_announces_protocol.clone(), encoded)),
-					Message::<B>::BlockAnnounce(message),
-				)
+				self.behaviour.write_notification(
+					who,
+					self.block_announces_protocol.clone(),
+					message.encode()
+				);
 			}
 		}
 	}
@@ -1637,24 +1511,6 @@ fn update_peer_request<B: BlockT, H: ExHashT>(
 	}
 }
 
-fn send_message<B: BlockT>(
-	behaviour: &mut GenericProto,
-	stats: &mut HashMap<&'static str, PacketStats>,
-	who: &PeerId,
-	message: Option<(Cow<'static, str>, Vec<u8>)>,
-	legacy_message: Message<B>,
-) {
-	let encoded = legacy_message.encode();
-	let mut stats = stats.entry(legacy_message.id()).or_default();
-	stats.bytes_out += encoded.len() as u64;
-	stats.count_out += 1;
-	if let Some((proto, msg)) = message {
-		behaviour.write_notification(who, proto, msg, encoded);
-	} else {
-		behaviour.send_packet(who, encoded);
-	}
-}
-
 impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 	type ProtocolsHandler = <GenericProto as NetworkBehaviour>::ProtocolsHandler;
 	type OutEvent = CustomMessageOutcome<B>;
diff --git a/client/network/src/protocol/generic_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs
index 56a5b3fb0ab2d..996a810605d13 100644
--- a/client/network/src/protocol/generic_proto/behaviour.rs
+++ b/client/network/src/protocol/generic_proto/behaviour.rs
@@ -553,7 +553,6 @@ impl GenericProto {
 		target: &PeerId,
 		protocol_name: Cow<'static, str>,
 		message: impl Into<Vec<u8>>,
-		encoded_fallback_message: Vec<u8>,
 	) {
 		let notifs_sink = match self.peers.get(target).and_then(|p| p.get_open()) {
 			None => {
@@ -574,33 +573,10 @@ impl GenericProto {
 		trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target);
 		notifs_sink.send_sync_notification(
 			protocol_name,
-			encoded_fallback_message,
 			message
 		);
 	}
 
-	/// Sends a message to a peer.
-	///
-	/// Has no effect if the custom protocol is not open with the given peer.
-	///
-	/// Also note that even we have a valid open substream, it may in fact be already closed
-	/// without us knowing, in which case the packet will not be received.
-	pub fn send_packet(&mut self, target: &PeerId, message: Vec<u8>) {
-		let notifs_sink = match self.peers.get(target).and_then(|p| p.get_open()) {
-			None => {
-				debug!(target: "sub-libp2p",
-					"Tried to sent packet to {:?} without an open channel.",
-					target);
-				return
-			}
-			Some(sink) => sink
-		};
-
-		trace!(target: "sub-libp2p", "External API => Packet for {:?}", target);
-		trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target);
-		notifs_sink.send_legacy(message);
-	}
-
 	/// Returns the state of the peerset manager, for debugging purposes.
 	pub fn peerset_debug_info(&mut self) -> serde_json::Value {
 		self.peerset.debug_info()
diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs
index 6804dd3c789da..acb241af2ad2d 100644
--- a/client/network/src/protocol/generic_proto/handler/group.rs
+++ b/client/network/src/protocol/generic_proto/handler/group.rs
@@ -262,16 +262,10 @@ struct NotificationsSinkInner {
 /// dedicated to the peer.
 #[derive(Debug)]
 enum NotificationsSinkMessage {
-	/// Message emitted by [`NotificationsSink::send_legacy`].
-	Legacy {
-		message: Vec<u8>,
-	},
-
 	/// Message emitted by [`NotificationsSink::reserve_notification`] and
 	/// [`NotificationsSink::write_notification_now`].
 	Notification {
 		protocol_name: Cow<'static, str>,
-		encoded_fallback_message: Vec<u8>,
 		message: Vec<u8>,
 	},
 
@@ -280,26 +274,6 @@ enum NotificationsSinkMessage {
 }
 
 impl NotificationsSink {
-	/// Sends a message to the peer using the legacy substream.
-	///
-	/// If too many messages are already buffered, the message is silently discarded and the
-	/// connection to the peer will be closed shortly after.
-	///
-	/// This method will be removed in a future version.
-	pub fn send_legacy<'a>(&'a self, message: impl Into<Vec<u8>>) {
-		let mut lock = self.inner.sync_channel.lock();
-		let result = lock.try_send(NotificationsSinkMessage::Legacy {
-			message: message.into()
-		});
-
-		if result.is_err() {
-			// Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the
-			// buffer, and therefore that `try_send` will succeed.
-			let _result2 = lock.clone().try_send(NotificationsSinkMessage::ForceClose);
-			debug_assert!(_result2.map(|()| true).unwrap_or_else(|err| err.is_disconnected()));
-		}
-	}
-
 	/// Sends a notification to the peer.
 	///
 	/// If too many messages are already buffered, the notification is silently discarded and the
@@ -312,13 +286,11 @@ impl NotificationsSink {
 	pub fn send_sync_notification<'a>(
 		&'a self,
 		protocol_name: Cow<'static, str>,
-		encoded_fallback_message: impl Into<Vec<u8>>,
 		message: impl Into<Vec<u8>>
 	) {
 		let mut lock = self.inner.sync_channel.lock();
 		let result = lock.try_send(NotificationsSinkMessage::Notification {
-			protocol_name: protocol_name,
-			encoded_fallback_message: encoded_fallback_message.into(),
+			protocol_name,
 			message: message.into()
 		});
 
@@ -364,12 +336,10 @@ impl<'a> Ready<'a> {
 	/// Returns an error if the substream has been closed.
 	pub fn send(
 		mut self,
-		encoded_fallback_message: impl Into<Vec<u8>>,
 		notification: impl Into<Vec<u8>>
 	) -> Result<(), ()> {
 		self.lock.start_send(NotificationsSinkMessage::Notification {
 			protocol_name: self.protocol_name,
-			encoded_fallback_message: encoded_fallback_message.into(),
 			message: notification.into(),
 		}).map_err(|_| ())
 	}
@@ -602,26 +572,38 @@ impl ProtocolsHandler for NotifsHandler {
 				};
 
 				match message {
-					NotificationsSinkMessage::Legacy { message } => {
-						self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage {
-							message
-						});
-					}
 					NotificationsSinkMessage::Notification {
 						protocol_name,
-						encoded_fallback_message,
 						message
 					} => {
+						let mut found_any_with_name = false;
+
 						for (handler, _) in &mut self.out_handlers {
-							if *handler.protocol_name() == protocol_name && handler.is_open() {
-								handler.send_or_discard(message);
-								continue 'poll_notifs_sink;
+							if *handler.protocol_name() == protocol_name {
+								found_any_with_name = true;
+								if handler.is_open() {
+									handler.send_or_discard(message);
+									continue 'poll_notifs_sink;
+								}
 							}
 						}
 
-						self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage {
-							message: encoded_fallback_message,
-						});
+						// This code can be reached via the following scenarios:
+						//
+						// - User tried to send a notification on a non-existing protocol. This
+						// most likely relates to https://github.com/paritytech/substrate/issues/6827
+						// - User tried to send a notification to a peer we're not or no longer
+						// connected to. This happens in a normal scenario due to the racy nature
+						// of connections and disconnections, and is benign.
+						//
+						// We print a warning in the former condition.
+						if !found_any_with_name {
+							log::warn!(
+								target: "sub-libp2p",
+								"Tried to send a notification on non-registered protocol: {:?}",
+								protocol_name
+							);
+						}
 					}
 					NotificationsSinkMessage::ForceClose => {
 						return Poll::Ready(ProtocolsHandlerEvent::Close(NotifsHandlerError::SyncNotificationsClogged));
diff --git a/client/network/src/protocol/generic_proto/handler/legacy.rs b/client/network/src/protocol/generic_proto/handler/legacy.rs
index 7d31ed323a43b..d98d864dfc6fa 100644
--- a/client/network/src/protocol/generic_proto/handler/legacy.rs
+++ b/client/network/src/protocol/generic_proto/handler/legacy.rs
@@ -204,12 +204,6 @@ pub enum LegacyProtoHandlerIn {
 
 	/// The node should stop using custom protocols.
 	Disable,
-
-	/// Sends a message through a custom protocol substream.
-	SendCustomMessage {
-		/// The message to send.
-		message: Vec<u8>,
-	},
 }
 
 /// Event that can be emitted by a `LegacyProtoHandler`.
@@ -495,17 +489,6 @@ impl LegacyProtoHandler {
 			ProtocolState::KillAsap => ProtocolState::KillAsap,
 		};
 	}
-
-	/// Sends a message to the remote.
-	fn send_message(&mut self, message: Vec<u8>) {
-		match self.state {
-			ProtocolState::Normal { ref mut substreams, .. } =>
-				substreams[0].send_message(message),
-
-			_ => debug!(target: "sub-libp2p", "Tried to send message over closed protocol \
-				with {:?}", self.remote_peer_id)
-		}
-	}
 }
 
 impl ProtocolsHandler for LegacyProtoHandler {
@@ -539,12 +522,9 @@ impl ProtocolsHandler for LegacyProtoHandler {
 		match message {
 			LegacyProtoHandlerIn::Disable => self.disable(),
 			LegacyProtoHandlerIn::Enable => self.enable(),
-			LegacyProtoHandlerIn::SendCustomMessage { message } =>
-				self.send_message(message),
 		}
 	}
 
-	#[inline]
 	fn inject_dial_upgrade_error(&mut self, _: (), err: ProtocolsHandlerUpgrErr<io::Error>) {
 		let is_severe = match err {
 			ProtocolsHandlerUpgrErr::Upgrade(_) => true,
diff --git a/client/network/src/protocol/generic_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs
index 15c4a17df8d3a..dbe02c350100f 100644
--- a/client/network/src/protocol/generic_proto/tests.rs
+++ b/client/network/src/protocol/generic_proto/tests.rs
@@ -16,19 +16,16 @@
 
 #![cfg(test)]
 
-use futures::{prelude::*, ready};
-use codec::{Encode, Decode};
-use libp2p::core::connection::{ConnectionId, ListenerId};
-use libp2p::core::ConnectedPoint;
-use libp2p::swarm::{Swarm, ProtocolsHandler, IntoProtocolsHandler};
-use libp2p::swarm::{PollParameters, NetworkBehaviour, NetworkBehaviourAction};
+use crate::protocol::generic_proto::{GenericProto, GenericProtoOut};
+
+use futures::prelude::*;
 use libp2p::{PeerId, Multiaddr, Transport};
-use rand::seq::SliceRandom;
+use libp2p::core::{connection::{ConnectionId, ListenerId}, ConnectedPoint};
+use libp2p::swarm::{
+	Swarm, ProtocolsHandler, IntoProtocolsHandler, PollParameters,
+	NetworkBehaviour, NetworkBehaviourAction
+};
 use std::{error, io, task::Context, task::Poll, time::Duration};
-use std::collections::HashSet;
-use crate::protocol::message::{generic::BlockResponse, Message};
-use crate::protocol::generic_proto::{GenericProto, GenericProtoOut};
-use sp_test_primitives::Block;
 
 /// Builds two nodes that have each other as bootstrap nodes.
 /// This is to be used only for testing, and a panic will happen if something goes wrong.
@@ -216,137 +213,6 @@ impl NetworkBehaviour for CustomProtoWithAddr {
 	}
 }
 
-#[test]
-fn two_nodes_transfer_lots_of_packets() {
-	// We spawn two nodes, then make the first one send lots of packets to the second one. The test
-	// ends when the second one has received all of them.
-
-	// This test consists in transferring this given number of packets. Considering that (by
-	// design) the connection gets closed if one of the remotes can't follow the pace, this number
-	// should not exceed the size of the buffer of pending notifications.
-	const NUM_PACKETS: u32 = 512;
-
-	let (mut service1, mut service2) = build_nodes();
-
-	let fut1 = future::poll_fn(move |cx| -> Poll<()> {
-		loop {
-			match ready!(service1.poll_next_unpin(cx)) {
-				Some(GenericProtoOut::CustomProtocolOpen { peer_id, .. }) => {
-					for n in 0 .. NUM_PACKETS {
-						service1.send_packet(
-							&peer_id,
-							Message::<Block>::BlockResponse(BlockResponse {
-								id: n as _,
-								blocks: Vec::new(),
-							}).encode()
-						);
-					}
-				},
-				// An empty handshake is being sent after opening.
-				Some(GenericProtoOut::LegacyMessage { message, .. }) if message.is_empty() => {},
-				_ => panic!(),
-			}
-		}
-	});
-
-	let mut packet_counter = 0u32;
-	let fut2 = future::poll_fn(move |cx| {
-		loop {
-			match ready!(service2.poll_next_unpin(cx)) {
-				Some(GenericProtoOut::CustomProtocolOpen { .. }) => {},
-				// An empty handshake is being sent after opening.
-				Some(GenericProtoOut::LegacyMessage { message, .. }) if message.is_empty() => {},
-				Some(GenericProtoOut::LegacyMessage { message, .. }) => {
-					match Message::<Block>::decode(&mut &message[..]).unwrap() {
-						Message::<Block>::BlockResponse(BlockResponse { id: _, blocks }) => {
-							assert!(blocks.is_empty());
-							packet_counter += 1;
-							if packet_counter == NUM_PACKETS {
-								return Poll::Ready(())
-							}
-						},
-						_ => panic!(),
-					}
-				}
-				_ => panic!(),
-			}
-		}
-	});
-
-	futures::executor::block_on(async move {
-		future::select(fut1, fut2).await;
-	});
-}
-
-#[test]
-fn basic_two_nodes_requests_in_parallel() {
-	let (mut service1, mut service2) = build_nodes();
-
-	// Generate random messages with or without a request id.
-	let mut to_send = {
-		let mut to_send = Vec::new();
-		let mut existing_ids = HashSet::new();
-		for _ in 0..200 { // Note: don't make that number too high or the CPU usage will explode.
-			let req_id = loop {
-				let req_id = rand::random::<u64>();
-
-				// ensure uniqueness - odds of randomly sampling collisions
-				// is unlikely, but possible to cause spurious test failures.
-				if existing_ids.insert(req_id) {
-					break req_id;
-				}
-			};
-
-			to_send.push(Message::<Block>::BlockResponse(
-				BlockResponse { id: req_id, blocks: Vec::new() }
-			));
-		}
-		to_send
-	};
-
-	// Clone `to_send` in `to_receive`. Below we will remove from `to_receive` the messages we
-	// receive, until the list is empty.
-	let mut to_receive = to_send.clone();
-	to_send.shuffle(&mut rand::thread_rng());
-
-	let fut1 = future::poll_fn(move |cx| -> Poll<()> {
-		loop {
-			match ready!(service1.poll_next_unpin(cx)) {
-				Some(GenericProtoOut::CustomProtocolOpen { peer_id, .. }) => {
-					for msg in to_send.drain(..) {
-						service1.send_packet(&peer_id, msg.encode());
-					}
-				},
-				// An empty handshake is being sent after opening.
-				Some(GenericProtoOut::LegacyMessage { message, .. }) if message.is_empty() => {},
-				_ => panic!(),
-			}
-		}
-	});
-
-	let fut2 = future::poll_fn(move |cx| {
-		loop {
-			match ready!(service2.poll_next_unpin(cx)) {
-				Some(GenericProtoOut::CustomProtocolOpen { .. }) => {},
-				// An empty handshake is being sent after opening.
-				Some(GenericProtoOut::LegacyMessage { message, .. }) if message.is_empty() => {},
-				Some(GenericProtoOut::LegacyMessage { message, .. }) => {
-					let pos = to_receive.iter().position(|m| m.encode() == message).unwrap();
-					to_receive.remove(pos);
-					if to_receive.is_empty() {
-						return Poll::Ready(())
-					}
-				}
-				_ => panic!(),
-			}
-		}
-	});
-
-	futures::executor::block_on(async move {
-		future::select(fut1, fut2).await;
-	});
-}
-
 #[test]
 fn reconnect_after_disconnect() {
 	// We connect two nodes together, then force a disconnect (through the API of the `Service`),
diff --git a/client/network/src/protocol/generic_proto/upgrade/legacy.rs b/client/network/src/protocol/generic_proto/upgrade/legacy.rs
index 0937a7798be98..1b2b97253d1ae 100644
--- a/client/network/src/protocol/generic_proto/upgrade/legacy.rs
+++ b/client/network/src/protocol/generic_proto/upgrade/legacy.rs
@@ -123,15 +123,6 @@ impl<TSubstream> RegisteredProtocolSubstream<TSubstream> {
 		self.is_closing = true;
 		self.send_queue.clear();
 	}
-
-	/// Sends a message to the substream.
-	pub fn send_message(&mut self, data: Vec<u8>) {
-		if self.is_closing {
-			return
-		}
-
-		self.send_queue.push_back(From::from(&data[..]));
-	}
 }
 
 /// Event produced by the `RegisteredProtocolSubstream`.
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index a3ac8371dc739..4fa37c64c75e2 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -635,18 +635,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 		// Determine the wire protocol name corresponding to this `engine_id`.
 		let protocol_name = self.protocol_name_by_engine.lock().get(&engine_id).cloned();
 		if let Some(protocol_name) = protocol_name {
-			// For backwards-compatibility reason, we have to duplicate the message and pass it
-			// in the situation where the remote still uses the legacy substream.
-			let fallback = codec::Encode::encode(&{
-				protocol::message::generic::Message::<(), (), (), ()>::Consensus({
-					protocol::message::generic::ConsensusMessage {
-						engine_id,
-						data: message.clone(),
-					}
-				})
-			});
-
-			sink.send_sync_notification(protocol_name, fallback, message);
+			sink.send_sync_notification(protocol_name, message);
 		} else {
 			return;
 		}
@@ -751,7 +740,6 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 		Ok(NotificationSender {
 			sink,
 			protocol_name,
-			engine_id,
 			notification_size_metric: self.notifications_sizes_metric.as_ref().map(|histogram| {
 				histogram.with_label_values(&["out", &maybe_utf8_bytes_to_string(&engine_id)])
 			}),
@@ -1064,9 +1052,6 @@ pub struct NotificationSender {
 	/// Name of the protocol on the wire.
 	protocol_name: Cow<'static, str>,
 
-	/// Engine ID used for the fallback message.
-	engine_id: ConsensusEngineId,
-
 	/// Field extracted from the [`Metrics`] struct and necessary to report the
 	/// notifications-related metrics.
 	notification_size_metric: Option<Histogram>,
@@ -1080,7 +1065,6 @@ impl NotificationSender {
 				Ok(r) => r,
 				Err(()) => return Err(NotificationSenderError::Closed),
 			},
-			engine_id: self.engine_id,
 			notification_size_metric: self.notification_size_metric.clone(),
 		})
 	}
@@ -1091,9 +1075,6 @@ impl NotificationSender {
 pub struct NotificationSenderReady<'a> {
 	ready: Ready<'a>,
 
-	/// Engine ID used for the fallback message.
-	engine_id: ConsensusEngineId,
-
 	/// Field extracted from the [`Metrics`] struct and necessary to report the
 	/// notifications-related metrics.
 	notification_size_metric: Option<Histogram>,
@@ -1108,18 +1089,8 @@ impl<'a> NotificationSenderReady<'a> {
 			notification_size_metric.observe(notification.len() as f64);
 		}
 
-		// For backwards-compatibility reason, we have to duplicate the message and pass it
-		// in the situation where the remote still uses the legacy substream.
-		let fallback = codec::Encode::encode(&{
-			protocol::message::generic::Message::<(), (), (), ()>::Consensus({
-				protocol::message::generic::ConsensusMessage {
-					engine_id: self.engine_id,
-					data: notification.clone(),
-				}
-			})
-		});
-
-		self.ready.send(fallback, notification)
+		self.ready
+			.send(notification)
 			.map_err(|()| NotificationSenderError::Closed)
 	}
 }
diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs
index d269842386cdd..587feebe55c14 100644
--- a/client/network/test/src/lib.rs
+++ b/client/network/test/src/lib.rs
@@ -22,7 +22,10 @@ mod block_import;
 #[cfg(test)]
 mod sync;
 
-use std::{collections::HashMap, pin::Pin, sync::Arc, marker::PhantomData, task::{Poll, Context as FutureContext}};
+use std::{
+	borrow::Cow, collections::HashMap, pin::Pin, sync::Arc, marker::PhantomData,
+	task::{Poll, Context as FutureContext}
+};
 
 use libp2p::build_multiaddr;
 use log::trace;
@@ -55,7 +58,7 @@ use sp_core::H256;
 use sc_network::config::ProtocolConfig;
 use sp_runtime::generic::{BlockId, OpaqueDigestItemId};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor};
-use sp_runtime::Justification;
+use sp_runtime::{ConsensusEngineId, Justification};
 use substrate_test_runtime_client::{self, AccountKeyring};
 use sc_service::client::Client;
 pub use sc_network::config::EmptyTransactionPool;
@@ -553,6 +556,8 @@ pub struct FullPeerConfig {
 	pub keep_blocks: Option<u32>,
 	/// Block announce validator.
 	pub block_announce_validator: Option<Box<dyn BlockAnnounceValidator<Block> + Send + Sync>>,
+	/// List of notification protocols that the network must support.
+	pub notifications_protocols: Vec<(ConsensusEngineId, Cow<'static, str>)>,
 }
 
 pub trait TestNetFactory: Sized {
@@ -663,6 +668,7 @@ pub trait TestNetFactory: Sized {
 		network_config.transport = TransportConfig::MemoryOnly;
 		network_config.listen_addresses = vec![listen_addr.clone()];
 		network_config.allow_non_globals_in_dht = true;
+		network_config.notifications_protocols = config.notifications_protocols;
 
 		let network = NetworkWorker::new(sc_network::config::Params {
 			role: Role::Full,

From 658c389d45b1ad0c1a445c34d2802e2cb2efe972 Mon Sep 17 00:00:00 2001
From: Seun Lanlege <seunlanlege@gmail.com>
Date: Thu, 3 Sep 2020 13:55:12 +0100
Subject: [PATCH 040/122] manual seal is now consensus agnostic (#7010)

* manual seal is now consensus agnostic

* pr grumbles
---
 Cargo.lock                                    |   6 +
 client/consensus/babe/src/aux_schema.rs       |   2 +-
 client/consensus/babe/src/lib.rs              |   5 +-
 client/consensus/manual-seal/Cargo.toml       |  28 ++-
 client/consensus/manual-seal/src/consensus.rs |  44 ++++
 .../manual-seal/src/consensus/babe.rs         | 197 ++++++++++++++++++
 client/consensus/manual-seal/src/error.rs     |   1 +
 client/consensus/manual-seal/src/lib.rs       | 194 ++++++++++++-----
 client/consensus/manual-seal/src/rpc.rs       |   3 +-
 .../src/{seal_new_block.rs => seal_block.rs}  |  47 +++--
 10 files changed, 439 insertions(+), 88 deletions(-)
 create mode 100644 client/consensus/manual-seal/src/consensus.rs
 create mode 100644 client/consensus/manual-seal/src/consensus/babe.rs
 rename client/consensus/manual-seal/src/{seal_new_block.rs => seal_block.rs} (73%)

diff --git a/Cargo.lock b/Cargo.lock
index fbf7f307962b5..9f29feece6d86 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6673,13 +6673,19 @@ dependencies = [
  "parking_lot 0.10.2",
  "sc-basic-authorship",
  "sc-client-api",
+ "sc-consensus-babe",
+ "sc-consensus-epochs",
+ "sc-keystore",
  "sc-transaction-pool",
  "serde",
+ "sp-api",
  "sp-blockchain",
  "sp-consensus",
+ "sp-consensus-babe",
  "sp-core",
  "sp-inherents",
  "sp-runtime",
+ "sp-timestamp",
  "sp-transaction-pool",
  "substrate-prometheus-endpoint",
  "substrate-test-runtime-client",
diff --git a/client/consensus/babe/src/aux_schema.rs b/client/consensus/babe/src/aux_schema.rs
index 4f26568d83343..74078b4ee7b8a 100644
--- a/client/consensus/babe/src/aux_schema.rs
+++ b/client/consensus/babe/src/aux_schema.rs
@@ -51,7 +51,7 @@ fn load_decode<B, T>(backend: &B, key: &[u8]) -> ClientResult<Option<T>>
 }
 
 /// Load or initialize persistent epoch change data from backend.
-pub(crate) fn load_epoch_changes<Block: BlockT, B: AuxStore>(
+pub fn load_epoch_changes<Block: BlockT, B: AuxStore>(
 	backend: &B,
 	config: &BabeGenesisConfiguration,
 ) -> ClientResult<SharedEpochChanges<Block, Epoch>> {
diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs
index 67aca1dd43e7a..95f1653d8646d 100644
--- a/client/consensus/babe/src/lib.rs
+++ b/client/consensus/babe/src/lib.rs
@@ -126,9 +126,10 @@ use schnorrkel::SignatureError;
 use codec::{Encode, Decode};
 use sp_api::ApiExt;
 
-mod aux_schema;
 mod verification;
 mod migration;
+
+pub mod aux_schema;
 pub mod authorship;
 #[cfg(test)]
 mod tests;
@@ -1051,7 +1052,7 @@ where
 }
 
 /// Register the babe inherent data provider, if not registered already.
-fn register_babe_inherent_data_provider(
+pub fn register_babe_inherent_data_provider(
 	inherent_data_providers: &InherentDataProviders,
 	slot_duration: u64,
 ) -> Result<(), sp_consensus::Error> {
diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml
index b557f171c35db..8305f856e09a2 100644
--- a/client/consensus/manual-seal/Cargo.toml
+++ b/client/consensus/manual-seal/Cargo.toml
@@ -22,20 +22,28 @@ parking_lot = "0.10.0"
 serde = { version = "1.0", features=["derive"] }
 assert_matches = "1.3.0"
 
-sc-client-api = { path = "../../../client/api", version = "2.0.0-rc6" }
-sc-transaction-pool = { path = "../../transaction-pool", version = "2.0.0-rc6" }
-sp-blockchain = { path = "../../../primitives/blockchain", version = "2.0.0-rc6" }
-sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common", version = "0.8.0-rc6" }
-sp-inherents = { path = "../../../primitives/inherents", version = "2.0.0-rc6" }
-sp-runtime = {  path = "../../../primitives/runtime", version = "2.0.0-rc6" }
-sp-core = {  path = "../../../primitives/core", version = "2.0.0-rc6" }
-sp-transaction-pool = { path = "../../../primitives/transaction-pool", version = "2.0.0-rc6" }
-prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc6" }
+sc-client-api = { path = "../../api", version = "2.0.0-rc5" }
+sc-consensus-babe = { path = "../../consensus/babe", version = "0.8.0-rc5" }
+sc-consensus-epochs = { path = "../../consensus/epochs", version = "0.8.0-rc5" }
+sp-consensus-babe = { path = "../../../primitives/consensus/babe", version = "0.8.0-rc5" }
+sc-keystore = { path = "../../keystore", version = "2.0.0-rc5" }
+
+sc-transaction-pool = { path = "../../transaction-pool", version = "2.0.0-rc5" }
+sp-blockchain = { path = "../../../primitives/blockchain", version = "2.0.0-rc5" }
+sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common", version = "0.8.0-rc5" }
+sp-inherents = { path = "../../../primitives/inherents", version = "2.0.0-rc5" }
+sp-runtime = {  path = "../../../primitives/runtime", version = "2.0.0-rc5" }
+sp-core = {  path = "../../../primitives/core", version = "2.0.0-rc5" }
+sp-api = {  path = "../../../primitives/api", version = "2.0.0-rc5" }
+sp-transaction-pool = { path = "../../../primitives/transaction-pool", version = "2.0.0-rc5" }
+sp-timestamp = { path = "../../../primitives/timestamp", version = "2.0.0-rc6" }
+
+prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc5" }
 
 [dev-dependencies]
+tokio = { version = "0.2", features = ["rt-core", "macros"] }
 sc-basic-authorship = { path = "../../basic-authorship", version = "0.8.0-rc6" }
 substrate-test-runtime-client = { path = "../../../test-utils/runtime/client", version = "2.0.0-rc6" }
 substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool", version = "2.0.0-rc6" }
-tokio = { version = "0.2", features = ["rt-core", "macros"] }
 env_logger = "0.7.0"
 tempfile = "3.1.0"
diff --git a/client/consensus/manual-seal/src/consensus.rs b/client/consensus/manual-seal/src/consensus.rs
new file mode 100644
index 0000000000000..7bafeb50207d4
--- /dev/null
+++ b/client/consensus/manual-seal/src/consensus.rs
@@ -0,0 +1,44 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Extensions for manual seal to produce blocks valid for any runtime.
+use super::Error;
+
+use sp_runtime::traits::{Block as BlockT, DigestFor};
+use sp_inherents::InherentData;
+use sp_consensus::BlockImportParams;
+
+pub mod babe;
+
+/// Consensus data provider, manual seal uses this trait object for authoring blocks valid 
+/// for any runtime.
+pub trait ConsensusDataProvider<B: BlockT>: Send + Sync {
+	/// Block import transaction type
+	type Transaction;
+
+	/// Attempt to create a consensus digest.
+	fn create_digest(&self, parent: &B::Header, inherents: &InherentData) -> Result<DigestFor<B>, Error>;
+
+	/// set up the neccessary import params.
+	fn append_block_import(
+		&self,
+		parent: &B::Header,
+		params: &mut BlockImportParams<B, Self::Transaction>,
+		inherents: &InherentData
+	) -> Result<(), Error>;
+}
diff --git a/client/consensus/manual-seal/src/consensus/babe.rs b/client/consensus/manual-seal/src/consensus/babe.rs
new file mode 100644
index 0000000000000..71dd250733ad9
--- /dev/null
+++ b/client/consensus/manual-seal/src/consensus/babe.rs
@@ -0,0 +1,197 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! BABE consensus data provider
+
+use super::ConsensusDataProvider;
+use crate::Error;
+
+use std::{
+	any::Any,
+	borrow::Cow,
+	sync::{Arc, atomic},
+	time::SystemTime,
+};
+use sc_client_api::AuxStore;
+use sc_consensus_babe::{
+	Config, Epoch, authorship, CompatibleDigestItem, BabeIntermediate,
+	register_babe_inherent_data_provider, INTERMEDIATE_KEY,
+};
+use sc_consensus_epochs::{SharedEpochChanges, descendent_query};
+use sc_keystore::KeyStorePtr;
+
+use sp_api::{ProvideRuntimeApi, TransactionFor};
+use sp_blockchain::{HeaderBackend, HeaderMetadata};
+use sp_consensus::BlockImportParams;
+use sp_consensus_babe::{BabeApi, inherents::BabeInherentData};
+use sp_inherents::{InherentDataProviders, InherentData, ProvideInherentData, InherentIdentifier};
+use sp_runtime::{
+	traits::{DigestItemFor, DigestFor, Block as BlockT, Header as _},
+	generic::Digest,
+};
+use sp_timestamp::{InherentType, InherentError, INHERENT_IDENTIFIER};
+
+/// Provides BABE-compatible predigests and BlockImportParams.
+/// Intended for use with BABE runtimes.
+pub struct BabeConsensusDataProvider<B: BlockT, C> {
+	/// shared reference to keystore
+	keystore: KeyStorePtr,
+
+	/// Shared reference to the client.
+	client: Arc<C>,
+
+	/// Shared epoch changes
+	epoch_changes: SharedEpochChanges<B, Epoch>,
+
+	/// BABE config, gotten from the runtime.
+	config: Config,
+}
+
+impl<B, C> BabeConsensusDataProvider<B, C>
+	where
+		B: BlockT,
+		C: AuxStore + ProvideRuntimeApi<B>,
+		C::Api: BabeApi<B, Error = sp_blockchain::Error>,
+{
+	pub fn new(
+		client: Arc<C>,
+		keystore: KeyStorePtr,
+		provider: &InherentDataProviders,
+		epoch_changes: SharedEpochChanges<B, Epoch>,
+	) -> Result<Self, Error> {
+		let config = Config::get_or_compute(&*client)?;
+		let timestamp_provider = SlotTimestampProvider::new(config.slot_duration)?;
+
+		provider.register_provider(timestamp_provider)?;
+		register_babe_inherent_data_provider(provider, config.slot_duration)?;
+
+		Ok(Self {
+			config,
+			client,
+			keystore,
+			epoch_changes,
+		})
+	}
+}
+
+impl<B, C> ConsensusDataProvider<B> for BabeConsensusDataProvider<B, C>
+	where
+		B: BlockT,
+		C: AuxStore + HeaderBackend<B> + HeaderMetadata<B, Error = sp_blockchain::Error> + ProvideRuntimeApi<B>,
+		C::Api: BabeApi<B, Error = sp_blockchain::Error>,
+{
+	type Transaction = TransactionFor<C, B>;
+
+	fn create_digest(&self, parent: &B::Header, inherents: &InherentData) -> Result<DigestFor<B>, Error> {
+		let slot_number = inherents.babe_inherent_data()?;
+
+		let epoch_changes = self.epoch_changes.lock();
+		let epoch_descriptor = epoch_changes
+			.epoch_descriptor_for_child_of(
+				descendent_query(&*self.client),
+				&parent.hash(),
+				parent.number().clone(),
+				slot_number,
+			)
+			.map_err(|e| Error::StringError(format!("failed to fetch epoch_descriptor: {}", e)))?
+			.ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet)?;
+
+		let epoch = epoch_changes
+			.viable_epoch(
+				&epoch_descriptor,
+				|slot| Epoch::genesis(&self.config, slot),
+			)
+			.ok_or_else(|| {
+				log::info!(target: "babe", "create_digest: no viable_epoch :(");
+				sp_consensus::Error::InvalidAuthoritiesSet
+			})?;
+
+		// this is a dev node environment, we should always be able to claim a slot.
+		let (predigest, _) = authorship::claim_slot(slot_number, epoch.as_ref(), &self.keystore)
+			.ok_or_else(|| Error::StringError("failed to claim slot for authorship".into()))?;
+
+		Ok(Digest {
+			logs: vec![
+				<DigestItemFor<B> as CompatibleDigestItem>::babe_pre_digest(predigest),
+			],
+		})
+	}
+
+	fn append_block_import(
+		&self,
+		parent: &B::Header,
+		params: &mut BlockImportParams<B, Self::Transaction>,
+		inherents: &InherentData
+	) -> Result<(), Error> {
+		let slot_number = inherents.babe_inherent_data()?;
+
+		let epoch_descriptor = self.epoch_changes.lock()
+			.epoch_descriptor_for_child_of(
+				descendent_query(&*self.client),
+				&parent.hash(),
+				parent.number().clone(),
+				slot_number,
+			)
+			.map_err(|e| Error::StringError(format!("failed to fetch epoch data: {}", e)))?
+			.ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet)?;
+
+		params.intermediates.insert(
+			Cow::from(INTERMEDIATE_KEY),
+			Box::new(BabeIntermediate::<B> { epoch_descriptor }) as Box<dyn Any>,
+		);
+
+		Ok(())
+	}
+}
+
+/// Provide duration since unix epoch in millisecond for timestamp inherent.
+/// Mocks the timestamp inherent to always produce the timestamp for the next babe slot.
+struct SlotTimestampProvider {
+	time: atomic::AtomicU64,
+	slot_duration: u64
+}
+
+impl SlotTimestampProvider {
+	/// create a new mocked time stamp provider.
+	fn new(slot_duration: u64) -> Result<Self, Error> {
+		let now = SystemTime::now();
+		let duration = now.duration_since(SystemTime::UNIX_EPOCH)
+			.map_err(|err| Error::StringError(format!("{}", err)))?;
+		Ok(Self {
+			time: atomic::AtomicU64::new(duration.as_millis() as u64),
+			slot_duration,
+		})
+	}
+}
+
+impl ProvideInherentData for SlotTimestampProvider {
+	fn inherent_identifier(&self) -> &'static InherentIdentifier {
+		&INHERENT_IDENTIFIER
+	}
+
+	fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), sp_inherents::Error> {
+		// we update the time here.
+		let duration: InherentType = self.time.fetch_add(self.slot_duration, atomic::Ordering::SeqCst);
+		inherent_data.put_data(INHERENT_IDENTIFIER, &duration)?;
+		Ok(())
+	}
+
+	fn error_to_string(&self, error: &[u8]) -> Option<String> {
+		InherentError::try_from(&INHERENT_IDENTIFIER, error).map(|e| format!("{:?}", e))
+	}
+}
\ No newline at end of file
diff --git a/client/consensus/manual-seal/src/error.rs b/client/consensus/manual-seal/src/error.rs
index 2411a839b027c..e2628008c24c7 100644
--- a/client/consensus/manual-seal/src/error.rs
+++ b/client/consensus/manual-seal/src/error.rs
@@ -18,6 +18,7 @@
 
 //! A manual sealing engine: the engine listens for rpc calls to seal blocks and create forks.
 //! This is suitable for a testing environment.
+
 use sp_consensus::{Error as ConsensusError, ImportResult};
 use sp_blockchain::Error as BlockchainError;
 use sp_inherents::Error as InherentsError;
diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs
index 36aeffd9794f0..0a8ed28a27c81 100644
--- a/client/consensus/manual-seal/src/lib.rs
+++ b/client/consensus/manual-seal/src/lib.rs
@@ -21,8 +21,9 @@
 
 use futures::prelude::*;
 use sp_consensus::{
-	Environment, Proposer, ForkChoiceStrategy, BlockImportParams, BlockOrigin, SelectChain,
-	import_queue::{BasicQueue, CacheKeyId, Verifier, BoxBlockImport},
+	Environment, Proposer, SelectChain, BlockImport,
+	ForkChoiceStrategy, BlockImportParams, BlockOrigin,
+	import_queue::{Verifier, BasicQueue, CacheKeyId, BoxBlockImport},
 };
 use sp_blockchain::HeaderBackend;
 use sp_inherents::InherentDataProviders;
@@ -34,17 +35,19 @@ use prometheus_endpoint::Registry;
 
 mod error;
 mod finalize_block;
-mod seal_new_block;
+mod seal_block;
+
+pub mod consensus;
 pub mod rpc;
 
-use self::{
-	finalize_block::{finalize_block, FinalizeBlockParams},
-	seal_new_block::{seal_new_block, SealBlockParams},
-};
 pub use self::{
 	error::Error,
+	consensus::ConsensusDataProvider,
+	finalize_block::{finalize_block, FinalizeBlockParams},
+	seal_block::{SealBlockParams, seal_block, MAX_PROPOSAL_DURATION},
 	rpc::{EngineCommand, CreatedBlock},
 };
+use sp_api::{ProvideRuntimeApi, TransactionFor};
 
 /// The verifier for the manual seal engine; instantly finalizes.
 struct ManualSealVerifier;
@@ -87,25 +90,83 @@ pub fn import_queue<Block, Transaction>(
 	)
 }
 
+/// Params required to start the instant sealing authorship task.
+pub struct ManualSealParams<B: BlockT, BI, E, C: ProvideRuntimeApi<B>, A: txpool::ChainApi, SC, CS> {
+	/// Block import instance for well. importing blocks.
+	pub block_import: BI,
+
+	/// The environment we are producing blocks for.
+	pub env: E,
+
+	/// Client instance
+	pub client: Arc<C>,
+
+	/// Shared reference to the transaction pool.
+	pub pool: Arc<txpool::Pool<A>>,
+
+	/// Stream<Item = EngineCommands>, Basically the receiving end of a channel for sending commands to
+	/// the authorship task.
+	pub commands_stream: CS,
+
+	/// SelectChain strategy.
+	pub select_chain: SC,
+
+	/// Digest provider for inclusion in blocks.
+	pub consensus_data_provider: Option<Box<dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>>,
+
+	/// Provider for inherents to include in blocks.
+	pub inherent_data_providers: InherentDataProviders,
+}
+
+/// Params required to start the manual sealing authorship task.
+pub struct InstantSealParams<B: BlockT, BI, E, C: ProvideRuntimeApi<B>, A: txpool::ChainApi, SC> {
+	/// Block import instance for well. importing blocks.
+	pub block_import: BI,
+
+	/// The environment we are producing blocks for.
+	pub env: E,
+
+	/// Client instance
+	pub client: Arc<C>,
+
+	/// Shared reference to the transaction pool.
+	pub pool: Arc<txpool::Pool<A>>,
+
+	/// SelectChain strategy.
+	pub select_chain: SC,
+
+	/// Digest provider for inclusion in blocks.
+	pub consensus_data_provider: Option<Box<dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>>,
+
+	/// Provider for inherents to include in blocks.
+	pub inherent_data_providers: InherentDataProviders,
+}
+
 /// Creates the background authorship task for the manual seal engine.
-pub async fn run_manual_seal<B, CB, E, C, A, SC, S, T>(
-	mut block_import: BoxBlockImport<B, T>,
-	mut env: E,
-	client: Arc<C>,
-	pool: Arc<txpool::Pool<A>>,
-	mut commands_stream: S,
-	select_chain: SC,
-	inherent_data_providers: InherentDataProviders,
+pub async fn run_manual_seal<B, BI, CB, E, C, A, SC, CS>(
+	ManualSealParams {
+		mut block_import,
+		mut env,
+		client,
+		pool,
+		mut commands_stream,
+		select_chain,
+		inherent_data_providers,
+		consensus_data_provider,
+		..
+	}: ManualSealParams<B, BI, E, C, A, SC, CS>
 )
 	where
 		A: txpool::ChainApi<Block=B> + 'static,
 		B: BlockT + 'static,
-		C: HeaderBackend<B> + Finalizer<B, CB> + 'static,
+		BI: BlockImport<B, Error = sp_consensus::Error, Transaction = sp_api::TransactionFor<C, B>>
+			+ Send + Sync + 'static,
+		C: HeaderBackend<B> + Finalizer<B, CB> + ProvideRuntimeApi<B> + 'static,
 		CB: ClientBackend<B> + 'static,
 		E: Environment<B> + 'static,
 		E::Error: std::fmt::Display,
 		<E::Proposer as Proposer<B>>::Error: std::fmt::Display,
-		S: Stream<Item=EngineCommand<<B as BlockT>::Hash>> + Unpin + 'static,
+		CS: Stream<Item=EngineCommand<<B as BlockT>::Hash>> + Unpin + 'static,
 		SC: SelectChain<B> + 'static,
 {
 	while let Some(command) = commands_stream.next().await {
@@ -116,7 +177,7 @@ pub async fn run_manual_seal<B, CB, E, C, A, SC, S, T>(
 				parent_hash,
 				sender,
 			} => {
-				seal_new_block(
+				seal_block(
 					SealBlockParams {
 						sender,
 						parent_hash,
@@ -126,6 +187,7 @@ pub async fn run_manual_seal<B, CB, E, C, A, SC, S, T>(
 						select_chain: &select_chain,
 						block_import: &mut block_import,
 						inherent_data_provider: &inherent_data_providers,
+						consensus_data_provider: consensus_data_provider.as_ref().map(|p| &**p),
 						pool: pool.clone(),
 						client: client.clone(),
 					}
@@ -149,18 +211,24 @@ pub async fn run_manual_seal<B, CB, E, C, A, SC, S, T>(
 /// runs the background authorship task for the instant seal engine.
 /// instant-seal creates a new block for every transaction imported into
 /// the transaction pool.
-pub async fn run_instant_seal<B, CB, E, C, A, SC, T>(
-	block_import: BoxBlockImport<B, T>,
-	env: E,
-	client: Arc<C>,
-	pool: Arc<txpool::Pool<A>>,
-	select_chain: SC,
-	inherent_data_providers: InherentDataProviders,
+pub async fn run_instant_seal<B, BI, CB, E, C, A, SC>(
+	InstantSealParams {
+		block_import,
+		env,
+		client,
+		pool,
+		select_chain,
+		consensus_data_provider,
+		inherent_data_providers,
+		..
+	}: InstantSealParams<B, BI, E, C, A, SC>
 )
 	where
 		A: txpool::ChainApi<Block=B> + 'static,
 		B: BlockT + 'static,
-		C: HeaderBackend<B> + Finalizer<B, CB> + 'static,
+		BI: BlockImport<B, Error = sp_consensus::Error, Transaction = sp_api::TransactionFor<C, B>>
+			+ Send + Sync + 'static,
+		C: HeaderBackend<B> + Finalizer<B, CB> + ProvideRuntimeApi<B> + 'static,
 		CB: ClientBackend<B> + 'static,
 		E: Environment<B> + 'static,
 		E::Error: std::fmt::Display,
@@ -181,13 +249,16 @@ pub async fn run_instant_seal<B, CB, E, C, A, SC, T>(
 		});
 
 	run_manual_seal(
-		block_import,
-		env,
-		client,
-		pool,
-		commands_stream,
-		select_chain,
-		inherent_data_providers,
+		ManualSealParams {
+			block_import,
+			env,
+			client,
+			pool,
+			commands_stream,
+			select_chain,
+			consensus_data_provider,
+			inherent_data_providers,
+		}
 	).await
 }
 
@@ -233,7 +304,7 @@ mod tests {
 		// this test checks that blocks are created as soon as transactions are imported into the pool.
 		let (sender, receiver) = futures::channel::oneshot::channel();
 		let mut sender = Arc::new(Some(sender));
-		let stream = pool.pool().validated_pool().import_notification_stream()
+		let commands_stream = pool.pool().validated_pool().import_notification_stream()
 			.map(move |_| {
 				// we're only going to submit one tx so this fn will only be called once.
 				let mut_sender =  Arc::get_mut(&mut sender).unwrap();
@@ -246,13 +317,16 @@ mod tests {
 				}
 			});
 		let future = run_manual_seal(
-			Box::new(client.clone()),
-			env,
-			client.clone(),
-			pool.pool().clone(),
-			stream,
-			select_chain,
-			inherent_data_providers,
+			ManualSealParams {
+				block_import: client.clone(),
+				env,
+				client: client.clone(),
+				pool: pool.pool().clone(),
+				commands_stream,
+				select_chain,
+				inherent_data_providers,
+				consensus_data_provider: None,
+			}
 		);
 		std::thread::spawn(|| {
 			let mut rt = tokio::runtime::Runtime::new().unwrap();
@@ -299,15 +373,18 @@ mod tests {
 			None,
 		);
 		// this test checks that blocks are created as soon as an engine command is sent over the stream.
-		let (mut sink, stream) = futures::channel::mpsc::channel(1024);
+		let (mut sink, commands_stream) = futures::channel::mpsc::channel(1024);
 		let future = run_manual_seal(
-			Box::new(client.clone()),
-			env,
-			client.clone(),
-			pool.pool().clone(),
-			stream,
-			select_chain,
-			inherent_data_providers,
+			ManualSealParams {
+				block_import: client.clone(),
+				env,
+				client: client.clone(),
+				pool: pool.pool().clone(),
+				commands_stream,
+				select_chain,
+				consensus_data_provider: None,
+				inherent_data_providers,
+			}
 		);
 		std::thread::spawn(|| {
 			let mut rt = tokio::runtime::Runtime::new().unwrap();
@@ -371,15 +448,18 @@ mod tests {
 			None,
 		);
 		// this test checks that blocks are created as soon as an engine command is sent over the stream.
-		let (mut sink, stream) = futures::channel::mpsc::channel(1024);
+		let (mut sink, commands_stream) = futures::channel::mpsc::channel(1024);
 		let future = run_manual_seal(
-			Box::new(client.clone()),
-			env,
-			client.clone(),
-			pool.pool().clone(),
-			stream,
-			select_chain,
-			inherent_data_providers,
+			ManualSealParams {
+				block_import: client.clone(),
+				env,
+				client: client.clone(),
+				pool: pool.pool().clone(),
+				commands_stream,
+				select_chain,
+				consensus_data_provider: None,
+				inherent_data_providers,
+			}
 		);
 		std::thread::spawn(|| {
 			let mut rt = tokio::runtime::Runtime::new().unwrap();
diff --git a/client/consensus/manual-seal/src/rpc.rs b/client/consensus/manual-seal/src/rpc.rs
index f3f0fe4a128ed..690b6c1eb9996 100644
--- a/client/consensus/manual-seal/src/rpc.rs
+++ b/client/consensus/manual-seal/src/rpc.rs
@@ -14,7 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-//! RPC interface for the ManualSeal Engine.
+//! RPC interface for the `ManualSeal` Engine.
+
 use sp_consensus::ImportedAux;
 use jsonrpc_core::Error;
 use jsonrpc_derive::rpc;
diff --git a/client/consensus/manual-seal/src/seal_new_block.rs b/client/consensus/manual-seal/src/seal_block.rs
similarity index 73%
rename from client/consensus/manual-seal/src/seal_new_block.rs
rename to client/consensus/manual-seal/src/seal_block.rs
index c5aea11ced316..58f017f2d41ad 100644
--- a/client/consensus/manual-seal/src/seal_new_block.rs
+++ b/client/consensus/manual-seal/src/seal_block.rs
@@ -16,7 +16,7 @@
 
 //! Block sealing utilities
 
-use crate::{Error, rpc};
+use crate::{Error, rpc, CreatedBlock, ConsensusDataProvider};
 use std::sync::Arc;
 use sp_runtime::{
 	traits::{Block as BlockT, Header as HeaderT},
@@ -24,24 +24,21 @@ use sp_runtime::{
 };
 use futures::prelude::*;
 use sc_transaction_pool::txpool;
-use rpc::CreatedBlock;
-
 use sp_consensus::{
-	self, BlockImport, Environment, Proposer,
-	ForkChoiceStrategy, BlockImportParams, BlockOrigin,
-	ImportResult, SelectChain,
-	import_queue::BoxBlockImport,
+	self, BlockImport, Environment, Proposer, ForkChoiceStrategy,
+	BlockImportParams, BlockOrigin, ImportResult, SelectChain,
 };
 use sp_blockchain::HeaderBackend;
 use std::collections::HashMap;
 use std::time::Duration;
 use sp_inherents::InherentDataProviders;
+use sp_api::{ProvideRuntimeApi, TransactionFor};
 
 /// max duration for creating a proposal in secs
-const MAX_PROPOSAL_DURATION: u64 = 10;
+pub const MAX_PROPOSAL_DURATION: u64 = 10;
 
 /// params for sealing a new block
-pub struct SealBlockParams<'a, B: BlockT, SC, HB, E, T, P: txpool::ChainApi> {
+pub struct SealBlockParams<'a, B: BlockT, BI, SC, C: ProvideRuntimeApi<B>, E, P: txpool::ChainApi> {
 	/// if true, empty blocks(without extrinsics) will be created.
 	/// otherwise, will return Error::EmptyTransactionPool.
 	pub create_empty: bool,
@@ -54,19 +51,21 @@ pub struct SealBlockParams<'a, B: BlockT, SC, HB, E, T, P: txpool::ChainApi> {
 	/// transaction pool
 	pub pool: Arc<txpool::Pool<P>>,
 	/// header backend
-	pub client: Arc<HB>,
+	pub client: Arc<C>,
 	/// Environment trait object for creating a proposer
 	pub env: &'a mut E,
 	/// SelectChain object
 	pub select_chain: &'a SC,
+	/// Digest provider for inclusion in blocks.
+	pub consensus_data_provider: Option<&'a dyn ConsensusDataProvider<B, Transaction = TransactionFor<C, B>>>,
 	/// block import object
-	pub block_import: &'a mut BoxBlockImport<B, T>,
+	pub block_import: &'a mut BI,
 	/// inherent data provider
 	pub inherent_data_provider: &'a InherentDataProviders,
 }
 
 /// seals a new block with the given params
-pub async fn seal_new_block<B, SC, HB, E, T, P>(
+pub async fn seal_block<B, BI, SC, C, E, P>(
 	SealBlockParams {
 		create_empty,
 		finalize,
@@ -77,13 +76,16 @@ pub async fn seal_new_block<B, SC, HB, E, T, P>(
 		block_import,
 		env,
 		inherent_data_provider,
+		consensus_data_provider: digest_provider,
 		mut sender,
 		..
-	}: SealBlockParams<'_, B, SC, HB, E, T, P>
+	}: SealBlockParams<'_, B, BI, SC, C, E, P>
 )
 	where
 		B: BlockT,
-		HB: HeaderBackend<B>,
+		BI: BlockImport<B, Error = sp_consensus::Error, Transaction = sp_api::TransactionFor<C, B>>
+			+ Send + Sync + 'static,
+		C: HeaderBackend<B> + ProvideRuntimeApi<B>,
 		E: Environment<B>,
 		<E as Environment<B>>::Error: std::fmt::Display,
 		<E::Proposer as Proposer<B>>::Error: std::fmt::Display,
@@ -98,7 +100,7 @@ pub async fn seal_new_block<B, SC, HB, E, T, P>(
 		// get the header to build this new block on.
 		// use the parent_hash supplied via `EngineCommand`
 		// or fetch the best_block.
-		let header = match parent_hash {
+		let parent = match parent_hash {
 			Some(hash) => {
 				match client.header(BlockId::Hash(hash))? {
 					Some(header) => header,
@@ -108,11 +110,18 @@ pub async fn seal_new_block<B, SC, HB, E, T, P>(
 			None => select_chain.best_chain()?
 		};
 
-		let proposer = env.init(&header)
+		let proposer = env.init(&parent)
 			.map_err(|err| Error::StringError(format!("{}", err))).await?;
 		let id = inherent_data_provider.create_inherent_data()?;
 		let inherents_len = id.len();
-		let proposal = proposer.propose(id, Default::default(), Duration::from_secs(MAX_PROPOSAL_DURATION), false.into())
+
+		let digest = if let Some(digest_provider) = digest_provider {
+			digest_provider.create_digest(&parent, &id)?
+		} else {
+			Default::default()
+		};
+
+		let proposal = proposer.propose(id.clone(), digest, Duration::from_secs(MAX_PROPOSAL_DURATION), false.into())
 			.map_err(|err| Error::StringError(format!("{}", err))).await?;
 
 		if proposal.block.extrinsics().len() == inherents_len && !create_empty {
@@ -125,6 +134,10 @@ pub async fn seal_new_block<B, SC, HB, E, T, P>(
 		params.finalized = finalize;
 		params.fork_choice = Some(ForkChoiceStrategy::LongestChain);
 
+		if let Some(digest_provider) = digest_provider {
+			digest_provider.append_block_import(&parent, &mut params, &id)?;
+		}
+
 		match block_import.import_block(params, HashMap::new())? {
 			ImportResult::Imported(aux) => {
 				Ok(CreatedBlock { hash: <B as BlockT>::Header::hash(&header), aux })

From b3092419c052fc2e9782a7163d63a7d644cc3696 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Silva?=
 <123550+andresilva@users.noreply.github.com>
Date: Fri, 4 Sep 2020 10:01:14 +0100
Subject: [PATCH 041/122] grandpa: report metrics on prevotes and precommits
 cast (#6970)

* grandpa: report metrics on prevotes and precommits cast

* Update client/finality-grandpa/src/environment.rs

Co-authored-by: Max Inden <mail@max-inden.de>

* Update client/finality-grandpa/src/environment.rs

Co-authored-by: Max Inden <mail@max-inden.de>

Co-authored-by: Max Inden <mail@max-inden.de>
---
 client/finality-grandpa/src/environment.rs | 70 +++++++++++++++++++---
 1 file changed, 62 insertions(+), 8 deletions(-)

diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs
index d862372770518..9215dcb323516 100644
--- a/client/finality-grandpa/src/environment.rs
+++ b/client/finality-grandpa/src/environment.rs
@@ -39,7 +39,7 @@ use sp_runtime::generic::BlockId;
 use sp_runtime::traits::{
 	Block as BlockT, Header as HeaderT, NumberFor, One, Zero,
 };
-use sc_telemetry::{telemetry, CONSENSUS_INFO};
+use sc_telemetry::{telemetry, CONSENSUS_DEBUG, CONSENSUS_INFO};
 
 use crate::{
 	CommandOrError, Commit, Config, Error, Precommit, Prevote,
@@ -59,7 +59,7 @@ use sp_finality_grandpa::{
 	AuthorityId, AuthoritySignature, Equivocation, EquivocationProof,
 	GrandpaApi, RoundNumber, SetId,
 };
-use prometheus_endpoint::{Gauge, U64, register, PrometheusError};
+use prometheus_endpoint::{register, Counter, Gauge, PrometheusError, U64};
 
 type HistoricalVotes<Block> = finality_grandpa::HistoricalVotes<
 	<Block as BlockT>::Hash,
@@ -378,14 +378,32 @@ impl<Block: BlockT> SharedVoterSetState<Block> {
 #[derive(Clone)]
 pub(crate) struct Metrics {
 	finality_grandpa_round: Gauge<U64>,
+	finality_grandpa_prevotes: Counter<U64>,
+	finality_grandpa_precommits: Counter<U64>,
 }
 
 impl Metrics {
-	pub(crate) fn register(registry: &prometheus_endpoint::Registry) -> Result<Self, PrometheusError> {
+	pub(crate) fn register(
+		registry: &prometheus_endpoint::Registry,
+	) -> Result<Self, PrometheusError> {
 		Ok(Self {
 			finality_grandpa_round: register(
 				Gauge::new("finality_grandpa_round", "Highest completed GRANDPA round.")?,
-				registry
+				registry,
+			)?,
+			finality_grandpa_prevotes: register(
+				Counter::new(
+					"finality_grandpa_prevotes_total",
+					"Total number of GRANDPA prevotes cast locally.",
+				)?,
+				registry,
+			)?,
+			finality_grandpa_precommits: register(
+				Counter::new(
+					"finality_grandpa_precommits_total",
+					"Total number of GRANDPA precommits cast locally.",
+				)?,
+				registry,
 			)?,
 		})
 	}
@@ -804,9 +822,22 @@ where
 			None => return Ok(()),
 		};
 
+		let report_prevote_metrics = |prevote: &Prevote<Block>| {
+			telemetry!(CONSENSUS_DEBUG; "afg.prevote_issued";
+				"round" => round,
+				"target_number" => ?prevote.target_number,
+				"target_hash" => ?prevote.target_hash,
+			);
+
+			if let Some(metrics) = self.metrics.as_ref() {
+				metrics.finality_grandpa_prevotes.inc();
+			}
+		};
+
 		self.update_voter_set_state(|voter_set_state| {
 			let (completed_rounds, current_rounds) = voter_set_state.with_current_round(round)?;
-			let current_round = current_rounds.get(&round)
+			let current_round = current_rounds
+				.get(&round)
 				.expect("checked in with_current_round that key exists; qed.");
 
 			if !current_round.can_prevote() {
@@ -816,6 +847,9 @@ where
 				return Ok(None);
 			}
 
+			// report to telemetry and prometheus
+			report_prevote_metrics(&prevote);
+
 			let propose = current_round.propose();
 
 			let mut current_rounds = current_rounds.clone();
@@ -837,7 +871,11 @@ where
 		Ok(())
 	}
 
-	fn precommitted(&self, round: RoundNumber, precommit: Precommit<Block>) -> Result<(), Self::Error> {
+	fn precommitted(
+		&self,
+		round: RoundNumber,
+		precommit: Precommit<Block>,
+	) -> Result<(), Self::Error> {
 		let local_id = crate::is_voter(&self.voters, self.config.keystore.as_ref());
 
 		let local_id = match local_id {
@@ -845,9 +883,22 @@ where
 			None => return Ok(()),
 		};
 
+		let report_precommit_metrics = |precommit: &Precommit<Block>| {
+			telemetry!(CONSENSUS_DEBUG; "afg.precommit_issued";
+				"round" => round,
+				"target_number" => ?precommit.target_number,
+				"target_hash" => ?precommit.target_hash,
+			);
+
+			if let Some(metrics) = self.metrics.as_ref() {
+				metrics.finality_grandpa_precommits.inc();
+			}
+		};
+
 		self.update_voter_set_state(|voter_set_state| {
 			let (completed_rounds, current_rounds) = voter_set_state.with_current_round(round)?;
-			let current_round = current_rounds.get(&round)
+			let current_round = current_rounds
+				.get(&round)
 				.expect("checked in with_current_round that key exists; qed.");
 
 			if !current_round.can_precommit() {
@@ -857,13 +908,16 @@ where
 				return Ok(None);
 			}
 
+			// report to telemetry and prometheus
+			report_precommit_metrics(&precommit);
+
 			let propose = current_round.propose();
 			let prevote = match current_round {
 				HasVoted::Yes(_, Vote::Prevote(_, prevote)) => prevote,
 				_ => {
 					let msg = "Voter precommitting before prevoting.";
 					return Err(Error::Safety(msg.to_string()));
-				},
+				}
 			};
 
 			let mut current_rounds = current_rounds.clone();

From 130b7d4e545b76a247437f1665e71a1b4f7bb52b Mon Sep 17 00:00:00 2001
From: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Date: Fri, 4 Sep 2020 14:29:34 +0200
Subject: [PATCH 042/122] Fix compact npos solution edge count calculation
 (#7021)

This edge count is used for weighing, and it is somewhat trivial to review and verify that the current implementation was ignoring `votes16` field of the struct. As reminder, the struct is like this:
```rust

struct Compact {
  votes1: ... ,
  votes2: ...,
  ...,
  votes16: ...,
}
```

I already will fix this in https://github.com/paritytech/substrate/pull/7007, but since it might take a while, this one can go in asap and make it to the very next runtime.
---
 primitives/npos-elections/compact/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/primitives/npos-elections/compact/src/lib.rs b/primitives/npos-elections/compact/src/lib.rs
index 03526d17981f9..54c94b6df65c2 100644
--- a/primitives/npos-elections/compact/src/lib.rs
+++ b/primitives/npos-elections/compact/src/lib.rs
@@ -157,7 +157,7 @@ fn struct_def(
 		)
 	}).collect::<TokenStream2>();
 
-	let edge_count_impl = (1..count).map(|c| {
+	let edge_count_impl = (1..=count).map(|c| {
 		let field_name = field_name_for(c);
 		quote!(
 			all_edges = all_edges.saturating_add(

From 29c993a001f72214f80796a3f6172e0b007ba8eb Mon Sep 17 00:00:00 2001
From: Roman Borschel <romanb@users.noreply.github.com>
Date: Sun, 6 Sep 2020 19:59:05 +0200
Subject: [PATCH 043/122] Refactor & detach network metrics. (#6986)

* Refactor sc-network/service metrics.

  1. Aggregate sc-network metrics into a submodule, introducing
  two more sourced metrics to avoid duplicate atomics.

  2. Decouple periodic sc-service network metrics from other
  metrics, so that they can be updated independently.

* Update client/service/src/metrics.rs

* Update client/service/src/metrics.rs
---
 client/informant/src/lib.rs           |   6 +-
 client/network/src/network_state.rs   |   4 -
 client/network/src/service.rs         | 294 ++-------------------
 client/network/src/service/metrics.rs | 358 ++++++++++++++++++++++++++
 client/rpc/src/system/tests.rs        |   4 -
 client/service/src/builder.rs         |  69 +----
 client/service/src/lib.rs             |  59 +++--
 client/service/src/metrics.rs         | 180 ++++++++++---
 8 files changed, 571 insertions(+), 403 deletions(-)
 create mode 100644 client/network/src/service/metrics.rs

diff --git a/client/informant/src/lib.rs b/client/informant/src/lib.rs
index 3daf29a9f7837..a1f0ba9ae5fac 100644
--- a/client/informant/src/lib.rs
+++ b/client/informant/src/lib.rs
@@ -23,7 +23,7 @@ use futures::prelude::*;
 use log::{info, trace, warn};
 use parity_util_mem::MallocSizeOf;
 use sc_client_api::{BlockchainEvents, UsageProvider};
-use sc_network::{network_state::NetworkState, NetworkStatus};
+use sc_network::NetworkStatus;
 use sp_blockchain::HeaderMetadata;
 use sp_runtime::traits::{Block as BlockT, Header};
 use sp_transaction_pool::TransactionPool;
@@ -81,7 +81,7 @@ impl<T: TransactionPool + MallocSizeOf> TransactionPoolAndMaybeMallogSizeOf for
 /// Builds the informant and returns a `Future` that drives the informant.
 pub fn build<B: BlockT, C>(
 	client: Arc<C>,
-	network_status_sinks: Arc<status_sinks::StatusSinks<(NetworkStatus<B>, NetworkState)>>,
+	network_status_sinks: Arc<status_sinks::StatusSinks<NetworkStatus<B>>>,
 	pool: Arc<impl TransactionPoolAndMaybeMallogSizeOf>,
 	format: OutputFormat,
 ) -> impl futures::Future<Output = ()>
@@ -96,7 +96,7 @@ where
 	network_status_sinks.push(Duration::from_millis(5000), network_status_sink);
 
 	let display_notifications = network_status_stream
-		.for_each(move |(net_status, _)| {
+		.for_each(move |net_status| {
 			let info = client_1.usage_info();
 			if let Some(ref usage) = info.usage {
 				trace!(target: "usage", "Usage statistics: {}", usage);
diff --git a/client/network/src/network_state.rs b/client/network/src/network_state.rs
index 2e24e9c5a9f58..db2b6429304bb 100644
--- a/client/network/src/network_state.rs
+++ b/client/network/src/network_state.rs
@@ -43,10 +43,6 @@ pub struct NetworkState {
 	pub connected_peers: HashMap<String, Peer>,
 	/// List of node that we know of but that we're not connected to.
 	pub not_connected_peers: HashMap<String, NotConnectedPeer>,
-	/// The total number of bytes received.
-	pub total_bytes_inbound: u64,
-	/// The total number of bytes sent.
-	pub total_bytes_outbound: u64,
 	/// State of the peerset manager.
 	pub peerset: serde_json::Value,
 }
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index 4fa37c64c75e2..28af928060002 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -28,7 +28,7 @@
 //! which is then processed by [`NetworkWorker::poll`].
 
 use crate::{
-	ExHashT, NetworkStateInfo,
+	ExHashT, NetworkStateInfo, NetworkStatus,
 	behaviour::{self, Behaviour, BehaviourOut},
 	config::{parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig},
 	DhtEvent,
@@ -49,12 +49,8 @@ use libp2p::kad::record;
 use libp2p::ping::handler::PingFailure;
 use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent, protocols_handler::NodeHandlerWrapperError};
 use log::{error, info, trace, warn};
+use metrics::{Metrics, MetricSources, Histogram, HistogramVec};
 use parking_lot::Mutex;
-use prometheus_endpoint::{
-	register, Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, Opts,
-	PrometheusError, Registry, U64,
-	SourcedCounter, MetricSource
-};
 use sc_peerset::PeersetHandle;
 use sp_consensus::import_queue::{BlockImportError, BlockImportResult, ImportQueue, Link};
 use sp_runtime::{
@@ -80,6 +76,7 @@ use wasm_timer::Instant;
 
 pub use behaviour::{ResponseFailure, InboundFailure, RequestFailure, OutboundFailure};
 
+mod metrics;
 mod out_events;
 #[cfg(test)]
 mod tests;
@@ -365,10 +362,11 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 		// Initialize the metrics.
 		let metrics = match &params.metrics_registry {
 			Some(registry) => {
-				// Sourced metrics.
-				BandwidthCounters::register(registry, bandwidth.clone())?;
-				// Other (i.e. new) metrics.
-				Some(Metrics::register(registry)?)
+				Some(metrics::register(registry, MetricSources {
+					bandwidth: bandwidth.clone(),
+					major_syncing: is_major_syncing.clone(),
+					connected_peers: num_connected.clone(),
+				})?)
 			}
 			None => None
 		};
@@ -423,6 +421,19 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 		})
 	}
 
+	/// High-level network status information.
+	pub fn status(&self) -> NetworkStatus<B> {
+		NetworkStatus {
+			sync_state: self.sync_state(),
+			best_seen_block: self.best_seen_block(),
+			num_sync_peers: self.num_sync_peers(),
+			num_connected_peers: self.num_connected_peers(),
+			num_active_peers: self.num_active_peers(),
+			total_bytes_inbound: self.total_bytes_inbound(),
+			total_bytes_outbound: self.total_bytes_outbound(),
+		}
+	}
+
 	/// Returns the total number of bytes received so far.
 	pub fn total_bytes_inbound(&self) -> u64 {
 		self.service.bandwidth.total_inbound()
@@ -562,8 +573,6 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 			peer_id: Swarm::<B, H>::local_peer_id(&swarm).to_base58(),
 			listened_addresses: Swarm::<B, H>::listeners(&swarm).cloned().collect(),
 			external_addresses: Swarm::<B, H>::external_addresses(&swarm).cloned().collect(),
-			total_bytes_inbound: self.service.bandwidth.total_inbound(),
-			total_bytes_outbound: self.service.bandwidth.total_outbound(),
 			connected_peers,
 			not_connected_peers,
 			peerset: swarm.user_protocol_mut().peerset_debug_info(),
@@ -1175,265 +1184,6 @@ pub struct NetworkWorker<B: BlockT + 'static, H: ExHashT> {
 	peers_notifications_sinks: Arc<Mutex<HashMap<(PeerId, ConsensusEngineId), NotificationsSink>>>,
 }
 
-struct Metrics {
-	// This list is ordered alphabetically
-	connections_closed_total: CounterVec<U64>,
-	connections_opened_total: CounterVec<U64>,
-	distinct_peers_connections_closed_total: Counter<U64>,
-	distinct_peers_connections_opened_total: Counter<U64>,
-	import_queue_blocks_submitted: Counter<U64>,
-	import_queue_finality_proofs_submitted: Counter<U64>,
-	import_queue_justifications_submitted: Counter<U64>,
-	incoming_connections_errors_total: CounterVec<U64>,
-	incoming_connections_total: Counter<U64>,
-	is_major_syncing: Gauge<U64>,
-	issued_light_requests: Counter<U64>,
-	kademlia_query_duration: HistogramVec,
-	kademlia_random_queries_total: CounterVec<U64>,
-	kademlia_records_count: GaugeVec<U64>,
-	kademlia_records_sizes_total: GaugeVec<U64>,
-	kbuckets_num_nodes: GaugeVec<U64>,
-	listeners_local_addresses: Gauge<U64>,
-	listeners_errors_total: Counter<U64>,
-	notifications_sizes: HistogramVec,
-	notifications_streams_closed_total: CounterVec<U64>,
-	notifications_streams_opened_total: CounterVec<U64>,
-	peers_count: Gauge<U64>,
-	peerset_num_discovered: Gauge<U64>,
-	peerset_num_requested: Gauge<U64>,
-	pending_connections: Gauge<U64>,
-	pending_connections_errors_total: CounterVec<U64>,
-	requests_in_failure_total: CounterVec<U64>,
-	requests_in_success_total: HistogramVec,
-	requests_out_failure_total: CounterVec<U64>,
-	requests_out_success_total: HistogramVec,
-	requests_out_started_total: CounterVec<U64>,
-}
-
-/// The source for bandwidth metrics.
-#[derive(Clone)]
-struct BandwidthCounters(Arc<transport::BandwidthSinks>);
-
-impl BandwidthCounters {
-	fn register(registry: &Registry, sinks: Arc<transport::BandwidthSinks>)
-		-> Result<(), PrometheusError>
-	{
-		register(SourcedCounter::new(
-			&Opts::new(
-				"sub_libp2p_network_bytes_total",
-				"Total bandwidth usage"
-			).variable_label("direction"),
-			BandwidthCounters(sinks),
-		)?, registry)?;
-
-		Ok(())
-	}
-}
-
-impl MetricSource for BandwidthCounters {
-	type N = u64;
-
-	fn collect(&self, mut set: impl FnMut(&[&str], Self::N)) {
-		set(&[&"in"], self.0.total_inbound());
-		set(&[&"out"], self.0.total_outbound());
-	}
-}
-
-impl Metrics {
-	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
-		Ok(Self {
-			// This list is ordered alphabetically
-			connections_closed_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_connections_closed_total",
-					"Total number of connections closed, by direction and reason"
-				),
-				&["direction", "reason"]
-			)?, registry)?,
-			connections_opened_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_connections_opened_total",
-					"Total number of connections opened by direction"
-				),
-				&["direction"]
-			)?, registry)?,
-			distinct_peers_connections_closed_total: register(Counter::new(
-					"sub_libp2p_distinct_peers_connections_closed_total",
-					"Total number of connections closed with distinct peers"
-			)?, registry)?,
-			distinct_peers_connections_opened_total: register(Counter::new(
-					"sub_libp2p_distinct_peers_connections_opened_total",
-					"Total number of connections opened with distinct peers"
-			)?, registry)?,
-			import_queue_blocks_submitted: register(Counter::new(
-				"import_queue_blocks_submitted",
-				"Number of blocks submitted to the import queue.",
-			)?, registry)?,
-			import_queue_finality_proofs_submitted: register(Counter::new(
-				"import_queue_finality_proofs_submitted",
-				"Number of finality proofs submitted to the import queue.",
-			)?, registry)?,
-			import_queue_justifications_submitted: register(Counter::new(
-				"import_queue_justifications_submitted",
-				"Number of justifications submitted to the import queue.",
-			)?, registry)?,
-			incoming_connections_errors_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_incoming_connections_handshake_errors_total",
-					"Total number of incoming connections that have failed during the \
-					initial handshake"
-				),
-				&["reason"]
-			)?, registry)?,
-			incoming_connections_total: register(Counter::new(
-				"sub_libp2p_incoming_connections_total",
-				"Total number of incoming connections on the listening sockets"
-			)?, registry)?,
-			is_major_syncing: register(Gauge::new(
-				"sub_libp2p_is_major_syncing", "Whether the node is performing a major sync or not.",
-			)?, registry)?,
-			issued_light_requests: register(Counter::new(
-				"issued_light_requests",
-				"Number of light client requests that our node has issued.",
-			)?, registry)?,
-			kademlia_query_duration: register(HistogramVec::new(
-				HistogramOpts {
-					common_opts: Opts::new(
-						"sub_libp2p_kademlia_query_duration",
-						"Duration of Kademlia queries per query type"
-					),
-					buckets: prometheus_endpoint::exponential_buckets(0.5, 2.0, 10)
-						.expect("parameters are always valid values; qed"),
-				},
-				&["type"]
-			)?, registry)?,
-			kademlia_random_queries_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_kademlia_random_queries_total",
-					"Number of random Kademlia queries started"
-				),
-				&["protocol"]
-			)?, registry)?,
-			kademlia_records_count: register(GaugeVec::new(
-				Opts::new(
-					"sub_libp2p_kademlia_records_count",
-					"Number of records in the Kademlia records store"
-				),
-				&["protocol"]
-			)?, registry)?,
-			kademlia_records_sizes_total: register(GaugeVec::new(
-				Opts::new(
-					"sub_libp2p_kademlia_records_sizes_total",
-					"Total size of all the records in the Kademlia records store"
-				),
-				&["protocol"]
-			)?, registry)?,
-			kbuckets_num_nodes: register(GaugeVec::new(
-				Opts::new(
-					"sub_libp2p_kbuckets_num_nodes",
-					"Number of nodes in the Kademlia k-buckets"
-				),
-				&["protocol"]
-			)?, registry)?,
-			listeners_local_addresses: register(Gauge::new(
-				"sub_libp2p_listeners_local_addresses", "Number of local addresses we're listening on"
-			)?, registry)?,
-			listeners_errors_total: register(Counter::new(
-				"sub_libp2p_listeners_errors_total",
-				"Total number of non-fatal errors reported by a listener"
-			)?, registry)?,
-			notifications_sizes: register(HistogramVec::new(
-				HistogramOpts {
-					common_opts: Opts::new(
-						"sub_libp2p_notifications_sizes",
-						"Sizes of the notifications send to and received from all nodes"
-					),
-					buckets: prometheus_endpoint::exponential_buckets(64.0, 4.0, 8)
-						.expect("parameters are always valid values; qed"),
-				},
-				&["direction", "protocol"]
-			)?, registry)?,
-			notifications_streams_closed_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_notifications_streams_closed_total",
-					"Total number of notification substreams that have been closed"
-				),
-				&["protocol"]
-			)?, registry)?,
-			notifications_streams_opened_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_notifications_streams_opened_total",
-					"Total number of notification substreams that have been opened"
-				),
-				&["protocol"]
-			)?, registry)?,
-			peers_count: register(Gauge::new(
-				"sub_libp2p_peers_count", "Number of network gossip peers",
-			)?, registry)?,
-			peerset_num_discovered: register(Gauge::new(
-				"sub_libp2p_peerset_num_discovered", "Number of nodes stored in the peerset manager",
-			)?, registry)?,
-			peerset_num_requested: register(Gauge::new(
-				"sub_libp2p_peerset_num_requested", "Number of nodes that the peerset manager wants us to be connected to",
-			)?, registry)?,
-			pending_connections: register(Gauge::new(
-				"sub_libp2p_pending_connections",
-				"Number of connections in the process of being established",
-			)?, registry)?,
-			pending_connections_errors_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_pending_connections_errors_total",
-					"Total number of pending connection errors"
-				),
-				&["reason"]
-			)?, registry)?,
-			requests_in_failure_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_requests_in_failure_total",
-					"Total number of incoming requests that the node has failed to answer"
-				),
-				&["protocol", "reason"]
-			)?, registry)?,
-			requests_in_success_total: register(HistogramVec::new(
-				HistogramOpts {
-					common_opts: Opts::new(
-						"sub_libp2p_requests_in_success_total",
-						"Total number of requests received and answered"
-					),
-					buckets: prometheus_endpoint::exponential_buckets(0.001, 2.0, 16)
-						.expect("parameters are always valid values; qed"),
-				},
-				&["protocol"]
-			)?, registry)?,
-			requests_out_failure_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_requests_out_failure_total",
-					"Total number of requests that have failed"
-				),
-				&["protocol", "reason"]
-			)?, registry)?,
-			requests_out_success_total: register(HistogramVec::new(
-				HistogramOpts {
-					common_opts: Opts::new(
-						"sub_libp2p_requests_out_success_total",
-						"For successful requests, time between a request's start and finish"
-					),
-					buckets: prometheus_endpoint::exponential_buckets(0.001, 2.0, 16)
-						.expect("parameters are always valid values; qed"),
-				},
-				&["protocol"]
-			)?, registry)?,
-			requests_out_started_total: register(CounterVec::new(
-				Opts::new(
-					"sub_libp2p_requests_out_started_total",
-					"Total number of requests emitted"
-				),
-				&["protocol"]
-			)?, registry)?,
-		})
-	}
-}
-
 impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 	type Output = ();
 
@@ -1902,7 +1652,6 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 		this.is_major_syncing.store(is_major_syncing, Ordering::Relaxed);
 
 		if let Some(metrics) = this.metrics.as_ref() {
-			metrics.is_major_syncing.set(is_major_syncing as u64);
 			for (proto, num_entries) in this.network_service.num_kbuckets_entries() {
 				metrics.kbuckets_num_nodes.with_label_values(&[&proto.as_ref()]).set(num_entries as u64);
 			}
@@ -1912,7 +1661,6 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 			for (proto, num_entries) in this.network_service.kademlia_records_total_size() {
 				metrics.kademlia_records_sizes_total.with_label_values(&[&proto.as_ref()]).set(num_entries as u64);
 			}
-			metrics.peers_count.set(num_connected_peers as u64);
 			metrics.peerset_num_discovered.set(this.network_service.user_protocol().num_discovered_peers() as u64);
 			metrics.peerset_num_requested.set(this.network_service.user_protocol().requested_peers().count() as u64);
 			metrics.pending_connections.set(Swarm::network_info(&this.network_service).num_connections_pending as u64);
diff --git a/client/network/src/service/metrics.rs b/client/network/src/service/metrics.rs
new file mode 100644
index 0000000000000..bbb0ba8056615
--- /dev/null
+++ b/client/network/src/service/metrics.rs
@@ -0,0 +1,358 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::transport::BandwidthSinks;
+use prometheus_endpoint::{
+	self as prometheus,
+	Counter, CounterVec, Gauge, GaugeVec, HistogramOpts,
+	PrometheusError, Registry, U64, Opts,
+	SourcedCounter, SourcedGauge, MetricSource,
+};
+use std::{
+	str,
+	sync::{
+		atomic::{AtomicBool, AtomicUsize, Ordering},
+		Arc,
+	},
+};
+
+pub use prometheus_endpoint::{Histogram, HistogramVec};
+
+/// Registers all networking metrics with the given registry.
+pub fn register(registry: &Registry, sources: MetricSources) -> Result<Metrics, PrometheusError> {
+	BandwidthCounters::register(registry, sources.bandwidth)?;
+	MajorSyncingGauge::register(registry, sources.major_syncing)?;
+	NumConnectedGauge::register(registry, sources.connected_peers)?;
+	Metrics::register(registry)
+}
+
+/// Predefined metric sources that are fed directly into prometheus.
+pub struct MetricSources {
+	pub bandwidth: Arc<BandwidthSinks>,
+	pub major_syncing: Arc<AtomicBool>,
+	pub connected_peers: Arc<AtomicUsize>,
+}
+
+/// Dedicated metrics.
+pub struct Metrics {
+	// This list is ordered alphabetically
+	pub connections_closed_total: CounterVec<U64>,
+	pub connections_opened_total: CounterVec<U64>,
+	pub distinct_peers_connections_closed_total: Counter<U64>,
+	pub distinct_peers_connections_opened_total: Counter<U64>,
+	pub import_queue_blocks_submitted: Counter<U64>,
+	pub import_queue_finality_proofs_submitted: Counter<U64>,
+	pub import_queue_justifications_submitted: Counter<U64>,
+	pub incoming_connections_errors_total: CounterVec<U64>,
+	pub incoming_connections_total: Counter<U64>,
+	pub issued_light_requests: Counter<U64>,
+	pub kademlia_query_duration: HistogramVec,
+	pub kademlia_random_queries_total: CounterVec<U64>,
+	pub kademlia_records_count: GaugeVec<U64>,
+	pub kademlia_records_sizes_total: GaugeVec<U64>,
+	pub kbuckets_num_nodes: GaugeVec<U64>,
+	pub listeners_local_addresses: Gauge<U64>,
+	pub listeners_errors_total: Counter<U64>,
+	pub notifications_sizes: HistogramVec,
+	pub notifications_streams_closed_total: CounterVec<U64>,
+	pub notifications_streams_opened_total: CounterVec<U64>,
+	pub peerset_num_discovered: Gauge<U64>,
+	pub peerset_num_requested: Gauge<U64>,
+	pub pending_connections: Gauge<U64>,
+	pub pending_connections_errors_total: CounterVec<U64>,
+	pub requests_in_failure_total: CounterVec<U64>,
+	pub requests_in_success_total: HistogramVec,
+	pub requests_out_failure_total: CounterVec<U64>,
+	pub requests_out_success_total: HistogramVec,
+	pub requests_out_started_total: CounterVec<U64>,
+}
+
+impl Metrics {
+	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Self {
+			// This list is ordered alphabetically
+			connections_closed_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_connections_closed_total",
+					"Total number of connections closed, by direction and reason"
+				),
+				&["direction", "reason"]
+			)?, registry)?,
+			connections_opened_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_connections_opened_total",
+					"Total number of connections opened by direction"
+				),
+				&["direction"]
+			)?, registry)?,
+			distinct_peers_connections_closed_total: prometheus::register(Counter::new(
+					"sub_libp2p_distinct_peers_connections_closed_total",
+					"Total number of connections closed with distinct peers"
+			)?, registry)?,
+			distinct_peers_connections_opened_total: prometheus::register(Counter::new(
+					"sub_libp2p_distinct_peers_connections_opened_total",
+					"Total number of connections opened with distinct peers"
+			)?, registry)?,
+			import_queue_blocks_submitted: prometheus::register(Counter::new(
+				"import_queue_blocks_submitted",
+				"Number of blocks submitted to the import queue.",
+			)?, registry)?,
+			import_queue_finality_proofs_submitted: prometheus::register(Counter::new(
+				"import_queue_finality_proofs_submitted",
+				"Number of finality proofs submitted to the import queue.",
+			)?, registry)?,
+			import_queue_justifications_submitted: prometheus::register(Counter::new(
+				"import_queue_justifications_submitted",
+				"Number of justifications submitted to the import queue.",
+			)?, registry)?,
+			incoming_connections_errors_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_incoming_connections_handshake_errors_total",
+					"Total number of incoming connections that have failed during the \
+					initial handshake"
+				),
+				&["reason"]
+			)?, registry)?,
+			incoming_connections_total: prometheus::register(Counter::new(
+				"sub_libp2p_incoming_connections_total",
+				"Total number of incoming connections on the listening sockets"
+			)?, registry)?,
+			issued_light_requests: prometheus::register(Counter::new(
+				"issued_light_requests",
+				"Number of light client requests that our node has issued.",
+			)?, registry)?,
+			kademlia_query_duration: prometheus::register(HistogramVec::new(
+				HistogramOpts {
+					common_opts: Opts::new(
+						"sub_libp2p_kademlia_query_duration",
+						"Duration of Kademlia queries per query type"
+					),
+					buckets: prometheus::exponential_buckets(0.5, 2.0, 10)
+						.expect("parameters are always valid values; qed"),
+				},
+				&["type"]
+			)?, registry)?,
+			kademlia_random_queries_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_kademlia_random_queries_total",
+					"Number of random Kademlia queries started"
+				),
+				&["protocol"]
+			)?, registry)?,
+			kademlia_records_count: prometheus::register(GaugeVec::new(
+				Opts::new(
+					"sub_libp2p_kademlia_records_count",
+					"Number of records in the Kademlia records store"
+				),
+				&["protocol"]
+			)?, registry)?,
+			kademlia_records_sizes_total: prometheus::register(GaugeVec::new(
+				Opts::new(
+					"sub_libp2p_kademlia_records_sizes_total",
+					"Total size of all the records in the Kademlia records store"
+				),
+				&["protocol"]
+			)?, registry)?,
+			kbuckets_num_nodes: prometheus::register(GaugeVec::new(
+				Opts::new(
+					"sub_libp2p_kbuckets_num_nodes",
+					"Number of nodes in the Kademlia k-buckets"
+				),
+				&["protocol"]
+			)?, registry)?,
+			listeners_local_addresses: prometheus::register(Gauge::new(
+				"sub_libp2p_listeners_local_addresses", "Number of local addresses we're listening on"
+			)?, registry)?,
+			listeners_errors_total: prometheus::register(Counter::new(
+				"sub_libp2p_listeners_errors_total",
+				"Total number of non-fatal errors reported by a listener"
+			)?, registry)?,
+			notifications_sizes: prometheus::register(HistogramVec::new(
+				HistogramOpts {
+					common_opts: Opts::new(
+						"sub_libp2p_notifications_sizes",
+						"Sizes of the notifications send to and received from all nodes"
+					),
+					buckets: prometheus::exponential_buckets(64.0, 4.0, 8)
+						.expect("parameters are always valid values; qed"),
+				},
+				&["direction", "protocol"]
+			)?, registry)?,
+			notifications_streams_closed_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_notifications_streams_closed_total",
+					"Total number of notification substreams that have been closed"
+				),
+				&["protocol"]
+			)?, registry)?,
+			notifications_streams_opened_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_notifications_streams_opened_total",
+					"Total number of notification substreams that have been opened"
+				),
+				&["protocol"]
+			)?, registry)?,
+			peerset_num_discovered: prometheus::register(Gauge::new(
+				"sub_libp2p_peerset_num_discovered", "Number of nodes stored in the peerset manager",
+			)?, registry)?,
+			peerset_num_requested: prometheus::register(Gauge::new(
+				"sub_libp2p_peerset_num_requested", "Number of nodes that the peerset manager wants us to be connected to",
+			)?, registry)?,
+			pending_connections: prometheus::register(Gauge::new(
+				"sub_libp2p_pending_connections",
+				"Number of connections in the process of being established",
+			)?, registry)?,
+			pending_connections_errors_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_pending_connections_errors_total",
+					"Total number of pending connection errors"
+				),
+				&["reason"]
+			)?, registry)?,
+			requests_in_failure_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_requests_in_failure_total",
+					"Total number of incoming requests that the node has failed to answer"
+				),
+				&["protocol", "reason"]
+			)?, registry)?,
+			requests_in_success_total: prometheus::register(HistogramVec::new(
+				HistogramOpts {
+					common_opts: Opts::new(
+						"sub_libp2p_requests_in_success_total",
+						"Total number of requests received and answered"
+					),
+					buckets: prometheus::exponential_buckets(0.001, 2.0, 16)
+						.expect("parameters are always valid values; qed"),
+				},
+				&["protocol"]
+			)?, registry)?,
+			requests_out_failure_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_requests_out_failure_total",
+					"Total number of requests that have failed"
+				),
+				&["protocol", "reason"]
+			)?, registry)?,
+			requests_out_success_total: prometheus::register(HistogramVec::new(
+				HistogramOpts {
+					common_opts: Opts::new(
+						"sub_libp2p_requests_out_success_total",
+						"For successful requests, time between a request's start and finish"
+					),
+					buckets: prometheus::exponential_buckets(0.001, 2.0, 16)
+						.expect("parameters are always valid values; qed"),
+				},
+				&["protocol"]
+			)?, registry)?,
+			requests_out_started_total: prometheus::register(CounterVec::new(
+				Opts::new(
+					"sub_libp2p_requests_out_started_total",
+					"Total number of requests emitted"
+				),
+				&["protocol"]
+			)?, registry)?,
+		})
+	}
+}
+
+/// The bandwidth counter metric.
+#[derive(Clone)]
+pub struct BandwidthCounters(Arc<BandwidthSinks>);
+
+impl BandwidthCounters {
+	/// Registers the `BandwidthCounters` metric whose values are
+	/// obtained from the given sinks.
+	fn register(registry: &Registry, sinks: Arc<BandwidthSinks>) -> Result<(), PrometheusError> {
+		prometheus::register(SourcedCounter::new(
+			&Opts::new(
+				"sub_libp2p_network_bytes_total",
+				"Total bandwidth usage"
+			).variable_label("direction"),
+			BandwidthCounters(sinks),
+		)?, registry)?;
+
+		Ok(())
+	}
+}
+
+impl MetricSource for BandwidthCounters {
+	type N = u64;
+
+	fn collect(&self, mut set: impl FnMut(&[&str], Self::N)) {
+		set(&[&"in"], self.0.total_inbound());
+		set(&[&"out"], self.0.total_outbound());
+	}
+}
+
+/// The "major syncing" metric.
+#[derive(Clone)]
+pub struct MajorSyncingGauge(Arc<AtomicBool>);
+
+impl MajorSyncingGauge {
+	/// Registers the `MajorSyncGauge` metric whose value is
+	/// obtained from the given `AtomicBool`.
+	fn register(registry: &Registry, value: Arc<AtomicBool>) -> Result<(), PrometheusError> {
+		prometheus::register(SourcedGauge::new(
+			&Opts::new(
+				"sub_libp2p_is_major_syncing",
+				"Whether the node is performing a major sync or not.",
+			),
+			MajorSyncingGauge(value),
+		)?, registry)?;
+
+		Ok(())
+	}
+}
+
+impl MetricSource for MajorSyncingGauge {
+	type N = u64;
+
+	fn collect(&self, mut set: impl FnMut(&[&str], Self::N)) {
+		set(&[], self.0.load(Ordering::Relaxed) as u64);
+	}
+}
+
+/// The connected peers metric.
+#[derive(Clone)]
+pub struct NumConnectedGauge(Arc<AtomicUsize>);
+
+impl NumConnectedGauge {
+	/// Registers the `MajorSyncingGauge` metric whose value is
+	/// obtained from the given `AtomicUsize`.
+	fn register(registry: &Registry, value: Arc<AtomicUsize>) -> Result<(), PrometheusError> {
+		prometheus::register(SourcedGauge::new(
+			&Opts::new(
+				"sub_libp2p_peers_count",
+				"Number of connected peers",
+			),
+			NumConnectedGauge(value),
+		)?, registry)?;
+
+		Ok(())
+	}
+}
+
+impl MetricSource for NumConnectedGauge {
+	type N = u64;
+
+	fn collect(&self, mut set: impl FnMut(&[&str], Self::N)) {
+		set(&[], self.0.load(Ordering::Relaxed) as u64);
+	}
+}
+
diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs
index dfe1fcc415159..099504bb009e6 100644
--- a/client/rpc/src/system/tests.rs
+++ b/client/rpc/src/system/tests.rs
@@ -87,8 +87,6 @@ fn api<T: Into<Option<Status>>>(sync: T) -> System<Block> {
 						external_addresses: Default::default(),
 						connected_peers: Default::default(),
 						not_connected_peers: Default::default(),
-						total_bytes_inbound: 0,
-						total_bytes_outbound: 0,
 						peerset: serde_json::Value::Null,
 					}).unwrap());
 				},
@@ -282,8 +280,6 @@ fn system_network_state() {
 			external_addresses: Default::default(),
 			connected_peers: Default::default(),
 			not_connected_peers: Default::default(),
-			total_bytes_inbound: 0,
-			total_bytes_outbound: 0,
 			peerset: serde_json::Value::Null,
 		}
 	);
diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs
index 5faf0899aa2e3..f4046ab722ba7 100644
--- a/client/service/src/builder.rs
+++ b/client/service/src/builder.rs
@@ -17,10 +17,10 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::{
-	NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm,
+	error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm,
 	TelemetryConnectionSinks, RpcHandlers, NetworkStatusSinks,
 	start_rpc_servers, build_network_future, TransactionPoolAdapter, TaskManager, SpawnTaskHandle,
-	status_sinks, metrics::MetricsService,
+	metrics::MetricsService,
 	client::{light, Client, ClientConfig},
 	config::{Configuration, KeystoreConfig, PrometheusConfig},
 };
@@ -472,7 +472,9 @@ pub fn spawn_tasks<TBl, TBackend, TExPool, TRpc, TCl>(
 		transaction_pool,
 		rpc_extensions_builder,
 		remote_blockchain,
-		network, network_status_sinks, system_rpc_tx,
+		network,
+		network_status_sinks,
+		system_rpc_tx,
 		telemetry_connection_sinks,
 	} = params;
 
@@ -521,15 +523,13 @@ pub fn spawn_tasks<TBl, TBackend, TExPool, TRpc, TCl>(
 		MetricsService::new()
 	};
 
-	// Periodically notify the telemetry.
-	spawn_handle.spawn("telemetry-periodic-send", telemetry_periodic_send(
-		client.clone(), transaction_pool.clone(), metrics_service, network_status_sinks.clone()
-	));
-
-	// Periodically send the network state to the telemetry.
-	spawn_handle.spawn(
-		"telemetry-periodic-network-state",
-		telemetry_periodic_network_state(network_status_sinks.clone()),
+	// Periodically updated metrics and telemetry updates.
+	spawn_handle.spawn("telemetry-periodic-send",
+		metrics_service.run(
+			client.clone(),
+			transaction_pool.clone(),
+			network_status_sinks.clone()
+		)
 	);
 
 	// RPC
@@ -574,7 +574,7 @@ pub fn spawn_tasks<TBl, TBackend, TExPool, TRpc, TCl>(
 	// Spawn informant task
 	spawn_handle.spawn("informant", sc_informant::build(
 		client.clone(),
-		network_status_sinks.clone().0,
+		network_status_sinks.status.clone(),
 		transaction_pool.clone(),
 		config.informant_output_format,
 	));
@@ -606,47 +606,6 @@ async fn transaction_notifications<TBl, TExPool>(
 		.await;
 }
 
-// Periodically notify the telemetry.
-async fn telemetry_periodic_send<TBl, TExPool, TCl>(
-	client: Arc<TCl>,
-	transaction_pool: Arc<TExPool>,
-	mut metrics_service: MetricsService,
-	network_status_sinks: NetworkStatusSinks<TBl>,
-)
-	where
-		TBl: BlockT,
-		TCl: ProvideRuntimeApi<TBl> + UsageProvider<TBl>,
-		TExPool: MaintainedTransactionPool<Block=TBl, Hash = <TBl as BlockT>::Hash>,
-{
-	let (state_tx, state_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat1");
-	network_status_sinks.0.push(std::time::Duration::from_millis(5000), state_tx);
-	state_rx.for_each(move |(net_status, _)| {
-		let info = client.usage_info();
-		metrics_service.tick(
-			&info,
-			&transaction_pool.status(),
-			&net_status,
-		);
-		ready(())
-	}).await;
-}
-
-async fn telemetry_periodic_network_state<TBl: BlockT>(
-	network_status_sinks: NetworkStatusSinks<TBl>,
-) {
-	// Periodically send the network state to the telemetry.
-	let (netstat_tx, netstat_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat2");
-	network_status_sinks.0.push(std::time::Duration::from_secs(30), netstat_tx);
-	netstat_rx.for_each(move |(_, network_state)| {
-		telemetry!(
-			SUBSTRATE_INFO;
-			"system.network_state";
-			"state" => network_state,
-		);
-		ready(())
-	}).await;
-}
-
 fn build_telemetry<TBl: BlockT>(
 	config: &mut Configuration,
 	endpoints: sc_telemetry::TelemetryEndpoints,
@@ -887,7 +846,7 @@ pub fn build_network<TBl, TExPool, TImpQu, TCl>(
 	let has_bootnodes = !network_params.network_config.boot_nodes.is_empty();
 	let network_mut = sc_network::NetworkWorker::new(network_params)?;
 	let network = network_mut.service().clone();
-	let network_status_sinks = NetworkStatusSinks::new(Arc::new(status_sinks::StatusSinks::new()));
+	let network_status_sinks = NetworkStatusSinks::new();
 
 	let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc");
 
diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs
index d19b9f5ea247d..fac09beb8bd60 100644
--- a/client/service/src/lib.rs
+++ b/client/service/src/lib.rs
@@ -126,24 +126,37 @@ impl RpcHandlers {
 /// Sinks to propagate network status updates.
 /// For each element, every time the `Interval` fires we push an element on the sender.
 #[derive(Clone)]
-pub struct NetworkStatusSinks<Block: BlockT>(
-	Arc<status_sinks::StatusSinks<(NetworkStatus<Block>, NetworkState)>>,
-);
+pub struct NetworkStatusSinks<Block: BlockT> {
+	status: Arc<status_sinks::StatusSinks<NetworkStatus<Block>>>,
+	state: Arc<status_sinks::StatusSinks<NetworkState>>,
+}
 
 impl<Block: BlockT> NetworkStatusSinks<Block> {
-	fn new(
-		sinks: Arc<status_sinks::StatusSinks<(NetworkStatus<Block>, NetworkState)>>
-	) -> Self {
-		Self(sinks)
+	fn new() -> Self {
+		Self {
+			status: Arc::new(status_sinks::StatusSinks::new()),
+			state: Arc::new(status_sinks::StatusSinks::new()),
+		}
 	}
 
-	/// Returns a receiver that periodically receives a status of the network.
-	pub fn network_status(&self, interval: Duration)
-		-> TracingUnboundedReceiver<(NetworkStatus<Block>, NetworkState)> {
+	/// Returns a receiver that periodically yields a [`NetworkStatus`].
+	pub fn status_stream(&self, interval: Duration)
+		-> TracingUnboundedReceiver<NetworkStatus<Block>>
+	{
 		let (sink, stream) = tracing_unbounded("mpsc_network_status");
-		self.0.push(interval, sink);
+		self.status.push(interval, sink);
+		stream
+	}
+
+	/// Returns a receiver that periodically yields a [`NetworkState`].
+	pub fn state_stream(&self, interval: Duration)
+		-> TracingUnboundedReceiver<NetworkState>
+	{
+		let (sink, stream) = tracing_unbounded("mpsc_network_state");
+		self.state.push(interval, sink);
 		stream
 	}
+
 }
 
 /// Sinks to propagate telemetry connection established events.
@@ -319,20 +332,16 @@ async fn build_network_future<
 			// the network.
 			_ = (&mut network).fuse() => {}
 
-			// At a regular interval, we send the state of the network on what is called
-			// the "status sinks".
-			ready_sink = status_sinks.0.next().fuse() => {
-				let status = NetworkStatus {
-					sync_state: network.sync_state(),
-					best_seen_block: network.best_seen_block(),
-					num_sync_peers: network.num_sync_peers(),
-					num_connected_peers: network.num_connected_peers(),
-					num_active_peers: network.num_active_peers(),
-					total_bytes_inbound: network.total_bytes_inbound(),
-					total_bytes_outbound: network.total_bytes_outbound(),
-				};
-				let state = network.network_state();
-				ready_sink.send((status, state));
+			// At a regular interval, we send high-level status as well as
+			// detailed state information of the network on what are called
+			// "status sinks".
+
+			status_sink = status_sinks.status.next().fuse() => {
+				status_sink.send(network.status());
+			}
+
+			state_sink = status_sinks.state.next().fuse() => {
+				state_sink.send(network.network_state());
 			}
 		}
 	}
diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs
index 90a77667581bf..0af393b53f517 100644
--- a/client/service/src/metrics.rs
+++ b/client/service/src/metrics.rs
@@ -18,14 +18,19 @@
 
 use std::{convert::TryFrom, time::SystemTime};
 
-use crate::{NetworkStatus, config::Configuration};
+use crate::{NetworkStatus, NetworkState, NetworkStatusSinks, config::Configuration};
+use futures_timer::Delay;
 use prometheus_endpoint::{register, Gauge, U64, Registry, PrometheusError, Opts, GaugeVec};
 use sc_telemetry::{telemetry, SUBSTRATE_INFO};
+use sp_api::ProvideRuntimeApi;
 use sp_runtime::traits::{NumberFor, Block, SaturatedConversion, UniqueSaturatedInto};
-use sp_transaction_pool::PoolStatus;
+use sp_transaction_pool::{PoolStatus, MaintainedTransactionPool};
 use sp_utils::metrics::register_globals;
-use sc_client_api::ClientInfo;
+use sp_utils::mpsc::TracingUnboundedReceiver;
+use sc_client_api::{ClientInfo, UsageProvider};
 use sc_network::config::Role;
+use std::sync::Arc;
+use std::time::Duration;
 use wasm_timer::Instant;
 
 struct PrometheusMetrics {
@@ -99,6 +104,9 @@ impl PrometheusMetrics {
 	}
 }
 
+/// A `MetricsService` periodically sends general client and
+/// network state to the telemetry as well as (optionally)
+/// a Prometheus endpoint.
 pub struct MetricsService {
 	metrics: Option<PrometheusMetrics>,
 	last_update: Instant,
@@ -107,6 +115,8 @@ pub struct MetricsService {
 }
 
 impl MetricsService {
+	/// Creates a `MetricsService` that only sends information
+	/// to the telemetry.
 	pub fn new() -> Self {
 		MetricsService {
 			metrics: None,
@@ -116,6 +126,8 @@ impl MetricsService {
 		}
 	}
 
+	/// Creates a `MetricsService` that sends metrics
+	/// to prometheus alongside the telemetry.
 	pub fn with_prometheus(
 		registry: &Registry,
 		config: &Configuration,
@@ -141,60 +153,109 @@ impl MetricsService {
 		})
 	}
 
-	pub fn tick<T: Block>(
+	/// Returns a never-ending `Future` that performs the
+	/// metric and telemetry updates with information from
+	/// the given sources.
+	pub async fn run<TBl, TExPool, TCl>(
+		mut self,
+		client: Arc<TCl>,
+		transactions: Arc<TExPool>,
+		network: NetworkStatusSinks<TBl>,
+	) where
+		TBl: Block,
+		TCl: ProvideRuntimeApi<TBl> + UsageProvider<TBl>,
+		TExPool: MaintainedTransactionPool<Block = TBl, Hash = <TBl as Block>::Hash>,
+	{
+		let mut timer = Delay::new(Duration::from_secs(0));
+		let timer_interval = Duration::from_secs(5);
+
+		// Metric and telemetry update interval.
+		let net_status_interval = timer_interval;
+		let net_state_interval = Duration::from_secs(30);
+
+		// Source of network information.
+		let mut net_status_rx = Some(network.status_stream(net_status_interval));
+		let mut net_state_rx = Some(network.state_stream(net_state_interval));
+
+		loop {
+			// Wait for the next tick of the timer.
+			(&mut timer).await;
+
+			// Try to get the latest network information.
+			let mut net_status = None;
+			let mut net_state = None;
+			if let Some(rx) = net_status_rx.as_mut() {
+				match Self::latest(rx) {
+					Ok(status) => { net_status = status; }
+					Err(()) => { net_status_rx = None; }
+				}
+			}
+			if let Some(rx) = net_state_rx.as_mut() {
+				match Self::latest(rx) {
+					Ok(state) => { net_state = state; }
+					Err(()) => { net_state_rx = None; }
+				}
+			}
+
+			// Update / Send the metrics.
+			self.update(
+				&client.usage_info(),
+				&transactions.status(),
+				net_status,
+				net_state,
+			);
+
+			// Schedule next tick.
+			timer.reset(timer_interval);
+		}
+	}
+
+	// Try to get the latest value from a receiver, dropping intermediate values.
+	fn latest<T>(rx: &mut TracingUnboundedReceiver<T>) -> Result<Option<T>, ()> {
+		let mut value = None;
+
+		while let Ok(next) = rx.try_next() {
+			match next {
+				Some(v) => {
+					value = Some(v)
+				}
+				None => {
+					log::error!("Receiver closed unexpectedly.");
+					return Err(())
+				}
+			}
+		}
+
+		Ok(value)
+	}
+
+	fn update<T: Block>(
 		&mut self,
 		info: &ClientInfo<T>,
 		txpool_status: &PoolStatus,
-		net_status: &NetworkStatus<T>,
+		net_status: Option<NetworkStatus<T>>,
+		net_state: Option<NetworkState>,
 	) {
 		let now = Instant::now();
 		let elapsed = (now - self.last_update).as_secs();
+		self.last_update = now;
 
 		let best_number = info.chain.best_number.saturated_into::<u64>();
 		let best_hash = info.chain.best_hash;
-		let num_peers = net_status.num_connected_peers;
 		let finalized_number: u64 = info.chain.finalized_number.saturated_into::<u64>();
-		let total_bytes_inbound = net_status.total_bytes_inbound;
-		let total_bytes_outbound = net_status.total_bytes_outbound;
-		let best_seen_block = net_status
-			.best_seen_block
-			.map(|num: NumberFor<T>| num.unique_saturated_into() as u64);
-
-		let diff_bytes_inbound = total_bytes_inbound - self.last_total_bytes_inbound;
-		let diff_bytes_outbound = total_bytes_outbound - self.last_total_bytes_outbound;
-		let (avg_bytes_per_sec_inbound, avg_bytes_per_sec_outbound) =
-			if elapsed > 0 {
-				self.last_total_bytes_inbound = total_bytes_inbound;
-				self.last_total_bytes_outbound = total_bytes_outbound;
-				(diff_bytes_inbound / elapsed, diff_bytes_outbound / elapsed)
-			} else {
-				(diff_bytes_inbound, diff_bytes_outbound)
-			};
-		self.last_update = now;
 
+		// Update/send metrics that are always available.
 		telemetry!(
 			SUBSTRATE_INFO;
 			"system.interval";
-			"peers" => num_peers,
 			"height" => best_number,
 			"best" => ?best_hash,
 			"txcount" => txpool_status.ready,
 			"finalized_height" => finalized_number,
 			"finalized_hash" => ?info.chain.finalized_hash,
-			"bandwidth_download" => avg_bytes_per_sec_inbound,
-			"bandwidth_upload" => avg_bytes_per_sec_outbound,
 			"used_state_cache_size" => info.usage.as_ref()
 				.map(|usage| usage.memory.state_cache.as_bytes())
 				.unwrap_or(0),
-			"used_db_cache_size" => info.usage.as_ref()
-				.map(|usage| usage.memory.database_cache.as_bytes())
-				.unwrap_or(0),
-			"disk_read_per_sec" => info.usage.as_ref()
-				.map(|usage| usage.io.bytes_read)
-				.unwrap_or(0),
-			"disk_write_per_sec" => info.usage.as_ref()
-				.map(|usage| usage.io.bytes_written)
-				.unwrap_or(0),
 		);
 
 		if let Some(metrics) = self.metrics.as_ref() {
@@ -213,10 +274,6 @@ impl MetricsService {
 
 			metrics.ready_transactions_number.set(txpool_status.ready as u64);
 
-			if let Some(best_seen_block) = best_seen_block {
-				metrics.block_height.with_label_values(&["sync_target"]).set(best_seen_block);
-			}
-
 			if let Some(info) = info.usage.as_ref() {
 				metrics.database_cache.set(info.memory.database_cache.as_bytes() as u64);
 				metrics.state_cache.set(info.memory.state_cache.as_bytes() as u64);
@@ -232,5 +289,50 @@ impl MetricsService {
 				);
 			}
 		}
+
+		// Update/send network status information, if any.
+		if let Some(net_status) = net_status {
+			let num_peers = net_status.num_connected_peers;
+			let total_bytes_inbound = net_status.total_bytes_inbound;
+			let total_bytes_outbound = net_status.total_bytes_outbound;
+
+			let diff_bytes_inbound = total_bytes_inbound - self.last_total_bytes_inbound;
+			let diff_bytes_outbound = total_bytes_outbound - self.last_total_bytes_outbound;
+			let (avg_bytes_per_sec_inbound, avg_bytes_per_sec_outbound) =
+				if elapsed > 0 {
+					self.last_total_bytes_inbound = total_bytes_inbound;
+					self.last_total_bytes_outbound = total_bytes_outbound;
+					(diff_bytes_inbound / elapsed, diff_bytes_outbound / elapsed)
+				} else {
+					(diff_bytes_inbound, diff_bytes_outbound)
+				};
+
+			telemetry!(
+				SUBSTRATE_INFO;
+				"system.interval";
+				"peers" => num_peers,
+				"bandwidth_download" => avg_bytes_per_sec_inbound,
+				"bandwidth_upload" => avg_bytes_per_sec_outbound,
+			);
+
+			if let Some(metrics) = self.metrics.as_ref() {
+				let best_seen_block = net_status
+					.best_seen_block
+					.map(|num: NumberFor<T>| num.unique_saturated_into() as u64);
+
+				if let Some(best_seen_block) = best_seen_block {
+					metrics.block_height.with_label_values(&["sync_target"]).set(best_seen_block);
+				}
+			}
+		}
+
+		// Send network state information, if any.
+		if let Some(net_state) = net_state {
+			telemetry!(
+				SUBSTRATE_INFO;
+				"system.network_state";
+				"state" => net_state,
+			);
+		}
 	}
 }

From 4dee242fe58aa0314bf4d3608428eab4c7b328a2 Mon Sep 17 00:00:00 2001
From: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>
Date: Mon, 7 Sep 2020 06:15:01 -0400
Subject: [PATCH 044/122] Node template complete import pipeline (#7014)

* Use complete import pipeline

* Line length

Co-authored-by: Dan Forbes <dan@danforbes.dev>
---
 bin/node-template/node/src/service.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs
index 5984d67322333..8fa935c375021 100644
--- a/bin/node-template/node/src/service.rs
+++ b/bin/node-template/node/src/service.rs
@@ -27,7 +27,12 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 	sp_consensus::DefaultImportQueue<Block, FullClient>,
 	sc_transaction_pool::FullPool<Block, FullClient>,
 	(
-		sc_finality_grandpa::GrandpaBlockImport<FullBackend, Block, FullClient, FullSelectChain>,
+		sc_consensus_aura::AuraBlockImport<
+			Block,
+			FullClient,
+			sc_finality_grandpa::GrandpaBlockImport<FullBackend, Block, FullClient, FullSelectChain>,
+			AuraPair
+		>,
 		sc_finality_grandpa::LinkHalf<Block, FullClient, FullSelectChain>
 	)
 >, ServiceError> {
@@ -56,7 +61,7 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 
 	let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair, _, _>(
 		sc_consensus_aura::slot_duration(&*client)?,
-		aura_block_import,
+		aura_block_import.clone(),
 		Some(Box::new(grandpa_block_import.clone())),
 		None,
 		client.clone(),
@@ -69,7 +74,7 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 	Ok(sc_service::PartialComponents {
 		client, backend, task_manager, import_queue, keystore, select_chain, transaction_pool,
 		inherent_data_providers,
-		other: (grandpa_block_import, grandpa_link),
+		other: (aura_block_import, grandpa_link),
 	})
 }
 

From 68bdfec62f693c478be01e0e527e7d6e5d1d0c9c Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Mon, 7 Sep 2020 12:17:28 +0200
Subject: [PATCH 045/122] client/authority-discovery: Throttle DHT requests
 (#7018)

* client/authority-discovery: Throttle DHT requests

Instead of passing one DHT query for each authority down to the network
every query interval, only pass MAX_IN_FLIGHT_LOOKUPS at a given point
in time, triggering new ones when previous ones return.

* client/authority-discovery/worker/test: Fix wrong constant
---
 client/authority-discovery/src/error.rs       |   6 +-
 client/authority-discovery/src/worker.rs      | 145 ++++++----
 .../authority-discovery/src/worker/tests.rs   | 266 +++++++++++-------
 3 files changed, 254 insertions(+), 163 deletions(-)

diff --git a/client/authority-discovery/src/error.rs b/client/authority-discovery/src/error.rs
index b1358485c37ee..48bcdf33114b1 100644
--- a/client/authority-discovery/src/error.rs
+++ b/client/authority-discovery/src/error.rs
@@ -34,10 +34,8 @@ pub enum Error {
 	HashingAuthorityId(libp2p::core::multiaddr::multihash::EncodeError),
 	/// Failed calling into the Substrate runtime.
 	CallingRuntime(sp_blockchain::Error),
-	/// From the Dht we only get the hashed authority id. In order to retrieve the actual authority id and to ensure it
-	/// is actually an authority, we match the hash against the hash of the authority id of all other authorities. This
-	/// error is the result of the above failing.
-	MatchingHashedAuthorityIdWithAuthorityId,
+	/// Received a dht record with a key that does not match any in-flight awaited keys.
+	ReceivingUnexpectedRecord,
 	/// Failed to set the authority discovery peerset priority group in the peerset module.
 	SettingPeersetPriorityGroup(String),
 	/// Failed to encode a protobuf payload.
diff --git a/client/authority-discovery/src/worker.rs b/client/authority-discovery/src/worker.rs
index 629ea4fb2f423..ff4d12dadd988 100644
--- a/client/authority-discovery/src/worker.rs
+++ b/client/authority-discovery/src/worker.rs
@@ -35,6 +35,7 @@ use libp2p::{core::multiaddr, multihash::Multihash};
 use log::{debug, error, log_enabled};
 use prometheus_endpoint::{Counter, CounterVec, Gauge, Opts, U64, register};
 use prost::Message;
+use rand::{seq::SliceRandom, thread_rng};
 use sc_client_api::blockchain::HeaderBackend;
 use sc_network::{
 	config::MultiaddrWithPeerId,
@@ -70,6 +71,9 @@ const AUTHORITIES_PRIORITY_GROUP_NAME: &'static str = "authorities";
 /// Maximum number of addresses cached per authority. Additional addresses are discarded.
 const MAX_ADDRESSES_PER_AUTHORITY: usize = 10;
 
+/// Maximum number of in-flight DHT lookups at any given point in time.
+const MAX_IN_FLIGHT_LOOKUPS: usize = 8;
+
 /// Role an authority discovery module can run as.
 pub enum Role {
 	/// Actual authority as well as a reference to its key store.
@@ -137,12 +141,17 @@ where
 
 	/// Interval to be proactive, publishing own addresses.
 	publish_interval: Interval,
-	/// Interval on which to query for addresses of other authorities.
+	/// Interval at which to request addresses of authorities, refilling the pending lookups queue.
 	query_interval: Interval,
 	/// Interval on which to set the peerset priority group to a new random
 	/// set of addresses.
 	priority_group_set_interval: Interval,
 
+	/// Queue of throttled lookups pending to be passed to the network.
+	pending_lookups: Vec<AuthorityId>,
+	/// Set of in-flight lookups.
+	in_flight_lookups: HashMap<libp2p::kad::record::Key, AuthorityId>,
+
 	addr_cache: addr_cache::AddrCache,
 
 	metrics: Option<Metrics>,
@@ -183,8 +192,8 @@ where
 			Duration::from_secs(12 * 60 * 60),
 		);
 
-		// External addresses of other authorities can change at any given point in time. The
-		// interval on which to query for external addresses of other authorities is a trade off
+		// External addresses of remote authorities can change at any given point in time. The
+		// interval on which to trigger new queries for the current authorities is a trade off
 		// between efficiency and performance.
 		let query_interval_start = Instant::now() + LIBP2P_KADEMLIA_BOOTSTRAP_TIME;
 		let query_interval_duration = Duration::from_secs(10 * 60);
@@ -193,9 +202,9 @@ where
 		// Querying 500 [`AuthorityId`]s takes ~1m on the Kusama DHT (10th of August 2020) when
 		// comparing `authority_discovery_authority_addresses_requested_total` and
 		// `authority_discovery_dht_event_received`. With that in mind set the peerset priority
-		// group on the same interval as the [`query_interval`] above, just delayed by 2 minutes.
+		// group on the same interval as the [`query_interval`] above, just delayed by 5 minutes.
 		let priority_group_set_interval = interval_at(
-			query_interval_start + Duration::from_secs(2 * 60),
+			query_interval_start + Duration::from_secs(5 * 60),
 			query_interval_duration,
 		);
 
@@ -229,6 +238,8 @@ where
 			publish_interval,
 			query_interval,
 			priority_group_set_interval,
+			pending_lookups: Vec::new(),
+			in_flight_lookups: HashMap::new(),
 			addr_cache,
 			role,
 			metrics,
@@ -270,7 +281,9 @@ where
 
 		if let Some(metrics) = &self.metrics {
 			metrics.publish.inc();
-			metrics.amount_last_published.set(addresses.len() as u64);
+			metrics.amount_addresses_last_published.set(
+				addresses.len().try_into().unwrap_or(std::u64::MAX),
+			);
 		}
 
 		let mut serialized_addresses = vec![];
@@ -314,15 +327,9 @@ where
 		Ok(())
 	}
 
-	fn request_addresses_of_others(&mut self) -> Result<()> {
+	fn refill_pending_lookups_queue(&mut self) -> Result<()> {
 		let id = BlockId::hash(self.client.info().best_hash);
 
-		let authorities = self
-			.client
-			.runtime_api()
-			.authorities(&id)
-			.map_err(Error::CallingRuntime)?;
-
 		let local_keys = match &self.role {
 			Role::Authority(key_store) => {
 				key_store.read()
@@ -333,21 +340,52 @@ where
 			Role::Sentry => HashSet::new(),
 		};
 
-		for authority_id in authorities.iter() {
-			// Make sure we don't look up our own keys.
-			if !local_keys.contains(authority_id.as_ref()) {
-				if let Some(metrics) = &self.metrics {
-					metrics.request.inc();
-				}
+		let mut authorities = self
+			.client
+			.runtime_api()
+			.authorities(&id)
+			.map_err(Error::CallingRuntime)?
+			.into_iter()
+			.filter(|id| !local_keys.contains(id.as_ref()))
+			.collect();
 
-				self.network
-					.get_value(&hash_authority_id(authority_id.as_ref()));
-			}
+		self.addr_cache.retain_ids(&authorities);
+
+		authorities.shuffle(&mut thread_rng());
+		self.pending_lookups = authorities;
+		// Ignore all still in-flight lookups. Those that are still in-flight are likely stalled as
+		// query interval ticks are far enough apart for all lookups to succeed.
+		self.in_flight_lookups.clear();
+
+		if let Some(metrics) = &self.metrics {
+			metrics.requests_pending.set(
+				self.pending_lookups.len().try_into().unwrap_or(std::u64::MAX),
+			);
 		}
 
 		Ok(())
 	}
 
+	fn start_new_lookups(&mut self) {
+		while self.in_flight_lookups.len() < MAX_IN_FLIGHT_LOOKUPS {
+			let authority_id = match self.pending_lookups.pop() {
+				Some(authority) => authority,
+				None => return,
+			};
+			let hash = hash_authority_id(authority_id.as_ref());
+			self.network
+				.get_value(&hash);
+			self.in_flight_lookups.insert(hash, authority_id);
+
+			if let Some(metrics) = &self.metrics {
+				metrics.requests.inc();
+				metrics.requests_pending.set(
+					self.pending_lookups.len().try_into().unwrap_or(std::u64::MAX),
+				);
+			}
+		}
+	}
+
 	/// Handle incoming Dht events.
 	///
 	/// Returns either:
@@ -385,10 +423,17 @@ where
 						metrics.dht_event_received.with_label_values(&["value_not_found"]).inc();
 					}
 
-					debug!(
-						target: LOG_TARGET,
-						"Value for hash '{:?}' not found on Dht.", hash
-					)
+					if self.in_flight_lookups.remove(&hash).is_some() {
+						debug!(
+							target: LOG_TARGET,
+							"Value for hash '{:?}' not found on Dht.", hash
+						)
+					} else {
+						debug!(
+							target: LOG_TARGET,
+							"Received 'ValueNotFound' for unexpected hash '{:?}'.", hash
+						)
+					}
 				},
 				Some(DhtEvent::ValuePut(hash)) => {
 					if let Some(metrics) = &self.metrics {
@@ -434,23 +479,9 @@ where
 			}
 		})?.ok_or(Error::ReceivingDhtValueFoundEventWithNoRecords)?;
 
-		let authorities = {
-			let block_id = BlockId::hash(self.client.info().best_hash);
-			// From the Dht we only get the hashed authority id. In order to retrieve the actual
-			// authority id and to ensure it is actually an authority, we match the hash against the
-			// hash of the authority id of all other authorities.
-			let authorities = self.client.runtime_api().authorities(&block_id)?;
-			self.addr_cache.retain_ids(&authorities);
-			authorities
-				.into_iter()
-				.map(|id| (hash_authority_id(id.as_ref()), id))
-				.collect::<HashMap<_, _>>()
-		};
-
-		// Check if the event origins from an authority in the current or next authority set.
-		let authority_id: &AuthorityId = authorities
-			.get(&remote_key)
-			.ok_or(Error::MatchingHashedAuthorityIdWithAuthorityId)?;
+		let authority_id: AuthorityId = self.in_flight_lookups
+			.remove(&remote_key)
+			.ok_or(Error::ReceivingUnexpectedRecord)?;
 
 		let local_peer_id = self.network.local_peer_id();
 
@@ -463,7 +494,7 @@ where
 				let signature = AuthoritySignature::decode(&mut &signature[..])
 					.map_err(Error::EncodingDecodingScale)?;
 
-				if !AuthorityPair::verify(&signature, &addresses, authority_id) {
+				if !AuthorityPair::verify(&signature, &addresses, &authority_id) {
 					return Err(Error::VerifyingDhtPayload);
 				}
 
@@ -503,7 +534,7 @@ where
 			.collect();
 
 		if !remote_addresses.is_empty() {
-			self.addr_cache.insert(authority_id.clone(), remote_addresses);
+			self.addr_cache.insert(authority_id, remote_addresses);
 			if let Some(metrics) = &self.metrics {
 				metrics.known_authorities_count.set(
 					self.addr_cache.num_ids().try_into().unwrap_or(std::u64::MAX)
@@ -610,15 +641,15 @@ where
 			}
 		}
 
-		// Request addresses of authorities.
+		// Request addresses of authorities, refilling the pending lookups queue.
 		if let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) {
 			// Register waker of underlying task for next interval.
 			while let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) {}
 
-			if let Err(e) = self.request_addresses_of_others() {
+			if let Err(e) = self.refill_pending_lookups_queue() {
 				error!(
 					target: LOG_TARGET,
-					"Failed to request addresses of authorities: {:?}", e,
+					"Failed to refill pending lookups queue: {:?}", e,
 				);
 			}
 		}
@@ -652,6 +683,8 @@ where
 			}
 		}
 
+		self.start_new_lookups();
+
 		Poll::Pending
 	}
 }
@@ -712,8 +745,9 @@ fn interval_at(start: Instant, duration: Duration) -> Interval {
 #[derive(Clone)]
 pub(crate) struct Metrics {
 	publish: Counter<U64>,
-	amount_last_published: Gauge<U64>,
-	request: Counter<U64>,
+	amount_addresses_last_published: Gauge<U64>,
+	requests: Counter<U64>,
+	requests_pending: Gauge<U64>,
 	dht_event_received: CounterVec<U64>,
 	handle_value_found_event_failure: Counter<U64>,
 	known_authorities_count: Gauge<U64>,
@@ -730,7 +764,7 @@ impl Metrics {
 				)?,
 				registry,
 			)?,
-			amount_last_published: register(
+			amount_addresses_last_published: register(
 				Gauge::new(
 					"authority_discovery_amount_external_addresses_last_published",
 					"Number of external addresses published when authority discovery last \
@@ -738,7 +772,7 @@ impl Metrics {
 				)?,
 				registry,
 			)?,
-			request: register(
+			requests: register(
 				Counter::new(
 					"authority_discovery_authority_addresses_requested_total",
 					"Number of times authority discovery has requested external addresses of a \
@@ -746,6 +780,13 @@ impl Metrics {
 				)?,
 				registry,
 			)?,
+			requests_pending: register(
+				Gauge::new(
+					"authority_discovery_authority_address_requests_pending",
+					"Number of pending authority address requests."
+				)?,
+				registry,
+			)?,
 			dht_event_received: register(
 				CounterVec::new(
 					Opts::new(
diff --git a/client/authority-discovery/src/worker/tests.rs b/client/authority-discovery/src/worker/tests.rs
index baa6bd0fc7d62..28192283054d1 100644
--- a/client/authority-discovery/src/worker/tests.rs
+++ b/client/authority-discovery/src/worker/tests.rs
@@ -221,6 +221,41 @@ impl NetworkStateInfo for TestNetwork {
 	}
 }
 
+fn build_dht_event(
+	addresses: Vec<Multiaddr>,
+	public_key: AuthorityId,
+	key_store: &BareCryptoStorePtr,
+) -> (libp2p::kad::record::Key, Vec<u8>) {
+	let mut serialized_addresses = vec![];
+	schema::AuthorityAddresses {
+		addresses: addresses.into_iter().map(|a| a.to_vec()).collect()
+	}.encode(&mut serialized_addresses)
+		.map_err(Error::EncodingProto)
+		.unwrap();
+
+	let signature = key_store.read()
+		.sign_with(
+			key_types::AUTHORITY_DISCOVERY,
+			&public_key.clone().into(),
+			serialized_addresses.as_slice(),
+		)
+		.map_err(|_| Error::Signing)
+		.unwrap();
+
+	let mut signed_addresses = vec![];
+	schema::SignedAuthorityAddresses {
+		addresses: serialized_addresses.clone(),
+		signature,
+	}
+	.encode(&mut signed_addresses)
+		.map_err(Error::EncodingProto)
+		.unwrap();
+
+	let key = hash_authority_id(&public_key.to_raw_vec());
+	let value = signed_addresses;
+	(key, value)
+}
+
 #[test]
 fn new_registers_metrics() {
 	let (_dht_event_tx, dht_event_rx) = channel(1000);
@@ -247,7 +282,7 @@ fn new_registers_metrics() {
 }
 
 #[test]
-fn request_addresses_of_others_triggers_dht_get_query() {
+fn triggers_dht_get_query() {
 	let _ = ::env_logger::try_init();
 	let (_dht_event_tx, dht_event_rx) = channel(1000);
 
@@ -262,7 +297,6 @@ fn request_addresses_of_others_triggers_dht_get_query() {
 	let network: Arc<TestNetwork> = Arc::new(Default::default());
 	let key_store = KeyStore::new();
 
-
 	let (_to_worker, from_service) = mpsc::channel(0);
 	let mut worker = Worker::new(
 		from_service,
@@ -274,7 +308,12 @@ fn request_addresses_of_others_triggers_dht_get_query() {
 		None,
 	);
 
-	worker.request_addresses_of_others().unwrap();
+	worker.refill_pending_lookups_queue().unwrap();
+
+	futures::executor::block_on(futures::future::poll_fn(|cx| {
+		assert_eq!(Poll::Pending, worker.poll_unpin(cx));
+		Poll::Ready(())
+	}));
 
 	// Expect authority discovery to request new records from the dht.
 	assert_eq!(network.get_value_call.lock().unwrap().len(), 2);
@@ -352,6 +391,9 @@ fn publish_discover_cycle() {
 	dht_event_tx.try_send(dht_event).unwrap();
 
 	let f = |cx: &mut Context<'_>| -> Poll<()> {
+		worker.refill_pending_lookups_queue().unwrap();
+		worker.start_new_lookups();
+
 		// Make authority discovery handle the event.
 		if let Poll::Ready(e) = worker.handle_dht_events(cx) {
 			panic!("Unexpected error: {:?}", e);
@@ -547,40 +589,11 @@ fn never_add_own_address_to_priority_group() {
 		))
 	};
 
-	let dht_event = {
-		let addresses = vec![
-			sentry_multiaddr.to_vec(),
-			random_multiaddr.to_vec(),
-		];
-
-		let mut serialized_addresses = vec![];
-		schema::AuthorityAddresses { addresses }
-		.encode(&mut serialized_addresses)
-			.map_err(Error::EncodingProto)
-			.unwrap();
-
-		let signature = validator_key_store.read()
-			.sign_with(
-				key_types::AUTHORITY_DISCOVERY,
-				&validator_public.clone().into(),
-				serialized_addresses.as_slice(),
-			)
-			.map_err(|_| Error::Signing)
-			.unwrap();
-
-		let mut signed_addresses = vec![];
-		schema::SignedAuthorityAddresses {
-			addresses: serialized_addresses.clone(),
-			signature,
-		}
-			.encode(&mut signed_addresses)
-			.map_err(Error::EncodingProto)
-			.unwrap();
-
-		let key = hash_authority_id(&validator_public.to_raw_vec());
-		let value = signed_addresses;
-		(key, value)
-	};
+	let dht_event = build_dht_event(
+		vec![sentry_multiaddr, random_multiaddr.clone()],
+		validator_public.into(),
+		&validator_key_store,
+	);
 
 	let (_dht_event_tx, dht_event_rx) = channel(1);
 	let sentry_test_api = Arc::new(TestApi {
@@ -599,6 +612,9 @@ fn never_add_own_address_to_priority_group() {
 		None,
 	);
 
+	sentry_worker.refill_pending_lookups_queue().unwrap();
+	sentry_worker.start_new_lookups();
+
 	sentry_worker.handle_dht_value_found_event(vec![dht_event]).unwrap();
 	sentry_worker.set_priority_group().unwrap();
 
@@ -625,43 +641,19 @@ fn limit_number_of_addresses_added_to_cache_per_authority() {
 		.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)
 		.unwrap();
 
-	let dht_event = {
-		let addresses = (0..100).map(|_| {
-			let peer_id = PeerId::random();
-			let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
-			address.with(multiaddr::Protocol::P2p(
-				peer_id.into(),
-			)).to_vec()
-		}).collect();
-
-		let mut serialized_addresses = vec![];
-		schema::AuthorityAddresses { addresses }
-		.encode(&mut serialized_addresses)
-			.map_err(Error::EncodingProto)
-			.unwrap();
-
-		let signature = remote_key_store.read()
-			.sign_with(
-				key_types::AUTHORITY_DISCOVERY,
-				&remote_public.clone().into(),
-				serialized_addresses.as_slice(),
-			)
-			.map_err(|_| Error::Signing)
-			.unwrap();
-
-		let mut signed_addresses = vec![];
-		schema::SignedAuthorityAddresses {
-			addresses: serialized_addresses.clone(),
-			signature,
-		}
-			.encode(&mut signed_addresses)
-			.map_err(Error::EncodingProto)
-			.unwrap();
+	let addresses = (0..100).map(|_| {
+		let peer_id = PeerId::random();
+		let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
+		address.with(multiaddr::Protocol::P2p(
+			peer_id.into(),
+		))
+	}).collect();
 
-		let key = hash_authority_id(&remote_public.to_raw_vec());
-		let value = signed_addresses;
-		(key, value)
-	};
+	let dht_event = build_dht_event(
+		addresses,
+		remote_public.into(),
+		&remote_key_store,
+	);
 
 	let (_dht_event_tx, dht_event_rx) = channel(1);
 
@@ -676,6 +668,9 @@ fn limit_number_of_addresses_added_to_cache_per_authority() {
 		None,
 	);
 
+	worker.refill_pending_lookups_queue().unwrap();
+	worker.start_new_lookups();
+
 	worker.handle_dht_value_found_event(vec![dht_event]).unwrap();
 	assert_eq!(
 		MAX_ADDRESSES_PER_AUTHORITY,
@@ -700,40 +695,14 @@ fn do_not_cache_addresses_without_peer_id() {
 
 	let multiaddr_without_peer_id: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
 
-	let dht_event = {
-		let addresses = vec![
-			multiaddr_with_peer_id.to_vec(),
-			multiaddr_without_peer_id.to_vec(),
-		];
-
-		let mut serialized_addresses = vec![];
-		schema::AuthorityAddresses { addresses }
-		.encode(&mut serialized_addresses)
-			.map_err(Error::EncodingProto)
-			.unwrap();
-
-		let signature = remote_key_store.read()
-			.sign_with(
-				key_types::AUTHORITY_DISCOVERY,
-				&remote_public.clone().into(),
-				serialized_addresses.as_slice(),
-			)
-			.map_err(|_| Error::Signing)
-			.unwrap();
-
-		let mut signed_addresses = vec![];
-		schema::SignedAuthorityAddresses {
-			addresses: serialized_addresses.clone(),
-			signature,
-		}
-			.encode(&mut signed_addresses)
-			.map_err(Error::EncodingProto)
-			.unwrap();
-
-		let key = hash_authority_id(&remote_public.to_raw_vec());
-		let value = signed_addresses;
-		(key, value)
-	};
+	let dht_event = build_dht_event(
+		vec![
+			multiaddr_with_peer_id.clone(),
+			multiaddr_without_peer_id,
+		],
+		remote_public.into(),
+		&remote_key_store,
+	);
 
 	let (_dht_event_tx, dht_event_rx) = channel(1);
 	let local_test_api = Arc::new(TestApi {
@@ -754,6 +723,9 @@ fn do_not_cache_addresses_without_peer_id() {
 		None,
 	);
 
+	local_worker.refill_pending_lookups_queue().unwrap();
+	local_worker.start_new_lookups();
+
 	local_worker.handle_dht_value_found_event(vec![dht_event]).unwrap();
 
 	assert_eq!(
@@ -826,3 +798,83 @@ fn addresses_to_publish_respects_existing_p2p_protocol() {
 		"Expected Multiaddr from `TestNetwork` to not be altered.",
 	);
 }
+
+#[test]
+fn lookup_throttling() {
+	let remote_multiaddr = {
+		let peer_id = PeerId::random();
+		let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap();
+
+		address.with(multiaddr::Protocol::P2p(
+			peer_id.into(),
+		))
+	};
+	let remote_key_store = KeyStore::new();
+	let remote_public_keys: Vec<AuthorityId> = (0..20).map(|_| {
+		remote_key_store
+			.write()
+			.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)
+			.unwrap().into()
+	}).collect();
+	let remote_hash_to_key = remote_public_keys.iter()
+		.map(|k| (hash_authority_id(k.as_ref()), k.clone()))
+		.collect::<HashMap<_, _>>();
+
+
+	let (mut dht_event_tx, dht_event_rx) = channel(1);
+	let (_to_worker, from_service) = mpsc::channel(0);
+	let network = Arc::new(TestNetwork::default());
+	let mut worker = Worker::new(
+		from_service,
+		Arc::new(TestApi { authorities: remote_public_keys.clone() }),
+		network.clone(),
+		vec![],
+		dht_event_rx.boxed(),
+		Role::Sentry,
+		None,
+	);
+
+	futures::executor::block_on(futures::future::poll_fn(|cx| {
+		worker.refill_pending_lookups_queue().unwrap();
+
+		// Assert worker to trigger MAX_IN_FLIGHT_LOOKUPS lookups.
+		assert_eq!(Poll::Pending, worker.poll_unpin(cx));
+		assert_eq!(worker.pending_lookups.len(), remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS);
+		assert_eq!(worker.in_flight_lookups.len(), MAX_IN_FLIGHT_LOOKUPS);
+		assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
+
+		// Make first lookup succeed.
+		let remote_hash = network.get_value_call.lock().unwrap().pop().unwrap();
+		let remote_key: AuthorityId = remote_hash_to_key.get(&remote_hash).unwrap().clone();
+		let dht_event = {
+			let (key, value) = build_dht_event(vec![remote_multiaddr.clone()], remote_key, &remote_key_store);
+			sc_network::DhtEvent::ValueFound(vec![(key, value)])
+		};
+		dht_event_tx.try_send(dht_event).expect("Channel has capacity of 1.");
+
+		// Assert worker to trigger another lookup.
+		assert_eq!(Poll::Pending, worker.poll_unpin(cx));
+		assert_eq!(worker.pending_lookups.len(), remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS - 1);
+		assert_eq!(worker.in_flight_lookups.len(), MAX_IN_FLIGHT_LOOKUPS);
+		assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
+
+		// Make second one fail.
+		let remote_hash = network.get_value_call.lock().unwrap().pop().unwrap();
+		let dht_event = sc_network::DhtEvent::ValueNotFound(remote_hash);
+		dht_event_tx.try_send(dht_event).expect("Channel has capacity of 1.");
+
+		// Assert worker to trigger another lookup.
+		assert_eq!(Poll::Pending, worker.poll_unpin(cx));
+		assert_eq!(worker.pending_lookups.len(), remote_public_keys.len() - MAX_IN_FLIGHT_LOOKUPS - 2);
+		assert_eq!(worker.in_flight_lookups.len(), MAX_IN_FLIGHT_LOOKUPS);
+		assert_eq!(network.get_value_call.lock().unwrap().len(), MAX_IN_FLIGHT_LOOKUPS);
+
+		worker.refill_pending_lookups_queue().unwrap();
+
+		// Assert worker to restock pending lookups and forget about in-flight lookups.
+		assert_eq!(worker.pending_lookups.len(), remote_public_keys.len());
+		assert_eq!(worker.in_flight_lookups.len(), 0);
+
+		Poll::Ready(())
+	}));
+}

From 8c922583a77513e7b0cece9efac8a72638ff37a9 Mon Sep 17 00:00:00 2001
From: Dan Forbes <dan@danforbes.dev>
Date: Mon, 7 Sep 2020 08:38:14 -0700
Subject: [PATCH 046/122] Update Nicks docs to clarify that it is not
 production-ready (#6990)

---
 frame/nicks/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs
index ce0d65d881669..8a38b827f8611 100644
--- a/frame/nicks/src/lib.rs
+++ b/frame/nicks/src/lib.rs
@@ -22,8 +22,10 @@
 //!
 //! ## Overview
 //!
-//! Nicks is a trivial module for keeping track of account names on-chain. It makes no effort to
-//! create a name hierarchy, be a DNS replacement or provide reverse lookups.
+//! Nicks is a non-production-ready module for keeping track of account names on-chain. It makes no
+//! effort to create a name hierarchy, be a DNS replacement or provide reverse lookups. Furthermore,
+//! the weights attached to this module's dispatchable functions are for demonstration purposes only
+//! and have not been designed to be economically secure.
 //!
 //! ## Interface
 //!

From 6deea30574074fc1a037667556af284744cc589e Mon Sep 17 00:00:00 2001
From: cheme <emericchevalier.pro@gmail.com>
Date: Tue, 8 Sep 2020 09:27:10 +0200
Subject: [PATCH 047/122] Ignore wasm_gc for debug build. (#6962)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Ignore gc for debug build.

* alternate implementation

* Update utils/wasm-builder/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* fix

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
---
 docs/README.adoc                       |  2 +-
 utils/wasm-builder/src/lib.rs          | 32 ++++++++++++++++++--------
 utils/wasm-builder/src/wasm_project.rs | 25 ++++++++++++--------
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/docs/README.adoc b/docs/README.adoc
index d1daeed07b5dc..e1ed86c2d5269 100644
--- a/docs/README.adoc
+++ b/docs/README.adoc
@@ -321,7 +321,7 @@ we support multiple environment variables:
 * `TRIGGER_WASM_BUILD` - Can be set to trigger a WASM build. On subsequent calls the value of the variable
                          needs to change. As WASM builder instructs `cargo` to watch for file changes
                          this environment variable should only be required in certain circumstances.
-* `WASM_TARGET_DIRECTORY` - Will copy any build WASM binary to the given directory. The path needs
+* `WASM_TARGET_DIRECTORY` - Will copy release build WASM binary to the given directory. The path needs
                             to be absolute.
 * `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary.
 * `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build.
diff --git a/utils/wasm-builder/src/lib.rs b/utils/wasm-builder/src/lib.rs
index ab64db56fec34..f1a1c7729a070 100644
--- a/utils/wasm-builder/src/lib.rs
+++ b/utils/wasm-builder/src/lib.rs
@@ -168,17 +168,29 @@ pub fn build_project_with_default_rustflags(
 		default_rustflags,
 	);
 
+	let (wasm_binary, wasm_binary_bloaty) = if let Some(wasm_binary) = wasm_binary {
+		(
+			wasm_binary.wasm_binary_path_escaped(),
+			bloaty.wasm_binary_bloaty_path_escaped(),
+		)
+	} else {
+		(
+			bloaty.wasm_binary_bloaty_path_escaped(),
+			bloaty.wasm_binary_bloaty_path_escaped(),
+		)
+	};
+	
 	write_file_if_changed(
-		file_name.into(),
-		format!(
-			r#"
-				pub const WASM_BINARY: Option<&[u8]> = Some(include_bytes!("{wasm_binary}"));
-				pub const WASM_BINARY_BLOATY: Option<&[u8]> = Some(include_bytes!("{wasm_binary_bloaty}"));
-			"#,
-			wasm_binary = wasm_binary.wasm_binary_path_escaped(),
-			wasm_binary_bloaty = bloaty.wasm_binary_bloaty_path_escaped(),
-		),
-	);
+			file_name.into(),
+			format!(
+				r#"
+					pub const WASM_BINARY: Option<&[u8]> = Some(include_bytes!("{wasm_binary}"));
+					pub const WASM_BINARY_BLOATY: Option<&[u8]> = Some(include_bytes!("{wasm_binary_bloaty}"));
+				"#,
+				wasm_binary = wasm_binary,
+				wasm_binary_bloaty = wasm_binary_bloaty,
+			),
+		);
 }
 
 /// Checks if the build of the WASM binary should be skipped.
diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs
index 6f8f47881b03f..4c927f7bdeabd 100644
--- a/utils/wasm-builder/src/wasm_project.rs
+++ b/utils/wasm-builder/src/wasm_project.rs
@@ -91,7 +91,7 @@ impl Drop for WorkspaceLock {
 pub fn create_and_compile(
 	cargo_manifest: &Path,
 	default_rustflags: &str,
-) -> (WasmBinary, WasmBinaryBloaty) {
+) -> (Option<WasmBinary>, WasmBinaryBloaty) {
 	let wasm_workspace_root = get_wasm_workspace_root();
 	let wasm_workspace = wasm_workspace_root.join("wbuild");
 
@@ -113,7 +113,9 @@ pub fn create_and_compile(
 		&wasm_workspace,
 	);
 
-	copy_wasm_to_target_directory(cargo_manifest, &wasm_binary);
+	wasm_binary.as_ref().map(|wasm_binary|
+		copy_wasm_to_target_directory(cargo_manifest, wasm_binary)
+	);
 
 	generate_rerun_if_changed_instructions(cargo_manifest, &project, &wasm_workspace);
 
@@ -469,18 +471,23 @@ fn compact_wasm_file(
 	project: &Path,
 	cargo_manifest: &Path,
 	wasm_workspace: &Path,
-) -> (WasmBinary, WasmBinaryBloaty) {
-	let target = if is_release_build() { "release" } else { "debug" };
+) -> (Option<WasmBinary>, WasmBinaryBloaty) {
+	let is_release_build = is_release_build();
+	let target = if is_release_build { "release" } else { "debug" };
 	let wasm_binary = get_wasm_binary_name(cargo_manifest);
 	let wasm_file = wasm_workspace.join("target/wasm32-unknown-unknown")
 		.join(target)
 		.join(format!("{}.wasm", wasm_binary));
-	let wasm_compact_file = project.join(format!("{}.compact.wasm", wasm_binary));
-
-	wasm_gc::garbage_collect_file(&wasm_file, &wasm_compact_file)
-		.expect("Failed to compact generated WASM binary.");
+	let wasm_compact_file = if is_release_build {
+		let wasm_compact_file = project.join(format!("{}.compact.wasm", wasm_binary));
+		wasm_gc::garbage_collect_file(&wasm_file, &wasm_compact_file)
+			.expect("Failed to compact generated WASM binary.");
+		Some(WasmBinary(wasm_compact_file))
+	} else {
+		None
+	};
 
-	(WasmBinary(wasm_compact_file), WasmBinaryBloaty(wasm_file))
+	(wasm_compact_file, WasmBinaryBloaty(wasm_file))
 }
 
 /// Custom wrapper for a [`cargo_metadata::Package`] to store it in

From 8f4db26dedb0b407c405a2b3976eebb668370dff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Tue, 8 Sep 2020 10:32:37 +0200
Subject: [PATCH 048/122] Make `--file` optional for `generate-node-key`
 (#7043)

This pr makes the `--file` argument optional to `generate-node-key`.
If the argument is not given, the secret node key will be printed to
`stdout`. The public node key will always be printed to `stderr`.
---
 bin/utils/subkey/src/lib.rs                  |  3 ++-
 client/cli/src/commands/generate_node_key.rs | 17 +++++++++++------
 client/cli/src/commands/key.rs               |  5 +++--
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/bin/utils/subkey/src/lib.rs b/bin/utils/subkey/src/lib.rs
index bb89541d5b14e..15f7bf538c4b4 100644
--- a/bin/utils/subkey/src/lib.rs
+++ b/bin/utils/subkey/src/lib.rs
@@ -31,7 +31,8 @@ use sp_core::crypto::Ss58Codec;
 	about = "Utility for generating and restoring with Substrate keys",
 )]
 pub enum Subkey {
-	/// Generate a random node libp2p key, save it to file and print its peer ID
+	/// Generate a random node libp2p key, save it to file or print it to stdout
+	/// and print its peer ID to stderr.
 	GenerateNodeKey(GenerateNodeKeyCmd),
 
 	/// Generate a random account
diff --git a/client/cli/src/commands/generate_node_key.rs b/client/cli/src/commands/generate_node_key.rs
index 197e0eb5d9034..ad292e4712d84 100644
--- a/client/cli/src/commands/generate_node_key.rs
+++ b/client/cli/src/commands/generate_node_key.rs
@@ -26,26 +26,31 @@ use libp2p::identity::{ed25519 as libp2p_ed25519, PublicKey};
 #[derive(Debug, StructOpt)]
 #[structopt(
 	name = "generate-node-key",
-	about = "Generate a random node libp2p key, save it to file and print its peer ID"
+	about = "Generate a random node libp2p key, save it to \
+			 file or print it to stdout and print its peer ID to stderr"
 )]
 pub struct GenerateNodeKeyCmd {
 	/// Name of file to save secret key to.
+	///
+	/// If not given, the secret key is printed to stdout.
 	#[structopt(long)]
-	file: PathBuf,
+	file: Option<PathBuf>,
 }
 
 impl GenerateNodeKeyCmd {
 	/// Run the command
 	pub fn run(&self) -> Result<(), Error> {
-		let file = &self.file;
-
 		let keypair = libp2p_ed25519::Keypair::generate();
 		let secret = keypair.secret();
 		let peer_id = PublicKey::Ed25519(keypair.public()).into_peer_id();
+		let secret_hex = hex::encode(secret.as_ref());
 
-		fs::write(file, hex::encode(secret.as_ref()))?;
+		match &self.file {
+			Some(file) => fs::write(file, secret_hex)?,
+			None => print!("{}", secret_hex),
+		}
 
-		println!("{}", peer_id);
+		eprintln!("{}", peer_id);
 
 		Ok(())
 	}
diff --git a/client/cli/src/commands/key.rs b/client/cli/src/commands/key.rs
index 50142208b881d..930acd7925ac6 100644
--- a/client/cli/src/commands/key.rs
+++ b/client/cli/src/commands/key.rs
@@ -28,10 +28,11 @@ use super::{
 	generate_node_key::GenerateNodeKeyCmd,
 };
 
-/// key utilities for the cli.
+/// Key utilities for the cli.
 #[derive(Debug, StructOpt)]
 pub enum KeySubcommand {
-	/// Generate a random node libp2p key, save it to file and print its peer ID
+	/// Generate a random node libp2p key, save it to file or print it to stdout
+	/// and print its peer ID to stderr.
 	GenerateNodeKey(GenerateNodeKeyCmd),
 
 	/// Generate a random account

From 2bc69432ec96e2ff55bbaa8e72c3f51664195d53 Mon Sep 17 00:00:00 2001
From: Benjamin Kampmann <ben@gnunicorn.org>
Date: Tue, 8 Sep 2020 10:52:04 +0200
Subject: [PATCH 049/122] Downgrade wabt = 0.9.1 (#7042)

---
 Cargo.lock                    | 8 ++++----
 bin/node/executor/Cargo.toml  | 2 +-
 bin/node/testing/Cargo.toml   | 2 +-
 client/executor/Cargo.toml    | 2 +-
 primitives/sandbox/Cargo.toml | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9f29feece6d86..edf9eef4d796f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9753,9 +9753,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 
 [[package]]
 name = "wabt"
-version = "0.9.2"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c5c5c1286c6e578416982609f47594265f9d489f9b836157d403ad605a46693"
+checksum = "94b5f5d6984ca42df66280baa8a15ac188a173ddaf4580b574a98931c01920e7"
 dependencies = [
  "serde",
  "serde_derive",
@@ -9765,9 +9765,9 @@ dependencies = [
 
 [[package]]
 name = "wabt-sys"
-version = "0.7.2"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01c695f98f7eb81fd4e2f6b65301ccc916a950dc2265eeefc4d376b34ce666df"
+checksum = "b064c81821100adb4b71923cecfc67fef083db21c3bbd454b0162c7ffe63eeaa"
 dependencies = [
  "cc",
  "cmake",
diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml
index d8fb2e4078bd1..84a2cf377e6f0 100644
--- a/bin/node/executor/Cargo.toml
+++ b/bin/node/executor/Cargo.toml
@@ -41,7 +41,7 @@ sp-application-crypto = { version = "2.0.0-rc6", path = "../../../primitives/app
 sp-runtime = { version = "2.0.0-rc6", path = "../../../primitives/runtime" }
 sp-externalities = { version = "0.8.0-rc6", path = "../../../primitives/externalities" }
 substrate-test-client = { version = "2.0.0-rc6", path = "../../../test-utils/client" }
-wabt = "0.9.2"
+wabt = "0.9.1"
 
 [features]
 wasmtime = [
diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml
index 23bf10336dcfe..1d4d6ccaa63f8 100644
--- a/bin/node/testing/Cargo.toml
+++ b/bin/node/testing/Cargo.toml
@@ -39,7 +39,7 @@ substrate-test-client = { version = "2.0.0-rc6", path = "../../../test-utils/cli
 pallet-timestamp = { version = "2.0.0-rc6", path = "../../../frame/timestamp" }
 pallet-transaction-payment = { version = "2.0.0-rc6", path = "../../../frame/transaction-payment" }
 pallet-treasury = { version = "2.0.0-rc6", path = "../../../frame/treasury" }
-wabt = "0.9.2"
+wabt = "0.9.1"
 sp-api = { version = "2.0.0-rc6", path = "../../../primitives/api" }
 sp-finality-tracker = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/finality-tracker" }
 sp-timestamp = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/timestamp" }
diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml
index f963068ea37b2..029e01923ac70 100644
--- a/client/executor/Cargo.toml
+++ b/client/executor/Cargo.toml
@@ -37,7 +37,7 @@ libsecp256k1 = "0.3.4"
 
 [dev-dependencies]
 assert_matches = "1.3.0"
-wabt = "0.9.2"
+wabt = "0.9.1"
 hex-literal = "0.3.1"
 sc-runtime-test = { version = "2.0.0-rc6", path = "runtime-test" }
 substrate-test-runtime = { version = "2.0.0-rc6", path = "../../test-utils/runtime" }
diff --git a/primitives/sandbox/Cargo.toml b/primitives/sandbox/Cargo.toml
index 98376c77464d5..0ee2feea2b94e 100755
--- a/primitives/sandbox/Cargo.toml
+++ b/primitives/sandbox/Cargo.toml
@@ -20,7 +20,7 @@ sp-wasm-interface = { version = "2.0.0-rc6", default-features = false, path = ".
 codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false }
 
 [dev-dependencies]
-wabt = "0.9.2"
+wabt = "0.9.1"
 assert_matches = "1.3.0"
 
 [features]

From 52ae515714e23448c61b8d35ec4ec54fc1bf9e35 Mon Sep 17 00:00:00 2001
From: joshua-mir <joshua@parity.io>
Date: Tue, 8 Sep 2020 11:41:27 +0200
Subject: [PATCH 050/122] Add metadata shadows to multisig pallet (#7029)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add metadata shadows to multisig pallet

* Update frame/multisig/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
---
 frame/multisig/src/lib.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/frame/multisig/src/lib.rs b/frame/multisig/src/lib.rs
index f8f6e8ed63bc9..72a0f7cd070a2 100644
--- a/frame/multisig/src/lib.rs
+++ b/frame/multisig/src/lib.rs
@@ -262,6 +262,16 @@ decl_module! {
 		/// Deposit one of this module's events by using the default implementation.
 		fn deposit_event() = default;
 
+		/// The base amount of currency needed to reserve for creating a multisig execution or to store
+		/// a dispatch call for later.
+		const DepositBase: BalanceOf<T> = T::DepositBase::get();
+
+		/// The amount of currency needed per unit threshold when creating a multisig execution.
+		const DepositFactor: BalanceOf<T> = T::DepositFactor::get();
+
+		/// The maximum amount of signatories allowed for a given multisig.
+		const MaxSignatories: u16 = T::MaxSignatories::get();
+
 		/// Immediately dispatch a multi-signature call using a single approval from the caller.
 		///
 		/// The dispatch origin for this call must be _Signed_.

From 27aaf0a706524141ac93b99adaa8c75574198c4e Mon Sep 17 00:00:00 2001
From: Nikita Puzankov <humb1t@yandex.ru>
Date: Tue, 8 Sep 2020 13:56:48 +0400
Subject: [PATCH 051/122] Fix broken link to democracy pallet. (#7026)

Old link was broken, and I put a new one.
---
 frame/sudo/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frame/sudo/README.md b/frame/sudo/README.md
index fb8d1974c121a..c2bea4f81bcb8 100644
--- a/frame/sudo/README.md
+++ b/frame/sudo/README.md
@@ -61,10 +61,10 @@ You need to set an initial superuser account as the sudo `key`.
 
 ## Related Modules
 
-* [Democracy](../pallet_democracy/index.html)
+* [Democracy](https://github.com/paritytech/substrate/blob/master/frame/democracy/README.md)
 
 [`Call`]: ./enum.Call.html
 [`Trait`]: ./trait.Trait.html
 [`Origin`]: https://docs.substrate.dev/docs/substrate-types
 
-License: Apache-2.0
\ No newline at end of file
+License: Apache-2.0

From 49191fd19d168970d0677a9fb855860474527c11 Mon Sep 17 00:00:00 2001
From: Gavin Wood <gavin@parity.io>
Date: Tue, 8 Sep 2020 11:59:12 +0200
Subject: [PATCH 052/122] Revert "Fix broken link to democracy pallet. (#7026)"
 (#7047)

This reverts commit 008cb24c50f8b1d0b6f66dc5d90182cd1e501da2.
---
 frame/sudo/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frame/sudo/README.md b/frame/sudo/README.md
index c2bea4f81bcb8..fb8d1974c121a 100644
--- a/frame/sudo/README.md
+++ b/frame/sudo/README.md
@@ -61,10 +61,10 @@ You need to set an initial superuser account as the sudo `key`.
 
 ## Related Modules
 
-* [Democracy](https://github.com/paritytech/substrate/blob/master/frame/democracy/README.md)
+* [Democracy](../pallet_democracy/index.html)
 
 [`Call`]: ./enum.Call.html
 [`Trait`]: ./trait.Trait.html
 [`Origin`]: https://docs.substrate.dev/docs/substrate-types
 
-License: Apache-2.0
+License: Apache-2.0
\ No newline at end of file

From 79cc67d35306b9583dc7fea749825da7ea1a4292 Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Tue, 8 Sep 2020 11:59:37 +0200
Subject: [PATCH 053/122] Update the service tasks Grafana dashboard (#7038)

---
 .../substrate-service-tasks.json              | 400 +++++++-----------
 1 file changed, 143 insertions(+), 257 deletions(-)

diff --git a/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json b/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json
index 245071c210bfc..539fdec086a37 100644
--- a/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json
+++ b/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json
@@ -44,14 +44,14 @@
       {
         "datasource": "$data_source",
         "enable": true,
-        "expr": "increase(${metric_namespace}_tasks_ended_total{reason=\"panic\", instance=~\"${nodename}\"}[5m])",
+        "expr": "increase(${metric_namespace}_tasks_ended_total{reason=\"panic\", instance=~\"${nodename}\"}[10m])",
         "hide": true,
         "iconColor": "rgba(255, 96, 96, 1)",
         "limit": 100,
         "name": "Task panics",
         "rawQuery": "SELECT\n  extract(epoch from time_column) AS time,\n  text_column as text,\n  tags_column as tags\nFROM\n  metric_table\nWHERE\n  $__timeFilter(time_column)\n",
         "showIn": 0,
-        "step": "",
+        "step": "10m",
         "tags": [],
         "textFormat": "{{instance}} - {{task_name}}",
         "titleFormat": "Panic!",
@@ -60,12 +60,12 @@
       {
         "datasource": "$data_source",
         "enable": true,
-        "expr": "changes(${metric_namespace}_process_start_time_seconds{instance=~\"${nodename}\"}[5m])",
+        "expr": "changes(${metric_namespace}_process_start_time_seconds{instance=~\"${nodename}\"}[10m])",
         "hide": false,
         "iconColor": "#8AB8FF",
         "name": "Node reboots",
         "showIn": 0,
-        "step": "",
+        "step": "10m",
         "textFormat": "{{instance}}",
         "titleFormat": "Reboots"
       }
@@ -75,7 +75,7 @@
   "gnetId": null,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1594822742772,
+  "iteration": 1599471940817,
   "links": [],
   "panels": [
     {
@@ -87,9 +87,9 @@
         "x": 0,
         "y": 0
       },
-      "id": 25,
+      "id": 29,
       "panels": [],
-      "title": "CPU & Memory",
+      "title": "Tasks",
       "type": "row"
     },
     {
@@ -107,130 +107,23 @@
         "y": 1
       },
       "hiddenSeries": false,
-      "id": 9,
+      "id": 11,
+      "interval": "1m",
       "legend": {
-        "avg": false,
+        "alignAsTable": true,
+        "avg": true,
         "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
         "max": false,
         "min": false,
-        "show": false,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
-      "percentage": false,
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [
-        {
-          "alias": "stddev-above",
-          "fillBelowTo": "stddev-below",
-          "hideTooltip": true,
-          "lines": false
-        },
-        {
-          "alias": "stddev-below",
-          "hideTooltip": true,
-          "lines": false
-        }
-      ],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "avg(${metric_namespace}_cpu_usage_percentage{instance=~\"${nodename}\"})",
-          "interval": "",
-          "legendFormat": "cpu-usage",
-          "refId": "A"
-        },
-        {
-          "expr": "avg(${metric_namespace}_cpu_usage_percentage{instance=~\"${nodename}\"}) - stddev(${metric_namespace}_cpu_usage_percentage{instance=~\"${nodename}\"})",
-          "interval": "",
-          "legendFormat": "stddev-below",
-          "refId": "B"
-        },
-        {
-          "expr": "avg(${metric_namespace}_cpu_usage_percentage{instance=~\"${nodename}\"}) + stddev(${metric_namespace}_cpu_usage_percentage{instance=~\"${nodename}\"})",
-          "interval": "",
-          "legendFormat": "stddev-above",
-          "refId": "C"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Average CPU usage and standard deviation",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
+        "rightSide": true,
         "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "percent",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "$data_source",
-      "fill": 0,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 6,
-        "w": 24,
-        "x": 0,
-        "y": 7
-      },
-      "hiddenSeries": false,
-      "id": 20,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": false,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
-      "linewidth": 1,
+      "linewidth": 2,
       "nullPointMode": "null",
       "options": {
         "dataLinks": []
@@ -242,12 +135,12 @@
       "seriesOverrides": [],
       "spaceLength": 10,
       "stack": false,
-      "steppedLine": false,
+      "steppedLine": true,
       "targets": [
         {
-          "expr": "${metric_namespace}_memory_usage_bytes{instance=~\"${nodename}\"}",
+          "expr": "avg(irate(${metric_namespace}_tasks_polling_duration_sum{instance=~\"${nodename}\"}[10m])) by (task_name)",
           "interval": "",
-          "legendFormat": "{{instance}}",
+          "legendFormat": "{{task_name}}",
           "refId": "A"
         }
       ],
@@ -255,7 +148,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Memory usage",
+      "title": "CPU time spent on each task (average per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -271,7 +164,7 @@
       },
       "yaxes": [
         {
-          "format": "decbytes",
+          "format": "percentunit",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -292,20 +185,6 @@
         "alignLevel": null
       }
     },
-    {
-      "collapsed": false,
-      "datasource": null,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 13
-      },
-      "id": 29,
-      "panels": [],
-      "title": "Tasks",
-      "type": "row"
-    },
     {
       "aliasColors": {},
       "bars": false,
@@ -318,19 +197,23 @@
         "h": 6,
         "w": 24,
         "x": 0,
-        "y": 14
+        "y": 7
       },
       "hiddenSeries": false,
-      "id": 11,
-      "interval": "1m",
+      "id": 30,
+      "interval": "",
       "legend": {
-        "avg": false,
+        "alignAsTable": true,
+        "avg": true,
         "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
         "max": false,
         "min": false,
-        "show": false,
+        "rightSide": true,
+        "show": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 2,
@@ -348,7 +231,7 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "avg(increase(${metric_namespace}_tasks_polling_duration_sum{instance=~\"${nodename}\"}[$__interval])) by (task_name) * 1000 / $__interval_ms",
+          "expr": "avg(irate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[10m])) by (task_name)",
           "interval": "",
           "legendFormat": "{{task_name}}",
           "refId": "A"
@@ -358,7 +241,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "CPU time spent on each task (average per node)",
+      "title": "Task polling rate per second (average per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -374,7 +257,7 @@
       },
       "yaxes": [
         {
-          "format": "percentunit",
+          "format": "cps",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -407,16 +290,16 @@
         "h": 6,
         "w": 24,
         "x": 0,
-        "y": 20
+        "y": 13
       },
       "hiddenSeries": false,
-      "id": 30,
+      "id": 31,
       "interval": "",
       "legend": {
         "alignAsTable": true,
-        "avg": true,
+        "avg": false,
         "current": false,
-        "max": false,
+        "max": true,
         "min": false,
         "rightSide": true,
         "show": true,
@@ -439,7 +322,7 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "avg(rate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[5m])) by (task_name)",
+          "expr": "max(irate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[10m])) by (task_name)",
           "interval": "",
           "legendFormat": "{{task_name}}",
           "refId": "A"
@@ -449,7 +332,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Task polling rate per second (average per node)",
+      "title": "Task polling rate per second (maximum per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -498,16 +381,16 @@
         "h": 6,
         "w": 24,
         "x": 0,
-        "y": 26
+        "y": 19
       },
       "hiddenSeries": false,
-      "id": 31,
+      "id": 15,
       "interval": "",
       "legend": {
         "alignAsTable": true,
-        "avg": false,
+        "avg": true,
         "current": false,
-        "max": true,
+        "max": false,
         "min": false,
         "rightSide": true,
         "show": true,
@@ -515,8 +398,8 @@
         "values": true
       },
       "lines": true,
-      "linewidth": 2,
-      "nullPointMode": "null",
+      "linewidth": 1,
+      "nullPointMode": "null as zero",
       "options": {
         "dataLinks": []
       },
@@ -530,7 +413,7 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "max(rate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[5m])) by (task_name)",
+          "expr": "avg by(task_name) (irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[10m]))",
           "interval": "",
           "legendFormat": "{{task_name}}",
           "refId": "A"
@@ -540,7 +423,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Task polling rate per second (maximum per node)",
+      "title": "Number of tasks started per second (average per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -556,11 +439,11 @@
       },
       "yaxes": [
         {
-          "format": "cps",
+          "format": "short",
           "label": null,
-          "logBase": 1,
+          "logBase": 10,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
@@ -569,7 +452,7 @@
           "logBase": 1,
           "max": null,
           "min": null,
-          "show": false
+          "show": true
         }
       ],
       "yaxis": {
@@ -589,21 +472,21 @@
         "h": 6,
         "w": 24,
         "x": 0,
-        "y": 32
+        "y": 25
       },
       "hiddenSeries": false,
-      "id": 15,
+      "id": 16,
       "interval": "",
       "legend": {
-        "alignAsTable": false,
+        "alignAsTable": true,
         "avg": false,
         "current": false,
-        "max": false,
+        "max": true,
         "min": false,
-        "rightSide": false,
-        "show": false,
+        "rightSide": true,
+        "show": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 1,
@@ -621,7 +504,7 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "avg by(task_name) (irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[5m]))",
+          "expr": "max by(task_name) (irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[10m]))",
           "interval": "",
           "legendFormat": "{{task_name}}",
           "refId": "A"
@@ -631,7 +514,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Number of tasks started per second (average per node)",
+      "title": "Number of tasks started per second (maximum over all nodes)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -680,21 +563,21 @@
         "h": 6,
         "w": 24,
         "x": 0,
-        "y": 38
+        "y": 31
       },
       "hiddenSeries": false,
-      "id": 16,
+      "id": 2,
       "interval": "",
       "legend": {
-        "alignAsTable": false,
-        "avg": false,
+        "alignAsTable": true,
+        "avg": true,
         "current": false,
         "max": false,
         "min": false,
-        "rightSide": false,
-        "show": false,
+        "rightSide": true,
+        "show": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 1,
@@ -712,7 +595,7 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "max by(task_name) (irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[5m]))",
+          "expr": "avg by(task_name) (${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"} - sum(${metric_namespace}_tasks_ended_total{instance=~\"${nodename}\"}) without(reason))",
           "interval": "",
           "legendFormat": "{{task_name}}",
           "refId": "A"
@@ -722,7 +605,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Number of tasks started per second (maximum over all nodes)",
+      "title": "Number of tasks running (average per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -771,21 +654,21 @@
         "h": 6,
         "w": 24,
         "x": 0,
-        "y": 44
+        "y": 37
       },
       "hiddenSeries": false,
-      "id": 2,
+      "id": 3,
       "interval": "",
       "legend": {
-        "alignAsTable": false,
+        "alignAsTable": true,
         "avg": false,
         "current": false,
-        "max": false,
+        "max": true,
         "min": false,
-        "rightSide": false,
-        "show": false,
+        "rightSide": true,
+        "show": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 1,
@@ -803,7 +686,7 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "avg by(task_name) (${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"} - sum(${metric_namespace}_tasks_ended_total{instance=~\"${nodename}\"}) without(reason))",
+          "expr": "max by(task_name) (${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"} - sum(${metric_namespace}_tasks_ended_total{instance=~\"${nodename}\"}) without(reason))",
           "interval": "",
           "legendFormat": "{{task_name}}",
           "refId": "A"
@@ -813,7 +696,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Number of tasks running (average per node)",
+      "title": "Number of tasks running (maximum over all nodes)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -856,27 +739,30 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$data_source",
+      "decimals": null,
       "fill": 0,
       "fillGradient": 0,
       "gridPos": {
         "h": 6,
         "w": 24,
         "x": 0,
-        "y": 50
+        "y": 43
       },
       "hiddenSeries": false,
-      "id": 3,
+      "id": 7,
       "interval": "",
       "legend": {
-        "alignAsTable": false,
-        "avg": false,
+        "alignAsTable": true,
+        "avg": true,
         "current": false,
+        "hideEmpty": true,
+        "hideZero": true,
         "max": false,
         "min": false,
-        "rightSide": false,
-        "show": false,
+        "rightSide": true,
+        "show": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 1,
@@ -890,11 +776,11 @@
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": false,
+      "stack": true,
       "steppedLine": true,
       "targets": [
         {
-          "expr": "max by(task_name) (${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"} - sum(${metric_namespace}_tasks_ended_total{instance=~\"${nodename}\"}) without(reason))",
+          "expr": "avg(\n  irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"+Inf\"}[10m])\n    - ignoring(le)\n  irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"1.024\"}[10m])\n) by (task_name) > 0",
           "interval": "",
           "legendFormat": "{{task_name}}",
           "refId": "A"
@@ -904,11 +790,11 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Number of tasks running (maximum over all nodes)",
+      "title": "Calls to `Future::poll` that took more than one second (average per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
-        "value_type": "individual"
+        "value_type": "cumulative"
       },
       "type": "graph",
       "xaxis": {
@@ -920,9 +806,10 @@
       },
       "yaxes": [
         {
-          "format": "short",
-          "label": null,
-          "logBase": 10,
+          "decimals": null,
+          "format": "cps",
+          "label": "Calls to `Future::poll`/second",
+          "logBase": 1,
           "max": null,
           "min": "0",
           "show": true
@@ -933,7 +820,7 @@
           "logBase": 1,
           "max": null,
           "min": null,
-          "show": true
+          "show": false
         }
       ],
       "yaxis": {
@@ -941,6 +828,20 @@
         "alignLevel": null
       }
     },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 49
+      },
+      "id": 27,
+      "panels": [],
+      "title": "Misc",
+      "type": "row"
+    },
     {
       "aliasColors": {},
       "bars": false,
@@ -950,21 +851,18 @@
       "fill": 0,
       "fillGradient": 0,
       "gridPos": {
-        "h": 6,
+        "h": 7,
         "w": 24,
         "x": 0,
-        "y": 56
+        "y": 50
       },
       "hiddenSeries": false,
-      "id": 7,
-      "interval": "",
+      "id": 32,
       "legend": {
         "alignAsTable": true,
         "avg": true,
         "current": false,
-        "hideEmpty": true,
-        "hideZero": true,
-        "max": false,
+        "max": true,
         "min": false,
         "rightSide": true,
         "show": true,
@@ -973,7 +871,7 @@
       },
       "lines": true,
       "linewidth": 1,
-      "nullPointMode": "null as zero",
+      "nullPointMode": "null",
       "options": {
         "dataLinks": []
       },
@@ -983,25 +881,25 @@
       "renderer": "flot",
       "seriesOverrides": [],
       "spaceLength": 10,
-      "stack": true,
-      "steppedLine": true,
+      "stack": false,
+      "steppedLine": false,
       "targets": [
         {
-          "expr": "avg(\n  rate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"+Inf\"}[1m])\n    - ignoring(le)\n  rate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"1.024\"}[1m])\n) by (task_name) > 0",
+          "expr": "avg(${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"} - ignoring(action) ${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"received\"}) by (entity)",
           "interval": "",
-          "legendFormat": "{{task_name}}",
-          "refId": "A"
+          "legendFormat": "{{entity}}",
+          "refId": "B"
         }
       ],
       "thresholds": [],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Calls to `Future::poll` that took more than one second (average per node)",
+      "title": "Unbounded channels size (average per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
-        "value_type": "cumulative"
+        "value_type": "individual"
       },
       "type": "graph",
       "xaxis": {
@@ -1013,11 +911,11 @@
       },
       "yaxes": [
         {
-          "format": "cps",
-          "label": "Calls to `Future::poll`/second",
+          "format": "short",
+          "label": null,
           "logBase": 1,
           "max": null,
-          "min": "0",
+          "min": null,
           "show": true
         },
         {
@@ -1034,20 +932,6 @@
         "alignLevel": null
       }
     },
-    {
-      "collapsed": false,
-      "datasource": null,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 62
-      },
-      "id": 27,
-      "panels": [],
-      "title": "Misc",
-      "type": "row"
-    },
     {
       "aliasColors": {},
       "bars": false,
@@ -1060,18 +944,20 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 63
+        "y": 57
       },
       "hiddenSeries": false,
-      "id": 23,
+      "id": 33,
       "legend": {
-        "avg": false,
+        "alignAsTable": true,
+        "avg": true,
         "current": false,
         "max": false,
         "min": false,
-        "show": false,
+        "rightSide": true,
+        "show": true,
         "total": false,
-        "values": false
+        "values": true
       },
       "lines": true,
       "linewidth": 1,
@@ -1089,17 +975,17 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "${metric_namespace}_threads{instance=~\"${nodename}\"}",
+          "expr": "avg(irate(${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"}[10m])) by (entity)",
           "interval": "",
-          "legendFormat": "{{instance}}",
-          "refId": "A"
+          "legendFormat": "{{entity}}",
+          "refId": "B"
         }
       ],
       "thresholds": [],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Number of threads",
+      "title": "Unbounded channels rate (average per node)",
       "tooltip": {
         "shared": true,
         "sort": 2,
@@ -1115,7 +1001,7 @@
       },
       "yaxes": [
         {
-          "format": "short",
+          "format": "cps",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -1137,7 +1023,7 @@
       }
     }
   ],
-  "refresh": "30s",
+  "refresh": false,
   "schemaVersion": 22,
   "style": "dark",
   "tags": [],
@@ -1147,7 +1033,7 @@
         "allValue": null,
         "current": {},
         "datasource": "$data_source",
-        "definition": "${metric_namespace}_cpu_usage_percentage",
+        "definition": "${metric_namespace}_process_start_time_seconds",
         "hide": 0,
         "includeAll": true,
         "index": -1,
@@ -1155,7 +1041,7 @@
         "multi": true,
         "name": "nodename",
         "options": [],
-        "query": "${metric_namespace}_cpu_usage_percentage",
+        "query": "${metric_namespace}_process_start_time_seconds",
         "refresh": 1,
         "regex": "/instance=\"(.*?)\"/",
         "skipUrlSync": false,
@@ -1228,5 +1114,5 @@
   "variables": {
     "list": []
   },
-  "version": 44
+  "version": 52
 }

From a22edf0a24ed5469d216a067f739e2cc45c78bb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Silva?=
 <123550+andresilva@users.noreply.github.com>
Date: Tue, 8 Sep 2020 11:05:36 +0100
Subject: [PATCH 054/122] babe, grandpa: waive fees on valid equivocation
 report (#6981)

* babe: waive fees on report_equivocation

* grandpa: waive fees on report_equivocation

* babe: add test for fee waiving on valid equivocation report

* grandpa: add test for fee waiving on valid equivocation report

* grandpa: remove stray comment
---
 frame/babe/src/lib.rs      | 24 +++++++-------
 frame/babe/src/tests.rs    | 59 +++++++++++++++++++++++++++++++++++
 frame/grandpa/src/lib.rs   | 19 ++++++-----
 frame/grandpa/src/tests.rs | 64 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 147 insertions(+), 19 deletions(-)

diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs
index 891411e8ede57..2c1b2b16efc07 100644
--- a/frame/babe/src/lib.rs
+++ b/frame/babe/src/lib.rs
@@ -24,8 +24,9 @@
 use codec::{Decode, Encode};
 use frame_support::{
 	decl_error, decl_module, decl_storage,
+	dispatch::DispatchResultWithPostInfo,
 	traits::{FindAuthor, Get, KeyOwnerProofSystem, Randomness as RandomnessT},
-	weights::Weight,
+	weights::{Pays, Weight},
 	Parameter,
 };
 use frame_system::{ensure_none, ensure_signed};
@@ -260,14 +261,14 @@ decl_module! {
 			origin,
 			equivocation_proof: EquivocationProof<T::Header>,
 			key_owner_proof: T::KeyOwnerProof,
-		) {
+		) -> DispatchResultWithPostInfo {
 			let reporter = ensure_signed(origin)?;
 
 			Self::do_report_equivocation(
 				Some(reporter),
 				equivocation_proof,
 				key_owner_proof,
-			)?;
+			)
 		}
 
 		/// Report authority equivocation/misbehavior. This method will verify
@@ -283,14 +284,14 @@ decl_module! {
 			origin,
 			equivocation_proof: EquivocationProof<T::Header>,
 			key_owner_proof: T::KeyOwnerProof,
-		) {
+		) -> DispatchResultWithPostInfo {
 			ensure_none(origin)?;
 
 			Self::do_report_equivocation(
 				T::HandleEquivocation::block_author(),
 				equivocation_proof,
 				key_owner_proof,
-			)?;
+			)
 		}
 	}
 }
@@ -637,13 +638,13 @@ impl<T: Trait> Module<T> {
 		reporter: Option<T::AccountId>,
 		equivocation_proof: EquivocationProof<T::Header>,
 		key_owner_proof: T::KeyOwnerProof,
-	) -> Result<(), Error<T>> {
+	) -> DispatchResultWithPostInfo {
 		let offender = equivocation_proof.offender.clone();
 		let slot_number = equivocation_proof.slot_number;
 
 		// validate the equivocation proof
 		if !sp_consensus_babe::check_equivocation_proof(equivocation_proof) {
-			return Err(Error::InvalidEquivocationProof.into());
+			return Err(Error::<T>::InvalidEquivocationProof.into());
 		}
 
 		let validator_set_count = key_owner_proof.validator_count();
@@ -655,13 +656,13 @@ impl<T: Trait> Module<T> {
 		// check that the slot number is consistent with the session index
 		// in the key ownership proof (i.e. slot is for that epoch)
 		if epoch_index != session_index {
-			return Err(Error::InvalidKeyOwnershipProof.into());
+			return Err(Error::<T>::InvalidKeyOwnershipProof.into());
 		}
 
 		// check the membership proof and extract the offender's id
 		let key = (sp_consensus_babe::KEY_TYPE, offender);
 		let offender = T::KeyOwnerProofSystem::check_proof(key, key_owner_proof)
-			.ok_or(Error::InvalidKeyOwnershipProof)?;
+			.ok_or(Error::<T>::InvalidKeyOwnershipProof)?;
 
 		let offence = BabeEquivocationOffence {
 			slot: slot_number,
@@ -676,9 +677,10 @@ impl<T: Trait> Module<T> {
 		};
 
 		T::HandleEquivocation::report_offence(reporters, offence)
-			.map_err(|_| Error::DuplicateOffenceReport)?;
+			.map_err(|_| Error::<T>::DuplicateOffenceReport)?;
 
-		Ok(())
+		// waive the fee since the report is valid and beneficial
+		Ok(Pays::No.into())
 	}
 
 	/// Submits an extrinsic to report an equivocation. This method will create
diff --git a/frame/babe/src/tests.rs b/frame/babe/src/tests.rs
index 2b24e1208de1d..66229e5a6c8a1 100644
--- a/frame/babe/src/tests.rs
+++ b/frame/babe/src/tests.rs
@@ -21,6 +21,7 @@ use super::{Call, *};
 use frame_support::{
 	assert_err, assert_ok,
 	traits::{Currency, OnFinalize},
+	weights::{GetDispatchInfo, Pays},
 };
 use mock::*;
 use pallet_session::ShouldEndSession;
@@ -608,3 +609,61 @@ fn report_equivocation_has_valid_weight() {
 			.all(|w| w[0] < w[1])
 	);
 }
+
+#[test]
+fn valid_equivocation_reports_dont_pay_fees() {
+	let (pairs, mut ext) = new_test_ext_with_pairs(3);
+
+	ext.execute_with(|| {
+		start_era(1);
+
+		let offending_authority_pair = &pairs[0];
+
+		// generate an equivocation proof.
+		let equivocation_proof =
+			generate_equivocation_proof(0, &offending_authority_pair, CurrentSlot::get());
+
+		// create the key ownership proof.
+		let key_owner_proof = Historical::prove((
+			sp_consensus_babe::KEY_TYPE,
+			&offending_authority_pair.public(),
+		))
+		.unwrap();
+
+		// check the dispatch info for the call.
+		let info = Call::<Test>::report_equivocation_unsigned(
+			equivocation_proof.clone(),
+			key_owner_proof.clone(),
+		)
+		.get_dispatch_info();
+
+		// it should have non-zero weight and the fee has to be paid.
+		assert!(info.weight > 0);
+		assert_eq!(info.pays_fee, Pays::Yes);
+
+		// report the equivocation.
+		let post_info = Babe::report_equivocation_unsigned(
+			Origin::none(),
+			equivocation_proof.clone(),
+			key_owner_proof.clone(),
+		)
+		.unwrap();
+
+		// the original weight should be kept, but given that the report
+		// is valid the fee is waived.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::No);
+
+		// report the equivocation again which is invalid now since it is
+		// duplicate.
+		let post_info =
+			Babe::report_equivocation_unsigned(Origin::none(), equivocation_proof, key_owner_proof)
+				.err()
+				.unwrap()
+				.post_info;
+
+		// the fee is not waived and the original weight is kept.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::Yes);
+	})
+}
diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs
index 09d32662d349a..e0f2d7beda2a6 100644
--- a/frame/grandpa/src/lib.rs
+++ b/frame/grandpa/src/lib.rs
@@ -40,8 +40,8 @@ use fg_primitives::{
 	GRANDPA_ENGINE_ID,
 };
 use frame_support::{
-	decl_error, decl_event, decl_module, decl_storage, storage, traits::KeyOwnerProofSystem,
-	Parameter,
+	decl_error, decl_event, decl_module, decl_storage, dispatch::DispatchResultWithPostInfo,
+	storage, traits::KeyOwnerProofSystem, weights::Pays, Parameter,
 };
 use frame_system::{ensure_none, ensure_root, ensure_signed};
 use pallet_finality_tracker::OnFinalizationStalled;
@@ -247,14 +247,14 @@ decl_module! {
 			origin,
 			equivocation_proof: EquivocationProof<T::Hash, T::BlockNumber>,
 			key_owner_proof: T::KeyOwnerProof,
-		) {
+		) -> DispatchResultWithPostInfo {
 			let reporter = ensure_signed(origin)?;
 
 			Self::do_report_equivocation(
 				Some(reporter),
 				equivocation_proof,
 				key_owner_proof,
-			)?;
+			)
 		}
 
 		/// Report voter equivocation/misbehavior. This method will verify the
@@ -271,14 +271,14 @@ decl_module! {
 			origin,
 			equivocation_proof: EquivocationProof<T::Hash, T::BlockNumber>,
 			key_owner_proof: T::KeyOwnerProof,
-		) {
+		) -> DispatchResultWithPostInfo {
 			ensure_none(origin)?;
 
 			Self::do_report_equivocation(
 				T::HandleEquivocation::block_author(),
 				equivocation_proof,
 				key_owner_proof,
-			)?;
+			)
 		}
 
 		/// Note that the current authority set of the GRANDPA finality gadget has
@@ -520,7 +520,7 @@ impl<T: Trait> Module<T> {
 		reporter: Option<T::AccountId>,
 		equivocation_proof: EquivocationProof<T::Hash, T::BlockNumber>,
 		key_owner_proof: T::KeyOwnerProof,
-	) -> Result<(), Error<T>> {
+	) -> DispatchResultWithPostInfo {
 		// we check the equivocation within the context of its set id (and
 		// associated session) and round. we also need to know the validator
 		// set count when the offence since it is required to calculate the
@@ -585,7 +585,10 @@ impl<T: Trait> Module<T> {
 				set_id,
 				round,
 			),
-		).map_err(|_| Error::<T>::DuplicateOffenceReport)
+		).map_err(|_| Error::<T>::DuplicateOffenceReport)?;
+
+		// waive the fee since the report is valid and beneficial
+		Ok(Pays::No.into())
 	}
 
 	/// Submits an extrinsic to report an equivocation. This method will create
diff --git a/frame/grandpa/src/tests.rs b/frame/grandpa/src/tests.rs
index aa1b48681d402..951b28df57ec9 100644
--- a/frame/grandpa/src/tests.rs
+++ b/frame/grandpa/src/tests.rs
@@ -26,6 +26,7 @@ use fg_primitives::ScheduledChange;
 use frame_support::{
 	assert_err, assert_ok,
 	traits::{Currency, OnFinalize},
+	weights::{GetDispatchInfo, Pays},
 };
 use frame_system::{EventRecord, Phase};
 use pallet_session::OneSessionHandler;
@@ -865,3 +866,66 @@ fn report_equivocation_has_valid_weight() {
 			.all(|w| w[0] < w[1])
 	);
 }
+
+#[test]
+fn valid_equivocation_reports_dont_pay_fees() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let equivocation_key = &Grandpa::grandpa_authorities()[0].0;
+		let equivocation_keyring = extract_keyring(equivocation_key);
+		let set_id = Grandpa::current_set_id();
+
+		// generate an equivocation proof.
+		let equivocation_proof = generate_equivocation_proof(
+			set_id,
+			(1, H256::random(), 10, &equivocation_keyring),
+			(1, H256::random(), 10, &equivocation_keyring),
+		);
+
+		// create the key ownership proof.
+		let key_owner_proof =
+			Historical::prove((sp_finality_grandpa::KEY_TYPE, &equivocation_key)).unwrap();
+
+		// check the dispatch info for the call.
+		let info = Call::<Test>::report_equivocation_unsigned(
+			equivocation_proof.clone(),
+			key_owner_proof.clone(),
+		)
+		.get_dispatch_info();
+
+		// it should have non-zero weight and the fee has to be paid.
+		assert!(info.weight > 0);
+		assert_eq!(info.pays_fee, Pays::Yes);
+
+		// report the equivocation.
+		let post_info = Grandpa::report_equivocation_unsigned(
+			Origin::none(),
+			equivocation_proof.clone(),
+			key_owner_proof.clone(),
+		)
+		.unwrap();
+
+		// the original weight should be kept, but given that the report
+		// is valid the fee is waived.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::No);
+
+		// report the equivocation again which is invalid now since it is
+		// duplicate.
+		let post_info = Grandpa::report_equivocation_unsigned(
+			Origin::none(),
+			equivocation_proof,
+			key_owner_proof,
+		)
+		.err()
+		.unwrap()
+		.post_info;
+
+		// the fee is not waived and the original weight is kept.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::Yes);
+	})
+}

From b4fbdda8e04f8d568499143994fdaba6a55ee8ed Mon Sep 17 00:00:00 2001
From: Dan Forbes <dan@danforbes.dev>
Date: Tue, 8 Sep 2020 10:35:21 -0700
Subject: [PATCH 055/122] Clarify Nicks docs (#7049)

---
 frame/nicks/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs
index 8a38b827f8611..87a6e3b0d8b38 100644
--- a/frame/nicks/src/lib.rs
+++ b/frame/nicks/src/lib.rs
@@ -22,10 +22,10 @@
 //!
 //! ## Overview
 //!
-//! Nicks is a non-production-ready module for keeping track of account names on-chain. It makes no
-//! effort to create a name hierarchy, be a DNS replacement or provide reverse lookups. Furthermore,
-//! the weights attached to this module's dispatchable functions are for demonstration purposes only
-//! and have not been designed to be economically secure.
+//! Nicks is an example module for keeping track of account names on-chain. It makes no effort to
+//! create a name hierarchy, be a DNS replacement or provide reverse lookups. Furthermore, the
+//! weights attached to this module's dispatchable functions are for demonstration purposes only and
+//! have not been designed to be economically secure. Do not use this pallet as-is in production.
 //!
 //! ## Interface
 //!

From 67c778bbfae8b6b1bab7af693aaed007d029379f Mon Sep 17 00:00:00 2001
From: Alan Sapede <alan.sapede@gmail.com>
Date: Tue, 8 Sep 2020 16:40:52 -0400
Subject: [PATCH 056/122] Improves EVM gas price check (#7051)

---
 frame/evm/src/lib.rs | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/frame/evm/src/lib.rs b/frame/evm/src/lib.rs
index 211946bed0e95..7719f5fb7efa0 100644
--- a/frame/evm/src/lib.rs
+++ b/frame/evm/src/lib.rs
@@ -333,7 +333,7 @@ decl_module! {
 				input,
 				value,
 				gas_limit,
-				Some(gas_price),
+				gas_price,
 				nonce,
 				true,
 			)? {
@@ -367,7 +367,7 @@ decl_module! {
 				init,
 				value,
 				gas_limit,
-				Some(gas_price),
+				gas_price,
 				nonce,
 				true,
 			)? {
@@ -402,7 +402,7 @@ decl_module! {
 				salt,
 				value,
 				gas_limit,
-				Some(gas_price),
+				gas_price,
 				nonce,
 				true,
 			)? {
@@ -482,7 +482,7 @@ impl<T: Trait> Module<T> {
 		init: Vec<u8>,
 		value: U256,
 		gas_limit: u32,
-		gas_price: Option<U256>,
+		gas_price: U256,
 		nonce: Option<U256>,
 		apply_state: bool,
 	) -> Result<(ExitReason, H160, U256), Error<T>> {
@@ -514,7 +514,7 @@ impl<T: Trait> Module<T> {
 		salt: H256,
 		value: U256,
 		gas_limit: u32,
-		gas_price: Option<U256>,
+		gas_price: U256,
 		nonce: Option<U256>,
 		apply_state: bool,
 	) -> Result<(ExitReason, H160, U256), Error<T>> {
@@ -548,7 +548,7 @@ impl<T: Trait> Module<T> {
 		input: Vec<u8>,
 		value: U256,
 		gas_limit: u32,
-		gas_price: Option<U256>,
+		gas_price: U256,
 		nonce: Option<U256>,
 		apply_state: bool,
 	) -> Result<(ExitReason, Vec<u8>, U256), Error<T>> {
@@ -574,20 +574,18 @@ impl<T: Trait> Module<T> {
 		source: H160,
 		value: U256,
 		gas_limit: u32,
-		gas_price: Option<U256>,
+		gas_price: U256,
 		nonce: Option<U256>,
 		apply_state: bool,
 		f: F,
 	) -> Result<(ExitReason, R, U256), Error<T>> where
 		F: FnOnce(&mut StackExecutor<Backend<T>>) -> (ExitReason, R),
 	{
-		let gas_price = match gas_price {
-			Some(gas_price) => {
-				ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::<T>::GasPriceTooLow);
-				gas_price
-			},
-			None => U256::zero(),
-		};
+
+		// Gas price check is skipped when performing a gas estimation.
+		if apply_state {
+			ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::<T>::GasPriceTooLow);
+		}
 
 		let vicinity = Vicinity {
 			gas_price,

From 39d8f4d128b35595290fd238b9ef3f1a59a85e9d Mon Sep 17 00:00:00 2001
From: Sergei Shulepov <sergei@parity.io>
Date: Tue, 8 Sep 2020 22:49:54 +0200
Subject: [PATCH 057/122] Change wabt to wat (#7050)

---
 Cargo.lock                                    | 54 +++----------------
 bin/node/executor/Cargo.toml                  |  2 +-
 bin/node/executor/tests/basic.rs              |  4 +-
 bin/node/testing/Cargo.toml                   |  2 +-
 client/executor/Cargo.toml                    |  2 +-
 .../executor/src/integration_tests/sandbox.rs | 19 ++++---
 primitives/sandbox/Cargo.toml                 |  2 +-
 primitives/sandbox/with_std.rs                |  9 ++--
 8 files changed, 27 insertions(+), 67 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index edf9eef4d796f..307c0f9257acb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -747,7 +747,7 @@ version = "0.29.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a"
 dependencies = [
- "glob 0.3.0",
+ "glob",
  "libc",
  "libloading",
 ]
@@ -785,15 +785,6 @@ dependencies = [
  "bitflags",
 ]
 
-[[package]]
-name = "cmake"
-version = "0.1.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e56268c17a6248366d66d4a47a3381369d068cce8409bb1716ed77ea32163bb"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "concurrent-queue"
 version = "1.1.1"
@@ -2087,12 +2078,6 @@ version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724"
 
-[[package]]
-name = "glob"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
-
 [[package]]
 name = "glob"
 version = "0.3.0"
@@ -3267,7 +3252,7 @@ checksum = "eb5b56f651c204634b936be2f92dbb42c36867e00ff7fe2405591f3b9fa66f09"
 dependencies = [
  "bindgen",
  "cc",
- "glob 0.3.0",
+ "glob",
  "libc",
 ]
 
@@ -3844,7 +3829,7 @@ dependencies = [
  "sp-trie",
  "substrate-test-client",
  "trie-root",
- "wabt",
+ "wat",
 ]
 
 [[package]]
@@ -4095,7 +4080,7 @@ dependencies = [
  "sp-timestamp",
  "substrate-test-client",
  "tempfile",
- "wabt",
+ "wat",
 ]
 
 [[package]]
@@ -6785,8 +6770,8 @@ dependencies = [
  "substrate-test-runtime",
  "test-case",
  "tracing",
- "wabt",
  "wasmi",
+ "wat",
 ]
 
 [[package]]
@@ -8387,8 +8372,8 @@ dependencies = [
  "sp-io",
  "sp-std",
  "sp-wasm-interface",
- "wabt",
  "wasmi",
+ "wat",
 ]
 
 [[package]]
@@ -9580,7 +9565,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbe777c4e2060f44d83892be1189f96200be8ed3d99569d5c2d5ee26e62c0ea9"
 dependencies = [
  "dissimilar",
- "glob 0.3.0",
+ "glob",
  "lazy_static",
  "serde",
  "serde_json",
@@ -9751,29 +9736,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 
-[[package]]
-name = "wabt"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94b5f5d6984ca42df66280baa8a15ac188a173ddaf4580b574a98931c01920e7"
-dependencies = [
- "serde",
- "serde_derive",
- "serde_json",
- "wabt-sys",
-]
-
-[[package]]
-name = "wabt-sys"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b064c81821100adb4b71923cecfc67fef083db21c3bbd454b0162c7ffe63eeaa"
-dependencies = [
- "cc",
- "cmake",
- "glob 0.2.11",
-]
-
 [[package]]
 name = "wait-timeout"
 version = "0.2.0"
@@ -10354,7 +10316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b89249644df056b522696b1bb9e7c18c87e8ffa3e2f0dc3b0155875d6498f01b"
 dependencies = [
  "cc",
- "glob 0.3.0",
+ "glob",
  "itertools 0.9.0",
  "libc",
 ]
diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml
index 84a2cf377e6f0..d92cfce3eb6fe 100644
--- a/bin/node/executor/Cargo.toml
+++ b/bin/node/executor/Cargo.toml
@@ -41,7 +41,7 @@ sp-application-crypto = { version = "2.0.0-rc6", path = "../../../primitives/app
 sp-runtime = { version = "2.0.0-rc6", path = "../../../primitives/runtime" }
 sp-externalities = { version = "0.8.0-rc6", path = "../../../primitives/externalities" }
 substrate-test-client = { version = "2.0.0-rc6", path = "../../../test-utils/client" }
-wabt = "0.9.1"
+wat = "1.0"
 
 [features]
 wasmtime = [
diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs
index e7744200bccd6..36ac49b8def42 100644
--- a/bin/node/executor/tests/basic.rs
+++ b/bin/node/executor/tests/basic.rs
@@ -36,7 +36,7 @@ use node_runtime::{
 	constants::currency::*,
 };
 use node_primitives::{Balance, Hash};
-use wabt;
+use wat;
 use node_testing::keyring::*;
 
 pub mod common;
@@ -580,7 +580,7 @@ const CODE_TRANSFER: &str = r#"
 
 #[test]
 fn deploying_wasm_contract_should_work() {
-	let transfer_code = wabt::wat2wasm(CODE_TRANSFER).unwrap();
+	let transfer_code = wat::parse_str(CODE_TRANSFER).unwrap();
 	let transfer_ch = <Runtime as frame_system::Trait>::Hashing::hash(&transfer_code);
 
 	let addr = <Runtime as pallet_contracts::Trait>::DetermineContractAddress::contract_address_for(
diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml
index 1d4d6ccaa63f8..00be39d6de626 100644
--- a/bin/node/testing/Cargo.toml
+++ b/bin/node/testing/Cargo.toml
@@ -39,7 +39,7 @@ substrate-test-client = { version = "2.0.0-rc6", path = "../../../test-utils/cli
 pallet-timestamp = { version = "2.0.0-rc6", path = "../../../frame/timestamp" }
 pallet-transaction-payment = { version = "2.0.0-rc6", path = "../../../frame/transaction-payment" }
 pallet-treasury = { version = "2.0.0-rc6", path = "../../../frame/treasury" }
-wabt = "0.9.1"
+wat = "1.0"
 sp-api = { version = "2.0.0-rc6", path = "../../../primitives/api" }
 sp-finality-tracker = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/finality-tracker" }
 sp-timestamp = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/timestamp" }
diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml
index 029e01923ac70..0b9829e6f342b 100644
--- a/client/executor/Cargo.toml
+++ b/client/executor/Cargo.toml
@@ -37,7 +37,7 @@ libsecp256k1 = "0.3.4"
 
 [dev-dependencies]
 assert_matches = "1.3.0"
-wabt = "0.9.1"
+wat = "1.0"
 hex-literal = "0.3.1"
 sc-runtime-test = { version = "2.0.0-rc6", path = "runtime-test" }
 substrate-test-runtime = { version = "2.0.0-rc6", path = "../../test-utils/runtime" }
diff --git a/client/executor/src/integration_tests/sandbox.rs b/client/executor/src/integration_tests/sandbox.rs
index f84e446b416c0..447e395c2fb08 100644
--- a/client/executor/src/integration_tests/sandbox.rs
+++ b/client/executor/src/integration_tests/sandbox.rs
@@ -21,7 +21,6 @@ use crate::WasmExecutionMethod;
 
 use codec::Encode;
 use test_case::test_case;
-use wabt;
 
 #[test_case(WasmExecutionMethod::Interpreted)]
 #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
@@ -29,7 +28,7 @@ fn sandbox_should_work(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(import "env" "assert" (func $assert (param i32)))
 			(import "env" "inc_counter" (func $inc_counter (param i32) (result i32)))
@@ -67,7 +66,7 @@ fn sandbox_trap(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(import "env" "assert" (func $assert (param i32)))
 			(func (export "call")
@@ -94,7 +93,7 @@ fn start_called(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(import "env" "assert" (func $assert (param i32)))
 			(import "env" "inc_counter" (func $inc_counter (param i32) (result i32)))
@@ -138,7 +137,7 @@ fn invoke_args(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(import "env" "assert" (func $assert (param i32)))
 
@@ -178,7 +177,7 @@ fn return_val(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(func (export "call") (param $x i32) (result i32)
 				(i32.add
@@ -206,7 +205,7 @@ fn unlinkable_module(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(import "env" "non-existent" (func))
 
@@ -252,7 +251,7 @@ fn start_fn_ok(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(func (export "call")
 			)
@@ -281,7 +280,7 @@ fn start_fn_traps(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(func (export "call")
 			)
@@ -311,7 +310,7 @@ fn get_global_val_works(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	let code = wabt::wat2wasm(r#"
+	let code = wat::parse_str(r#"
 		(module
 			(global (export "test_global") i64 (i64.const 500))
 		)
diff --git a/primitives/sandbox/Cargo.toml b/primitives/sandbox/Cargo.toml
index 0ee2feea2b94e..56e486178d3cc 100755
--- a/primitives/sandbox/Cargo.toml
+++ b/primitives/sandbox/Cargo.toml
@@ -20,7 +20,7 @@ sp-wasm-interface = { version = "2.0.0-rc6", default-features = false, path = ".
 codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false }
 
 [dev-dependencies]
-wabt = "0.9.1"
+wat = "1.0"
 assert_matches = "1.3.0"
 
 [features]
diff --git a/primitives/sandbox/with_std.rs b/primitives/sandbox/with_std.rs
index b5d6d89d0434f..0f46f49503cac 100755
--- a/primitives/sandbox/with_std.rs
+++ b/primitives/sandbox/with_std.rs
@@ -300,7 +300,6 @@ impl<T> Instance<T> {
 
 #[cfg(test)]
 mod tests {
-	use wabt;
 	use crate::{Error, Value, ReturnValue, HostError, EnvironmentDefinitionBuilder, Instance};
 	use assert_matches::assert_matches;
 
@@ -351,7 +350,7 @@ mod tests {
 
 	#[test]
 	fn invoke_args() {
-		let code = wabt::wat2wasm(r#"
+		let code = wat::parse_str(r#"
 		(module
 			(import "env" "assert" (func $assert (param i32)))
 
@@ -386,7 +385,7 @@ mod tests {
 
 	#[test]
 	fn return_value() {
-		let code = wabt::wat2wasm(r#"
+		let code = wat::parse_str(r#"
 		(module
 			(func (export "call") (param $x i32) (result i32)
 				(i32.add
@@ -408,7 +407,7 @@ mod tests {
 
 	#[test]
 	fn signatures_dont_matter() {
-		let code = wabt::wat2wasm(r#"
+		let code = wat::parse_str(r#"
 		(module
 			(import "env" "polymorphic_id" (func $id_i32 (param i32) (result i32)))
 			(import "env" "polymorphic_id" (func $id_i64 (param i64) (result i64)))
@@ -450,7 +449,7 @@ mod tests {
 		let mut env_builder = EnvironmentDefinitionBuilder::new();
 		env_builder.add_host_func("env", "returns_i32", env_returns_i32);
 
-		let code = wabt::wat2wasm(r#"
+		let code = wat::parse_str(r#"
 		(module
 			;; It's actually returns i32, but imported as if it returned i64
 			(import "env" "returns_i32" (func $returns_i32 (result i64)))

From 5fa219d7109c255d6083367e33bf2c573c1364fc Mon Sep 17 00:00:00 2001
From: Lovesh Harchandani <lovesh@users.noreply.github.com>
Date: Wed, 9 Sep 2020 02:35:15 +0530
Subject: [PATCH 058/122] Add Dock network id for address generation (#6714)

Taking 21 and 22 for testnet and mainnet

Signed-off-by: lovesh <lovesh.bond@gmail.com>
---
 primitives/core/src/crypto.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs
index a8d84eb57cff9..1e418c5c73fe0 100644
--- a/primitives/core/src/crypto.rs
+++ b/primitives/core/src/crypto.rs
@@ -467,6 +467,10 @@ ss58_address_format!(
 		(18, "darwinia", "Darwinia Chain mainnet, standard account (*25519).")
 	StafiAccount =>
 		(20, "stafi", "Stafi mainnet, standard account (*25519).")
+	DockTestAccount =>
+		(21, "dock-testnet", "Dock testnet, standard account (*25519).")
+	DockMainAccount =>
+		(22, "dock-mainnet", "Dock mainnet, standard account (*25519).")
 	ShiftNrg =>
 		(23, "shift", "ShiftNrg mainnet, standard account (*25519).")
 	SubsocialAccount =>

From 0a850aeb16b3ff74d20040125a88dc4260558fc8 Mon Sep 17 00:00:00 2001
From: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Date: Wed, 9 Sep 2020 14:59:50 +0200
Subject: [PATCH 059/122] Partial fix for transaction priority (#7034)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Partial fix for priority stuff.

* Small fix

* Fix tests.

* Update frame/transaction-payment/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Better doc

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
---
 frame/support/src/weights.rs                | 24 ++++++++++++++++++
 frame/system/src/extensions/check_nonce.rs  |  9 ++++---
 frame/system/src/extensions/check_weight.rs | 20 +++++++++------
 frame/transaction-payment/src/lib.rs        | 27 ++++++++++++++++-----
 4 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/frame/support/src/weights.rs b/frame/support/src/weights.rs
index db1e25ca7ab2e..1d19eeef70d79 100644
--- a/frame/support/src/weights.rs
+++ b/frame/support/src/weights.rs
@@ -242,6 +242,30 @@ impl Default for DispatchClass {
 	}
 }
 
+/// Primitives related to priority management of Frame.
+pub mod priority {
+	/// The starting point of all Operational transactions. 3/4 of u64::max_value().
+	pub const LIMIT: u64 = 13_835_058_055_282_163_711_u64;
+
+	/// Wrapper for priority of different dispatch classes.
+	///
+	/// This only makes sure that any value created for the operational dispatch class is
+	/// incremented by [`LIMIT`].
+	pub enum FrameTransactionPriority {
+		Normal(u64),
+		Operational(u64),
+	}
+
+	impl From<FrameTransactionPriority> for u64 {
+		fn from(priority: FrameTransactionPriority) -> Self {
+			match priority {
+				FrameTransactionPriority::Normal(inner) => inner,
+				FrameTransactionPriority::Operational(inner) => inner.saturating_add(LIMIT),
+			}
+		}
+	}
+}
+
 /// A bundle of static information collected from the `#[weight = $x]` attributes.
 #[derive(Clone, Copy, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode)]
 pub struct DispatchInfo {
diff --git a/frame/system/src/extensions/check_nonce.rs b/frame/system/src/extensions/check_nonce.rs
index 1af3a1210aaf4..e7316457aaffc 100644
--- a/frame/system/src/extensions/check_nonce.rs
+++ b/frame/system/src/extensions/check_nonce.rs
@@ -25,12 +25,15 @@ use sp_runtime::{
 	traits::{SignedExtension, DispatchInfoOf, Dispatchable, One},
 	transaction_validity::{
 		ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionValidity,
-		TransactionLongevity, TransactionPriority,
+		TransactionLongevity,
 	},
 };
 use sp_std::vec;
 
 /// Nonce check and increment to give replay protection for transactions.
+///
+/// Note that this does not set any priority by default. Make sure that AT LEAST one of the signed
+/// extension sets some kind of priority upon validating transactions.
 #[derive(Encode, Decode, Clone, Eq, PartialEq)]
 pub struct CheckNonce<T: Trait>(#[codec(compact)] T::Index);
 
@@ -90,7 +93,7 @@ impl<T: Trait> SignedExtension for CheckNonce<T> where
 		&self,
 		who: &Self::AccountId,
 		_call: &Self::Call,
-		info: &DispatchInfoOf<Self::Call>,
+		_info: &DispatchInfoOf<Self::Call>,
 		_len: usize,
 	) -> TransactionValidity {
 		// check index
@@ -107,7 +110,7 @@ impl<T: Trait> SignedExtension for CheckNonce<T> where
 		};
 
 		Ok(ValidTransaction {
-			priority: info.weight as TransactionPriority,
+			priority: 0,
 			requires,
 			provides,
 			longevity: TransactionLongevity::max_value(),
diff --git a/frame/system/src/extensions/check_weight.rs b/frame/system/src/extensions/check_weight.rs
index 1395aa87efbcf..092ac59da97c8 100644
--- a/frame/system/src/extensions/check_weight.rs
+++ b/frame/system/src/extensions/check_weight.rs
@@ -27,7 +27,7 @@ use sp_runtime::{
 };
 use frame_support::{
 	traits::{Get},
-	weights::{PostDispatchInfo, DispatchInfo, DispatchClass},
+	weights::{PostDispatchInfo, DispatchInfo, DispatchClass, priority::FrameTransactionPriority},
 	StorageValue,
 };
 
@@ -157,12 +157,18 @@ impl<T: Trait + Send + Sync> CheckWeight<T> where
 	}
 
 	/// get the priority of an extrinsic denoted by `info`.
+	///
+	/// Operational transaction will be given a fixed initial amount to be fairly distinguished from
+	/// the normal ones.
 	fn get_priority(info: &DispatchInfoOf<T::Call>) -> TransactionPriority {
 		match info.class {
-			DispatchClass::Normal => info.weight.into(),
-			// Don't use up the whole priority space, to allow things like `tip`
-			// to be taken into account as well.
-			DispatchClass::Operational => TransactionPriority::max_value() / 2,
+			// Normal transaction.
+			DispatchClass::Normal =>
+				FrameTransactionPriority::Normal(info.weight.into()).into(),
+			// Don't use up the whole priority space, to allow things like `tip` to be taken into
+			// account as well.
+			DispatchClass::Operational =>
+				FrameTransactionPriority::Operational(info.weight.into()).into(),
 			// Mandatory extrinsics are only for inherents; never transactions.
 			DispatchClass::Mandatory => TransactionPriority::min_value(),
 		}
@@ -496,7 +502,7 @@ mod tests {
 	}
 
 	#[test]
-	fn signed_ext() {
+	fn signed_ext_check_weight_works() {
 		new_test_ext().execute_with(|| {
 			let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes };
 			let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: Pays::Yes };
@@ -512,7 +518,7 @@ mod tests {
 				.validate(&1, CALL, &op, len)
 				.unwrap()
 				.priority;
-			assert_eq!(priority, u64::max_value() / 2);
+			assert_eq!(priority, frame_support::weights::priority::LIMIT + 100);
 		})
 	}
 
diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs
index 244b4280ade08..4e4bc5311dacd 100644
--- a/frame/transaction-payment/src/lib.rs
+++ b/frame/transaction-payment/src/lib.rs
@@ -467,6 +467,23 @@ impl<T: Trait + Send + Sync> ChargeTransactionPayment<T> where
 			Err(_) => Err(InvalidTransaction::Payment.into()),
 		}
 	}
+
+	/// Get an appropriate priority for a transaction with the given length and info.
+	///
+	/// This will try and optimise the `fee/weight` `fee/length`, whichever is consuming more of the
+	/// maximum corresponding limit.
+	///
+	/// For example, if a transaction consumed 1/4th of the block length and half of the weight, its
+	/// final priority is `fee * min(2, 4) = fee * 2`. If it consumed `1/4th` of the block length
+	/// and the entire block weight `(1/1)`, its priority is `fee * min(1, 4) = fee * 1`. This means
+	///  that the transaction which consumes more resources (either length or weight) with the same
+	/// `fee` ends up having lower priority.
+	fn get_priority(len: usize, info: &DispatchInfoOf<T::Call>, final_fee: BalanceOf<T>) -> TransactionPriority {
+		let weight_saturation = T::MaximumBlockWeight::get() / info.weight.max(1);
+		let len_saturation = T::MaximumBlockLength::get() as u64 / (len as u64).max(1);
+		let coefficient: BalanceOf<T> = weight_saturation.min(len_saturation).saturated_into::<BalanceOf<T>>();
+		final_fee.saturating_mul(coefficient).saturated_into::<TransactionPriority>()
+	}
 }
 
 impl<T: Trait + Send + Sync> sp_std::fmt::Debug for ChargeTransactionPayment<T> {
@@ -499,12 +516,10 @@ impl<T: Trait + Send + Sync> SignedExtension for ChargeTransactionPayment<T> whe
 		len: usize,
 	) -> TransactionValidity {
 		let (fee, _) = self.withdraw_fee(who, info, len)?;
-
-		let mut r = ValidTransaction::default();
-		// NOTE: we probably want to maximize the _fee (of any type) per weight unit_ here, which
-		// will be a bit more than setting the priority to tip. For now, this is enough.
-		r.priority = fee.saturated_into::<TransactionPriority>();
-		Ok(r)
+		Ok(ValidTransaction {
+			priority: Self::get_priority(len, info, fee),
+			..Default::default()
+		})
 	}
 
 	fn pre_dispatch(

From 29b42c3de683e4ee2ae591c392d6b774657d6465 Mon Sep 17 00:00:00 2001
From: Sergei Shulepov <sergei@parity.io>
Date: Wed, 9 Sep 2020 15:23:40 +0200
Subject: [PATCH 060/122] What happens if we remove wat? (#7056)

* What happens if we remove wat?

* Update Cargo.lock
---
 Cargo.lock                  | 1 -
 bin/node/testing/Cargo.toml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 307c0f9257acb..173624261f133 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4080,7 +4080,6 @@ dependencies = [
  "sp-timestamp",
  "substrate-test-client",
  "tempfile",
- "wat",
 ]
 
 [[package]]
diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml
index 00be39d6de626..89079d53ece4d 100644
--- a/bin/node/testing/Cargo.toml
+++ b/bin/node/testing/Cargo.toml
@@ -39,7 +39,6 @@ substrate-test-client = { version = "2.0.0-rc6", path = "../../../test-utils/cli
 pallet-timestamp = { version = "2.0.0-rc6", path = "../../../frame/timestamp" }
 pallet-transaction-payment = { version = "2.0.0-rc6", path = "../../../frame/transaction-payment" }
 pallet-treasury = { version = "2.0.0-rc6", path = "../../../frame/treasury" }
-wat = "1.0"
 sp-api = { version = "2.0.0-rc6", path = "../../../primitives/api" }
 sp-finality-tracker = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/finality-tracker" }
 sp-timestamp = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/timestamp" }

From 384f29fb130acf30cd402b016a01377d46736ed9 Mon Sep 17 00:00:00 2001
From: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Date: Wed, 9 Sep 2020 15:31:36 +0200
Subject: [PATCH 061/122] Make SlashingSpans Public (#6961)

* Make SlashingSpans Public

Offchain Applications will often need to inspect this type because it is directly used in staking election, thus worthy of being `pub`. Rest of the slashing api can remain private, only this and the `fn last_non_zero_slash()` of `SlashingSpans` are of interest.

* Update frame/staking/src/lib.rs
---
 frame/staking/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs
index a15b7ac5d7248..b49ec12109d2a 100644
--- a/frame/staking/src/lib.rs
+++ b/frame/staking/src/lib.rs
@@ -1162,7 +1162,7 @@ decl_storage! {
 			=> Option<BalanceOf<T>>;
 
 		/// Slashing spans for stash accounts.
-		SlashingSpans: map hasher(twox_64_concat) T::AccountId => Option<slashing::SlashingSpans>;
+		SlashingSpans get(fn slashing_spans): map hasher(twox_64_concat) T::AccountId => Option<slashing::SlashingSpans>;
 
 		/// Records information about the maximum slash of a stash within a slashing span,
 		/// as well as how much reward has been paid out.

From d123792a332bf98ebb830ffa25037ae4b4a27e73 Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Wed, 9 Sep 2020 17:08:26 +0200
Subject: [PATCH 062/122] client/authority-discovery/src/service: Improve docs
 (#7059)

---
 client/authority-discovery/src/service.rs | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/client/authority-discovery/src/service.rs b/client/authority-discovery/src/service.rs
index 01fb7134fb5d1..ed0205d262fc6 100644
--- a/client/authority-discovery/src/service.rs
+++ b/client/authority-discovery/src/service.rs
@@ -37,12 +37,18 @@ impl Service {
 		}
 	}
 
-	/// Get the addresses for the given [`AuthorityId`] from the local address cache.
+	/// Get the addresses for the given [`AuthorityId`] from the local address
+	/// cache.
 	///
-	/// Returns `None` if no entry was present or connection to the [`crate::Worker`] failed.
+	/// Returns `None` if no entry was present or connection to the
+	/// [`crate::Worker`] failed.
 	///
-	/// [`Multiaddr`]s returned always include a [`libp2p::core::multiaddr:Protocol::P2p`]
-	/// component.
+	/// [`Multiaddr`]s returned always include a [`PeerId`] via a
+	/// [`libp2p::core::multiaddr:Protocol::P2p`] component. [`Multiaddr`]s
+	/// might differ in their [`PeerId`], e.g. when each [`Multiaddr`]
+	/// represents a different sentry node. This might change once support for
+	/// sentry nodes is removed (see
+	/// https://github.com/paritytech/substrate/issues/6845).
 	pub async fn get_addresses_by_authority_id(&mut self, authority: AuthorityId) -> Option<Vec<Multiaddr>> {
 		let (tx, rx) = oneshot::channel();
 
@@ -54,9 +60,11 @@ impl Service {
 		rx.await.ok().flatten()
 	}
 
-	/// Get the [`AuthorityId`] for the given [`PeerId`] from the local address cache.
+	/// Get the [`AuthorityId`] for the given [`PeerId`] from the local address
+	/// cache.
 	///
-	/// Returns `None` if no entry was present or connection to the [`crate::Worker`] failed.
+	/// Returns `None` if no entry was present or connection to the
+	/// [`crate::Worker`] failed.
 	pub async fn get_authority_id_by_peer_id(&mut self, peer_id: PeerId) -> Option<AuthorityId> {
 		let (tx, rx) = oneshot::channel();
 

From d35044cdf012e6a24e8cf3273e4cf5673b0d95b1 Mon Sep 17 00:00:00 2001
From: s3krit <pugh@s3kr.it>
Date: Wed, 9 Sep 2020 22:32:46 +0200
Subject: [PATCH 063/122] Decrease poll interval (#7063)

---
 .github/workflows/polkadot-companion-labels.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/polkadot-companion-labels.yml b/.github/workflows/polkadot-companion-labels.yml
index 27f743e1bd452..3c3987b5f4d56 100644
--- a/.github/workflows/polkadot-companion-labels.yml
+++ b/.github/workflows/polkadot-companion-labels.yml
@@ -16,9 +16,10 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           contexts: 'continuous-integration/gitlab-check-polkadot-companion-build'
           timeout: 1800
-          notPresentTimeout: 3600 # It can take quite a while before the job starts...
+          notPresentTimeout: 3600 # It can take quite a while before the job starts on Gitlab when the CI queue is large
           failureStates: failure
           interruptedStates: error # Error = job was probably cancelled. We don't want to label the PR in that case
+          pollInterval: 30
       - name: Label success
         uses: andymckay/labeler@master
         if: steps.check-companion-status.outputs.result == 'success'

From de59f048717d7099dc25727c8f516007d69d6e1f Mon Sep 17 00:00:00 2001
From: Jimmy Chu <jimmy@parity.io>
Date: Thu, 10 Sep 2020 04:35:16 +0800
Subject: [PATCH 064/122] Remove unused code (#7027)

Signed-off-by: Jimmy Chu <jimmychu0807@gmail.com>
---
 frame/example-offchain-worker/src/lib.rs | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/frame/example-offchain-worker/src/lib.rs b/frame/example-offchain-worker/src/lib.rs
index f6a4a68e3cb3d..b9ee6d3d8b5ed 100644
--- a/frame/example-offchain-worker/src/lib.rs
+++ b/frame/example-offchain-worker/src/lib.rs
@@ -165,7 +165,7 @@ decl_storage! {
 decl_event!(
 	/// Events generated by the module.
 	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId {
-		/// Event generated when new price is accepted to contribute to the average. 
+		/// Event generated when new price is accepted to contribute to the average.
 		/// [price, who]
 		NewPrice(u32, AccountId),
 	}
@@ -461,16 +461,6 @@ impl<T: Trait> Module<T> {
 		// Note this call will block until response is received.
 		let price = Self::fetch_price().map_err(|_| "Failed to fetch price")?;
 
-		// Received price is wrapped into a call to `submit_price_unsigned` public function of this
-		// pallet. This means that the transaction, when executed, will simply call that function
-		// passing `price` as an argument.
-		let call = Call::submit_price_unsigned(block_number, price);
-
-		// Now let's create a transaction out of this call and submit it to the pool.
-		// Here we showcase two ways to send an unsigned transaction with a signed payload
-		SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into())
-			.map_err(|()| "Unable to submit unsigned transaction.")?;
-
 		// -- Sign using any account
 		let (_, result) = Signer::<T, T::AuthorityId>::any_account().send_unsigned_transaction(
 			|account| PricePayload {
@@ -500,16 +490,6 @@ impl<T: Trait> Module<T> {
 		// Note this call will block until response is received.
 		let price = Self::fetch_price().map_err(|_| "Failed to fetch price")?;
 
-		// Received price is wrapped into a call to `submit_price_unsigned` public function of this
-		// pallet. This means that the transaction, when executed, will simply call that function
-		// passing `price` as an argument.
-		let call = Call::submit_price_unsigned(block_number, price);
-
-		// Now let's create a transaction out of this call and submit it to the pool.
-		// Here we showcase two ways to send an unsigned transaction with a signed payload
-		SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into())
-			.map_err(|()| "Unable to submit unsigned transaction.")?;
-
 		// -- Sign using all accounts
 		let transaction_results = Signer::<T, T::AuthorityId>::all_accounts()
 			.send_unsigned_transaction(

From f96646ab08c604c3467e0f7033087fab81afc29f Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Thu, 10 Sep 2020 01:47:24 +0200
Subject: [PATCH 065/122] Disambiguate `BlockNumber` type in `decl_module`
 (#7061)

* Disambiguate `BlockNumber` type in `decl_module`

* fix `frame-support-tests`

* fix ui tests

* fix trait order
---
 frame/support/src/dispatch.rs                 | 65 +++++++++++--------
 frame/support/test/src/lib.rs                 |  2 +-
 frame/support/test/tests/construct_runtime.rs |  4 +-
 ...served_keyword_two_times_integrity_test.rs |  2 +-
 ...ed_keyword_two_times_integrity_test.stderr |  4 +-
 ...eserved_keyword_two_times_on_initialize.rs |  2 +-
 ...ved_keyword_two_times_on_initialize.stderr |  4 +-
 frame/support/test/tests/decl_storage.rs      |  8 +--
 .../tests/decl_storage_ui/config_duplicate.rs |  2 +-
 .../decl_storage_ui/config_get_duplicate.rs   |  2 +-
 .../tests/decl_storage_ui/get_duplicate.rs    |  2 +-
 frame/support/test/tests/final_keys.rs        |  6 +-
 frame/support/test/tests/genesisconfig.rs     |  2 +-
 frame/support/test/tests/instance.rs          |  2 +-
 frame/support/test/tests/issue2219.rs         |  2 +-
 .../tests/reserved_keyword/on_initialize.rs   |  2 +-
 .../support/test/tests/storage_transaction.rs |  2 +-
 frame/support/test/tests/system.rs            |  2 +-
 frame/system/src/lib.rs                       |  2 +-
 19 files changed, 64 insertions(+), 53 deletions(-)

diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs
index 442a99effadbc..85599626ec2cc 100644
--- a/frame/support/src/dispatch.rs
+++ b/frame/support/src/dispatch.rs
@@ -1265,15 +1265,16 @@ macro_rules! decl_module {
 	};
 
 	(@impl_on_initialize
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 		fn on_initialize() -> $return:ty { $( $impl:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OnInitialize<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OnInitialize<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
-			fn on_initialize(_block_number_not_used: $trait_instance::BlockNumber) -> $return {
+			fn on_initialize(_block_number_not_used: <$trait_instance as $system::Trait>::BlockNumber) -> $return {
 				$crate::sp_tracing::enter_span!("on_initialize");
 				{ $( $impl )* }
 			}
@@ -1281,12 +1282,13 @@ macro_rules! decl_module {
 	};
 
 	(@impl_on_initialize
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 		fn on_initialize($param:ident : $param_ty:ty) -> $return:ty { $( $impl:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OnInitialize<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OnInitialize<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn on_initialize($param: $param_ty) -> $return {
@@ -1297,11 +1299,12 @@ macro_rules! decl_module {
 	};
 
 	(@impl_on_initialize
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OnInitialize<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OnInitialize<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{}
 	};
@@ -1362,15 +1365,16 @@ macro_rules! decl_module {
 	};
 
 	(@impl_on_finalize
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 		fn on_finalize() { $( $impl:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OnFinalize<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OnFinalize<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
-			fn on_finalize(_block_number_not_used: $trait_instance::BlockNumber) {
+			fn on_finalize(_block_number_not_used: <$trait_instance as $system::Trait>::BlockNumber) {
 				$crate::sp_tracing::enter_span!("on_finalize");
 				{ $( $impl )* }
 			}
@@ -1378,12 +1382,13 @@ macro_rules! decl_module {
 	};
 
 	(@impl_on_finalize
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 		fn on_finalize($param:ident : $param_ty:ty) { $( $impl:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OnFinalize<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OnFinalize<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn on_finalize($param: $param_ty) {
@@ -1394,36 +1399,39 @@ macro_rules! decl_module {
 	};
 
 	(@impl_on_finalize
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OnFinalize<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OnFinalize<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 		}
 	};
 
 	(@impl_offchain
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 		fn offchain_worker() { $( $impl:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OffchainWorker<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OffchainWorker<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
-			fn offchain_worker(_block_number_not_used: $trait_instance::BlockNumber) { $( $impl )* }
+			fn offchain_worker(_block_number_not_used: <$trait_instance as $system::Trait>::BlockNumber) { $( $impl )* }
 		}
 	};
 
 	(@impl_offchain
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 		fn offchain_worker($param:ident : $param_ty:ty) { $( $impl:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OffchainWorker<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OffchainWorker<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn offchain_worker($param: $param_ty) { $( $impl )* }
@@ -1431,11 +1439,12 @@ macro_rules! decl_module {
 	};
 
 	(@impl_offchain
+		{ $system:ident }
 		$module:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>;
 		{ $( $other_where_bounds:tt )* }
 	) => {
-		impl<$trait_instance: $trait_name$(<I>, $instance: $instantiable)?>
-			$crate::traits::OffchainWorker<$trait_instance::BlockNumber>
+		impl<$trait_instance: $system::Trait + $trait_name$(<I>, $instance: $instantiable)?>
+			$crate::traits::OffchainWorker<<$trait_instance as $system::Trait>::BlockNumber>
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{}
 	};
@@ -1635,6 +1644,7 @@ macro_rules! decl_module {
 
 		$crate::decl_module! {
 			@impl_on_initialize
+			{ $system }
 			$mod_type<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?>;
 			{ $( $other_where_bounds )* }
 			$( $on_initialize )*
@@ -1649,6 +1659,7 @@ macro_rules! decl_module {
 
 		$crate::decl_module! {
 			@impl_on_finalize
+			{ $system }
 			$mod_type<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?>;
 			{ $( $other_where_bounds )* }
 			$( $on_finalize )*
@@ -1656,6 +1667,7 @@ macro_rules! decl_module {
 
 		$crate::decl_module! {
 			@impl_offchain
+			{ $system }
 			$mod_type<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?>;
 			{ $( $other_where_bounds )* }
 			$( $offchain )*
@@ -2345,9 +2357,7 @@ mod tests {
 		IntegrityTest,
 	};
 
-	pub trait Trait: system::Trait + Sized where Self::AccountId: From<u32> {
-		type BlockNumber: Into<u32>;
-	}
+	pub trait Trait: system::Trait + Sized where Self::AccountId: From<u32> { }
 
 	pub mod system {
 		use codec::{Encode, Decode};
@@ -2357,6 +2367,7 @@ mod tests {
 			type Call;
 			type BaseCallFilter;
 			type Origin: crate::traits::OriginTrait<Call = Self::Call>;
+			type BlockNumber: Into<u32>;
 		}
 
 		#[derive(Clone, PartialEq, Eq, Debug, Encode, Decode)]
@@ -2480,10 +2491,7 @@ mod tests {
 	];
 
 	pub struct TraitImpl {}
-
-	impl Trait for TraitImpl {
-		type BlockNumber = u32;
-	}
+	impl Trait for TraitImpl { }
 
 	type Test = Module<TraitImpl>;
 
@@ -2502,6 +2510,7 @@ mod tests {
 		type AccountId = u32;
 		type Call = OuterCall;
 		type BaseCallFilter = ();
+		type BlockNumber = u32;
 	}
 
 	#[test]
diff --git a/frame/support/test/src/lib.rs b/frame/support/test/src/lib.rs
index c0baf448eed85..d5f49299880ca 100644
--- a/frame/support/test/src/lib.rs
+++ b/frame/support/test/src/lib.rs
@@ -32,5 +32,5 @@ pub trait Trait {
 
 frame_support::decl_module! {
 	/// Some test module
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 }
diff --git a/frame/support/test/tests/construct_runtime.rs b/frame/support/test/tests/construct_runtime.rs
index 10fc3319fb080..9cb3a2532a745 100644
--- a/frame/support/test/tests/construct_runtime.rs
+++ b/frame/support/test/tests/construct_runtime.rs
@@ -40,7 +40,7 @@ mod module1 {
 
 	frame_support::decl_module! {
 		pub struct Module<T: Trait<I>, I: Instance = DefaultInstance> for enum Call
-			where origin: <T as system::Trait>::Origin
+			where origin: <T as system::Trait>::Origin, system=system
 		{
 			#[weight = 0]
 			pub fn fail(_origin) -> frame_support::dispatch::DispatchResult {
@@ -67,7 +67,7 @@ mod module2 {
 
 	frame_support::decl_module! {
 		pub struct Module<T: Trait> for enum Call
-			where origin: <T as system::Trait>::Origin
+			where origin: <T as system::Trait>::Origin, system=system
 		{
 			#[weight = 0]
 			pub fn fail(_origin) -> frame_support::dispatch::DispatchResult {
diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs
index 4dbae05f07ff7..56eff29c5dc1b 100644
--- a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs
+++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs
@@ -1,5 +1,5 @@
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 		fn integrity_test() {}
 
 		fn integrity_test() {}
diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr
index d6498961d31c8..25f3b891d9b47 100644
--- a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr
+++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr
@@ -2,7 +2,7 @@ error: `integrity_test` can only be passed once as input.
  --> $DIR/reserved_keyword_two_times_integrity_test.rs:1:1
   |
 1 | / frame_support::decl_module! {
-2 | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+2 | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 3 | |         fn integrity_test() {}
 4 | |
 5 | |         fn integrity_test() {}
@@ -16,7 +16,7 @@ error[E0601]: `main` function not found in crate `$CRATE`
  --> $DIR/reserved_keyword_two_times_integrity_test.rs:1:1
   |
 1 | / frame_support::decl_module! {
-2 | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+2 | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 3 | |         fn integrity_test() {}
 4 | |
 5 | |         fn integrity_test() {}
diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs
index 4f05134997e81..3e1bc25c8d59c 100644
--- a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs
+++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs
@@ -1,5 +1,5 @@
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 		fn on_initialize() -> Weight {
 			0
 		}
diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.stderr b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.stderr
index 8a9f025046b7e..34c5ff3f941a1 100644
--- a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.stderr
+++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.stderr
@@ -2,7 +2,7 @@ error: `on_initialize` can only be passed once as input.
   --> $DIR/reserved_keyword_two_times_on_initialize.rs:1:1
    |
 1  | / frame_support::decl_module! {
-2  | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+2  | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 3  | |         fn on_initialize() -> Weight {
 4  | |             0
 ...  |
@@ -16,7 +16,7 @@ error[E0601]: `main` function not found in crate `$CRATE`
   --> $DIR/reserved_keyword_two_times_on_initialize.rs:1:1
    |
 1  | / frame_support::decl_module! {
-2  | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+2  | |     pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 3  | |         fn on_initialize() -> Weight {
 4  | |             0
 ...  |
diff --git a/frame/support/test/tests/decl_storage.rs b/frame/support/test/tests/decl_storage.rs
index cda1d810d225c..9bdc4226263df 100644
--- a/frame/support/test/tests/decl_storage.rs
+++ b/frame/support/test/tests/decl_storage.rs
@@ -25,7 +25,7 @@ mod tests {
 	use codec::{Encode, Decode, EncodeLike};
 
 	frame_support::decl_module! {
-		pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+		pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 	}
 
 	pub trait Trait {
@@ -420,7 +420,7 @@ mod test2 {
 	}
 
 	frame_support::decl_module! {
-		pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+		pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 	}
 
 	type PairOf<T> = (T, T);
@@ -455,7 +455,7 @@ mod test3 {
 		type BlockNumber;
 	}
 	frame_support::decl_module! {
-		pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+		pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 	}
 	frame_support::decl_storage! {
 		trait Store for Module<T: Trait> as Test {
@@ -485,7 +485,7 @@ mod test_append_and_len {
 	}
 
 	frame_support::decl_module! {
-		pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+		pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 	}
 
 	#[derive(PartialEq, Eq, Clone, Encode, Decode)]
diff --git a/frame/support/test/tests/decl_storage_ui/config_duplicate.rs b/frame/support/test/tests/decl_storage_ui/config_duplicate.rs
index 4d510da9f8936..f4f4ad7d48a97 100644
--- a/frame/support/test/tests/decl_storage_ui/config_duplicate.rs
+++ b/frame/support/test/tests/decl_storage_ui/config_duplicate.rs
@@ -21,7 +21,7 @@ pub trait Trait {
 }
 
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 }
 
 frame_support::decl_storage!{
diff --git a/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs b/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs
index 49897e6251868..3caa2d9c33608 100644
--- a/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs
+++ b/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs
@@ -21,7 +21,7 @@ pub trait Trait {
 }
 
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 }
 
 frame_support::decl_storage!{
diff --git a/frame/support/test/tests/decl_storage_ui/get_duplicate.rs b/frame/support/test/tests/decl_storage_ui/get_duplicate.rs
index 2fa78f4d17c56..1c24b3bf28eec 100644
--- a/frame/support/test/tests/decl_storage_ui/get_duplicate.rs
+++ b/frame/support/test/tests/decl_storage_ui/get_duplicate.rs
@@ -21,7 +21,7 @@ pub trait Trait {
 }
 
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 }
 
 frame_support::decl_storage!{
diff --git a/frame/support/test/tests/final_keys.rs b/frame/support/test/tests/final_keys.rs
index 34da1752da052..a9f0cdc8f184b 100644
--- a/frame/support/test/tests/final_keys.rs
+++ b/frame/support/test/tests/final_keys.rs
@@ -29,7 +29,7 @@ mod no_instance {
 	}
 
 	frame_support::decl_module! {
-		pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+		pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 	}
 
 	frame_support::decl_storage!{
@@ -50,11 +50,13 @@ mod no_instance {
 }
 
 mod instance {
+	use super::no_instance;
+
 	pub trait Trait<I = DefaultInstance>: super::no_instance::Trait {}
 
 	frame_support::decl_module! {
 		pub struct Module<T: Trait<I>, I: Instance = DefaultInstance>
-			for enum Call where origin: T::Origin {}
+			for enum Call where origin: T::Origin, system=no_instance {}
 	}
 
 	frame_support::decl_storage!{
diff --git a/frame/support/test/tests/genesisconfig.rs b/frame/support/test/tests/genesisconfig.rs
index 78b841d295017..af8b393800cf9 100644
--- a/frame/support/test/tests/genesisconfig.rs
+++ b/frame/support/test/tests/genesisconfig.rs
@@ -21,7 +21,7 @@ pub trait Trait {
 }
 
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 }
 
 frame_support::decl_storage! {
diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs
index 33e8cc1fd6c0f..b0df32ddf9c93 100644
--- a/frame/support/test/tests/instance.rs
+++ b/frame/support/test/tests/instance.rs
@@ -184,7 +184,7 @@ mod module3 {
 	}
 
 	frame_support::decl_module! {
-		pub struct Module<T: Trait> for enum Call where origin: <T as system::Trait>::Origin {}
+		pub struct Module<T: Trait> for enum Call where origin: <T as system::Trait>::Origin, system=system {}
 	}
 }
 
diff --git a/frame/support/test/tests/issue2219.rs b/frame/support/test/tests/issue2219.rs
index 7166f202c7325..2e47ef64926d6 100644
--- a/frame/support/test/tests/issue2219.rs
+++ b/frame/support/test/tests/issue2219.rs
@@ -84,7 +84,7 @@ mod module {
 	pub trait Trait: system::Trait {}
 
 	frame_support::decl_module! {
-		pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+		pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=system {}
 	}
 
 	#[derive(Encode, Decode, Copy, Clone, Serialize, Deserialize)]
diff --git a/frame/support/test/tests/reserved_keyword/on_initialize.rs b/frame/support/test/tests/reserved_keyword/on_initialize.rs
index 0751c600cccb2..db71fe9a1e26a 100644
--- a/frame/support/test/tests/reserved_keyword/on_initialize.rs
+++ b/frame/support/test/tests/reserved_keyword/on_initialize.rs
@@ -18,7 +18,7 @@ macro_rules! reserved {
 				}
 
 				frame_support::decl_module! {
-					pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+					pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 						#[weight = 0]
 						fn $reserved(_origin) -> dispatch::DispatchResult { unreachable!() }
 					}
diff --git a/frame/support/test/tests/storage_transaction.rs b/frame/support/test/tests/storage_transaction.rs
index a9711ec267e54..a7e4a75c27fcb 100644
--- a/frame/support/test/tests/storage_transaction.rs
+++ b/frame/support/test/tests/storage_transaction.rs
@@ -29,7 +29,7 @@ pub trait Trait {
 }
 
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 		#[weight = 0]
 		#[transactional]
 		fn value_commits(_origin, v: u32) {
diff --git a/frame/support/test/tests/system.rs b/frame/support/test/tests/system.rs
index 8ca2e97789d54..fd5fe20a69a2b 100644
--- a/frame/support/test/tests/system.rs
+++ b/frame/support/test/tests/system.rs
@@ -31,7 +31,7 @@ pub trait Trait: 'static + Eq + Clone {
 }
 
 frame_support::decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin, {}
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {}
 }
 
 impl<T: Trait> Module<T> {
diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs
index fcd31923a2453..d2c7e25676739 100644
--- a/frame/system/src/lib.rs
+++ b/frame/system/src/lib.rs
@@ -517,7 +517,7 @@ decl_error! {
 }
 
 decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin, system=self {
 		type Error = Error<T>;
 
 		/// The maximum number of blocks to allow in mortal eras.

From 89e6d66eda8591333ab41ae965173e976d53997c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Thu, 10 Sep 2020 03:24:40 +0200
Subject: [PATCH 066/122] Implement `FromStr` for `Ss58AddressFormat` (#7068)

* Implement `FromStr` for `Ss58AddressFormat`

* Update primitives/core/src/crypto.rs

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
---
 primitives/core/src/crypto.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs
index 1e418c5c73fe0..527808fab9ccc 100644
--- a/primitives/core/src/crypto.rs
+++ b/primitives/core/src/crypto.rs
@@ -408,6 +408,15 @@ macro_rules! ss58_address_format {
 			}
 		}
 
+		#[cfg(feature = "std")]
+		impl std::str::FromStr for Ss58AddressFormat {
+			type Err = ParseError;
+
+			fn from_str(data: &str) -> Result<Self, Self::Err> {
+				Self::try_from(data)
+			}
+		}
+
 		#[cfg(feature = "std")]
 		impl std::fmt::Display for ParseError {
 			fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

From bd444234be8a25d5793e01f6b927b9064d90cbbd Mon Sep 17 00:00:00 2001
From: kaichao <kaichaosuna@gmail.com>
Date: Thu, 10 Sep 2020 23:26:09 +0800
Subject: [PATCH 067/122] Set reserved nodes with offchain worker. (#6996)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add offchain worker api to set reserved nodes.

* new offchain api to get node public key.

* node public key from converter

* refactor set reserved nodes ocw api.

* new ndoe authorization pallet

* remove unnecessary clone and more.

* more

* tests for node authorization pallet

* remove dependency

* fix build

* more tests.

* refactor

* Update primitives/core/src/offchain/testing.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update frame/node-authorization/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update frame/node-authorization/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update frame/node-authorization/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* format code

* expose NetworkService

* remove NetworkStateInfo in offchain

* replace NodePublicKey with PeerId.

* set max length of peer id.

* clear more

* use BTreeSet for set of peers.

* decode opaque peer id.

* extract NetworkProvider for client offchain.

* use OpaquePeerId in node authorization pallet.

* fix test

* better documentation

* fix test

* doc

* more fix

* Update primitives/core/src/offchain/mod.rs

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Update client/offchain/src/api.rs

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* derive serialize and deserialize

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
---
 Cargo.lock                              |  14 +
 Cargo.toml                              |   1 +
 client/network/src/service.rs           |  18 +-
 client/offchain/src/api.rs              |  50 +-
 client/offchain/src/lib.rs              |  59 +-
 client/peerset/src/lib.rs               |  12 +
 client/service/src/builder.rs           |   2 +-
 frame/im-online/src/benchmarking.rs     |   3 +-
 frame/im-online/src/tests.rs            |   2 +-
 frame/node-authorization/Cargo.toml     |  35 +
 frame/node-authorization/src/lib.rs     | 861 ++++++++++++++++++++++++
 primitives/core/src/lib.rs              |  15 +-
 primitives/core/src/offchain/mod.rs     |  38 +-
 primitives/core/src/offchain/testing.rs |   5 +
 primitives/io/src/lib.rs                |   9 +-
 15 files changed, 1080 insertions(+), 44 deletions(-)
 create mode 100644 frame/node-authorization/Cargo.toml
 create mode 100644 frame/node-authorization/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index 173624261f133..3c525c6c063e6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4738,6 +4738,20 @@ dependencies = [
  "sp-std",
 ]
 
+[[package]]
+name = "pallet-node-authorization"
+version = "2.0.0-rc6"
+dependencies = [
+ "frame-support",
+ "frame-system",
+ "parity-scale-codec",
+ "serde",
+ "sp-core",
+ "sp-io",
+ "sp-runtime",
+ "sp-std",
+]
+
 [[package]]
 name = "pallet-offences"
 version = "2.0.0-rc6"
diff --git a/Cargo.toml b/Cargo.toml
index 7589e8d774124..534b71357cc76 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -88,6 +88,7 @@ members = [
 	"frame/metadata",
 	"frame/multisig",
 	"frame/nicks",
+	"frame/node-authorization",
 	"frame/offences",
 	"frame/proxy",
 	"frame/randomness-collective-flip",
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index 28af928060002..d1248057cc79f 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -20,7 +20,7 @@
 //!
 //! There are two main structs in this module: [`NetworkWorker`] and [`NetworkService`].
 //! The [`NetworkWorker`] *is* the network and implements the `Future` trait. It must be polled in
-//! order fo the network to advance.
+//! order for the network to advance.
 //! The [`NetworkService`] is merely a shared version of the [`NetworkWorker`]. You can obtain an
 //! `Arc<NetworkService>` by calling [`NetworkWorker::service`].
 //!
@@ -605,6 +605,22 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 		&self.local_peer_id
 	}
 
+	/// Set authorized peers.
+	///
+	/// Need a better solution to manage authorized peers, but now just use reserved peers for
+	/// prototyping.
+	pub fn set_authorized_peers(&self, peers: HashSet<PeerId>) {
+		self.peerset.set_reserved_peers(peers)
+	}
+
+	/// Set authorized_only flag.
+	///
+	/// Need a better solution to decide authorized_only, but now just use reserved_only flag for
+	/// prototyping.
+	pub fn set_authorized_only(&self, reserved_only: bool) {
+		self.peerset.set_reserved_only(reserved_only)
+	}
+
 	/// Appends a notification to the buffer of pending outgoing notifications with the given peer.
 	/// Has no effect if the notifications channel with this protocol name is not open.
 	///
diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs
index 5287ac8251eeb..a7ab07c549665 100644
--- a/client/offchain/src/api.rs
+++ b/client/offchain/src/api.rs
@@ -19,16 +19,18 @@ use std::{
 	sync::Arc,
 	convert::TryFrom,
 	thread::sleep,
+	collections::HashSet,
 };
 
-use sp_core::offchain::OffchainStorage;
+use crate::NetworkProvider;
 use futures::Future;
 use log::error;
-use sc_network::{PeerId, Multiaddr, NetworkStateInfo};
+use sc_network::{PeerId, Multiaddr};
 use codec::{Encode, Decode};
+use sp_core::OpaquePeerId;
 use sp_core::offchain::{
 	Externalities as OffchainExt, HttpRequestId, Timestamp, HttpRequestStatus, HttpError,
-	OpaqueNetworkState, OpaquePeerId, OpaqueMultiaddr, StorageKind,
+	OffchainStorage, OpaqueNetworkState, OpaqueMultiaddr, StorageKind,
 };
 pub use sp_offchain::STORAGE_PREFIX;
 pub use http::SharedClient;
@@ -49,8 +51,8 @@ mod timestamp;
 pub(crate) struct Api<Storage> {
 	/// Offchain Workers database.
 	db: Storage,
-	/// A NetworkState provider.
-	network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
+	/// A provider for substrate networking.
+	network_provider: Arc<dyn NetworkProvider + Send + Sync>,
 	/// Is this node a potential validator?
 	is_validator: bool,
 	/// Everything HTTP-related is handled by a different struct.
@@ -73,10 +75,10 @@ impl<Storage: OffchainStorage> OffchainExt for Api<Storage> {
 	}
 
 	fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
-		let external_addresses = self.network_state.external_addresses();
+		let external_addresses = self.network_provider.external_addresses();
 
 		let state = NetworkState::new(
-			self.network_state.local_peer_id(),
+			self.network_provider.local_peer_id(),
 			external_addresses,
 		);
 		Ok(OpaqueNetworkState::from(state))
@@ -180,6 +182,15 @@ impl<Storage: OffchainStorage> OffchainExt for Api<Storage> {
 	) -> Result<usize, HttpError> {
 		self.http.response_read_body(request_id, buffer, deadline)
 	}
+
+	fn set_authorized_nodes(&mut self, nodes: Vec<OpaquePeerId>, authorized_only: bool) {
+		let peer_ids: HashSet<PeerId> = nodes.into_iter()
+			.filter_map(|node| PeerId::from_bytes(node.0).ok())
+			.collect();
+		
+		self.network_provider.set_authorized_peers(peer_ids);
+		self.network_provider.set_authorized_only(authorized_only);
+	}
 }
 
 /// Information about the local node's network state.
@@ -256,10 +267,10 @@ pub(crate) struct AsyncApi {
 }
 
 impl AsyncApi {
-	/// Creates new Offchain extensions API implementation  an the asynchronous processing part.
+	/// Creates new Offchain extensions API implementation an the asynchronous processing part.
 	pub fn new<S: OffchainStorage>(
 		db: S,
-		network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
+		network_provider: Arc<dyn NetworkProvider + Send + Sync>,
 		is_validator: bool,
 		shared_client: SharedClient,
 	) -> (Api<S>, Self) {
@@ -267,7 +278,7 @@ impl AsyncApi {
 
 		let api = Api {
 			db,
-			network_state,
+			network_provider,
 			is_validator,
 			http: http_api,
 		};
@@ -292,11 +303,21 @@ mod tests {
 	use super::*;
 	use std::{convert::{TryFrom, TryInto}, time::SystemTime};
 	use sc_client_db::offchain::LocalStorage;
-	use sc_network::PeerId;
+	use sc_network::{NetworkStateInfo, PeerId};
 
-	struct MockNetworkStateInfo();
+	struct TestNetwork();
+
+	impl NetworkProvider for TestNetwork {
+		fn set_authorized_peers(&self, _peers: HashSet<PeerId>) {
+			unimplemented!()
+		}
 
-	impl NetworkStateInfo for MockNetworkStateInfo {
+		fn set_authorized_only(&self, _reserved_only: bool) {
+			unimplemented!()
+		}
+	}
+
+	impl NetworkStateInfo for TestNetwork {
 		fn external_addresses(&self) -> Vec<Multiaddr> {
 			Vec::new()
 		}
@@ -309,10 +330,9 @@ mod tests {
 	fn offchain_api() -> (Api<LocalStorage>, AsyncApi) {
 		let _ = env_logger::try_init();
 		let db = LocalStorage::new_test();
-		let mock = Arc::new(MockNetworkStateInfo());
+		let mock = Arc::new(TestNetwork());
 		let shared_client = SharedClient::new();
 
-
 		AsyncApi::new(
 			db,
 			mock,
diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs
index 3b17c14f19652..89f2b7b8100b2 100644
--- a/client/offchain/src/lib.rs
+++ b/client/offchain/src/lib.rs
@@ -33,14 +33,17 @@
 
 #![warn(missing_docs)]
 
-use std::{fmt, marker::PhantomData, sync::Arc};
+use std::{
+	fmt, marker::PhantomData, sync::Arc,
+	collections::HashSet,
+};
 
 use parking_lot::Mutex;
 use threadpool::ThreadPool;
 use sp_api::{ApiExt, ProvideRuntimeApi};
 use futures::future::Future;
 use log::{debug, warn};
-use sc_network::NetworkStateInfo;
+use sc_network::{ExHashT, NetworkService, NetworkStateInfo, PeerId};
 use sp_core::{offchain::{self, OffchainStorage}, ExecutionContext, traits::SpawnNamed};
 use sp_runtime::{generic::BlockId, traits::{self, Header}};
 use futures::{prelude::*, future::ready};
@@ -50,6 +53,30 @@ use api::SharedClient;
 
 pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX};
 
+/// NetworkProvider provides [`OffchainWorkers`] with all necessary hooks into the
+/// underlying Substrate networking.
+pub trait NetworkProvider: NetworkStateInfo {
+	/// Set the authorized peers.
+	fn set_authorized_peers(&self, peers: HashSet<PeerId>);
+	
+	/// Set the authorized only flag.
+	fn set_authorized_only(&self, reserved_only: bool);
+}
+
+impl<B, H> NetworkProvider for NetworkService<B, H>
+where
+	B: traits::Block + 'static,
+	H: ExHashT,
+{
+	fn set_authorized_peers(&self, peers: HashSet<PeerId>) {
+		self.set_authorized_peers(peers)
+	}
+
+	fn set_authorized_only(&self, reserved_only: bool) {
+		self.set_authorized_only(reserved_only)
+	}
+}
+
 /// An offchain workers manager.
 pub struct OffchainWorkers<Client, Storage, Block: traits::Block> {
 	client: Arc<Client>,
@@ -98,7 +125,7 @@ impl<Client, Storage, Block> OffchainWorkers<
 	pub fn on_block_imported(
 		&self,
 		header: &Block::Header,
-		network_state: Arc<dyn NetworkStateInfo + Send + Sync>,
+		network_provider: Arc<dyn NetworkProvider + Send + Sync>,
 		is_validator: bool,
 	) -> impl Future<Output = ()> {
 		let runtime = self.client.runtime_api();
@@ -122,7 +149,7 @@ impl<Client, Storage, Block> OffchainWorkers<
 		if version > 0 {
 			let (api, runner) = api::AsyncApi::new(
 				self.db.clone(),
-				network_state.clone(),
+				network_provider,
 				is_validator,
 				self.shared_client.clone(),
 			);
@@ -173,7 +200,7 @@ pub async fn notification_future<Client, Storage, Block, Spawner>(
 	client: Arc<Client>,
 	offchain: Arc<OffchainWorkers<Client, Storage, Block>>,
 	spawner: Spawner,
-	network_state_info: Arc<dyn NetworkStateInfo + Send + Sync>,
+	network_provider: Arc<dyn NetworkProvider + Send + Sync>,
 )
 	where
 		Block: traits::Block,
@@ -188,7 +215,7 @@ pub async fn notification_future<Client, Storage, Block, Spawner>(
 				"offchain-on-block",
 				offchain.on_block_imported(
 					&n.header,
-					network_state_info.clone(),
+					network_provider.clone(),
 					is_validator,
 				).boxed(),
 			);
@@ -213,9 +240,9 @@ mod tests {
 	use sc_transaction_pool::{BasicPool, FullChainApi};
 	use sp_transaction_pool::{TransactionPool, InPoolTransaction};
 
-	struct MockNetworkStateInfo();
+	struct TestNetwork();
 
-	impl NetworkStateInfo for MockNetworkStateInfo {
+	impl NetworkStateInfo for TestNetwork {
 		fn external_addresses(&self) -> Vec<Multiaddr> {
 			Vec::new()
 		}
@@ -225,6 +252,16 @@ mod tests {
 		}
 	}
 
+	impl NetworkProvider for TestNetwork {
+		fn set_authorized_peers(&self, _peers: HashSet<PeerId>) {
+			unimplemented!()
+		}
+
+		fn set_authorized_only(&self, _reserved_only: bool) {
+			unimplemented!()
+		}
+	}
+
 	struct TestPool(
 		Arc<BasicPool<FullChainApi<TestClient, Block>, Block>>
 	);
@@ -255,12 +292,14 @@ mod tests {
 			client.clone(),
 		));
 		let db = sc_client_db::offchain::LocalStorage::new_test();
-		let network_state = Arc::new(MockNetworkStateInfo());
+		let network = Arc::new(TestNetwork());
 		let header = client.header(&BlockId::number(0)).unwrap().unwrap();
 
 		// when
 		let offchain = OffchainWorkers::new(client, db);
-		futures::executor::block_on(offchain.on_block_imported(&header, network_state, false));
+		futures::executor::block_on(
+			offchain.on_block_imported(&header, network, false)
+		);
 
 		// then
 		assert_eq!(pool.0.status().ready, 1);
diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs
index 6f28dd036a3cc..575743afa079c 100644
--- a/client/peerset/src/lib.rs
+++ b/client/peerset/src/lib.rs
@@ -45,6 +45,7 @@ const FORGET_AFTER: Duration = Duration::from_secs(3600);
 enum Action {
 	AddReservedPeer(PeerId),
 	RemoveReservedPeer(PeerId),
+	SetReservedPeers(HashSet<PeerId>),
 	SetReservedOnly(bool),
 	ReportPeer(PeerId, ReputationChange),
 	SetPriorityGroup(String, HashSet<PeerId>),
@@ -102,6 +103,11 @@ impl PeersetHandle {
 	pub fn set_reserved_only(&self, reserved: bool) {
 		let _ = self.tx.unbounded_send(Action::SetReservedOnly(reserved));
 	}
+	
+	/// Set reserved peers to the new set.
+	pub fn set_reserved_peers(&self, peer_ids: HashSet<PeerId>) {
+		let _ = self.tx.unbounded_send(Action::SetReservedPeers(peer_ids));
+	}
 
 	/// Reports an adjustment to the reputation of the given peer.
 	pub fn report_peer(&self, peer_id: PeerId, score_diff: ReputationChange) {
@@ -246,6 +252,10 @@ impl Peerset {
 	fn on_remove_reserved_peer(&mut self, peer_id: PeerId) {
 		self.on_remove_from_priority_group(RESERVED_NODES, peer_id);
 	}
+	
+	fn on_set_reserved_peers(&mut self, peer_ids: HashSet<PeerId>) {
+		self.on_set_priority_group(RESERVED_NODES, peer_ids);
+	}
 
 	fn on_set_reserved_only(&mut self, reserved_only: bool) {
 		self.reserved_only = reserved_only;
@@ -655,6 +665,8 @@ impl Stream for Peerset {
 					self.on_add_reserved_peer(peer_id),
 				Action::RemoveReservedPeer(peer_id) =>
 					self.on_remove_reserved_peer(peer_id),
+				Action::SetReservedPeers(peer_ids) =>
+					self.on_set_reserved_peers(peer_ids),
 				Action::SetReservedOnly(reserved) =>
 					self.on_set_reserved_only(reserved),
 				Action::ReportPeer(peer_id, score_diff) =>
diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs
index f4046ab722ba7..93e6c3fc91ba7 100644
--- a/client/service/src/builder.rs
+++ b/client/service/src/builder.rs
@@ -431,7 +431,7 @@ pub fn build_offchain_workers<TBl, TBackend, TCl>(
 				client.clone(),
 				offchain,
 				Clone::clone(&spawn_handle),
-				network.clone()
+				network.clone(),
 			)
 		);
 	}
diff --git a/frame/im-online/src/benchmarking.rs b/frame/im-online/src/benchmarking.rs
index 92d9b9d5a5364..55f294505602e 100644
--- a/frame/im-online/src/benchmarking.rs
+++ b/frame/im-online/src/benchmarking.rs
@@ -23,7 +23,8 @@ use super::*;
 
 use frame_system::RawOrigin;
 use frame_benchmarking::benchmarks;
-use sp_core::offchain::{OpaquePeerId, OpaqueMultiaddr};
+use sp_core::OpaquePeerId;
+use sp_core::offchain::OpaqueMultiaddr;
 use sp_runtime::traits::{ValidateUnsigned, Zero};
 use sp_runtime::transaction_validity::TransactionSource;
 use frame_support::traits::UnfilteredDispatchable;
diff --git a/frame/im-online/src/tests.rs b/frame/im-online/src/tests.rs
index 835d8440e6d5b..22c6b4464c370 100644
--- a/frame/im-online/src/tests.rs
+++ b/frame/im-online/src/tests.rs
@@ -21,8 +21,8 @@
 
 use super::*;
 use crate::mock::*;
+use sp_core::OpaquePeerId;
 use sp_core::offchain::{
-	OpaquePeerId,
 	OffchainExt,
 	TransactionPoolExt,
 	testing::{TestOffchainExt, TestTransactionPoolExt},
diff --git a/frame/node-authorization/Cargo.toml b/frame/node-authorization/Cargo.toml
new file mode 100644
index 0000000000000..b05430c452cc6
--- /dev/null
+++ b/frame/node-authorization/Cargo.toml
@@ -0,0 +1,35 @@
+[package]
+name = "pallet-node-authorization"
+version = "2.0.0-rc6"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+description = "FRAME pallet for node authorization"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+serde = { version = "1.0.101", optional = true }
+codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] }
+frame-support = { version = "2.0.0-rc6", default-features = false, path = "../support" }
+frame-system = { version = "2.0.0-rc6", default-features = false, path = "../system" }
+sp-core = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/core" }
+sp-io = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/io" }
+sp-runtime = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/runtime" }
+sp-std = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/std" }
+
+[features]
+default = ["std"]
+std = [
+	"serde",
+	"codec/std",
+	"frame-support/std",
+	"frame-system/std",
+	"sp-core/std",
+	"sp-io/std",
+	"sp-runtime/std",
+	"sp-std/std",
+]
diff --git a/frame/node-authorization/src/lib.rs b/frame/node-authorization/src/lib.rs
new file mode 100644
index 0000000000000..9b401091beb02
--- /dev/null
+++ b/frame/node-authorization/src/lib.rs
@@ -0,0 +1,861 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! # Node authorization pallet
+//!
+//! This pallet manages a configurable set of nodes for a permissioned network.
+//! Each node is dentified by a PeerId (i.e. Vec<u8>). It provides two ways to
+//! authorize a node,
+//!
+//! - a set of well known nodes across different organizations in which the
+//! connections are allowed.
+//! - users can claim the ownership for each node, then manage the connections of
+//! the node.
+//!
+//! A node must have an owner. The owner can additionally change the connections
+//! for the node. Only one user is allowed to claim a specific node. To eliminate
+//! false claim, the maintainer of the node should claim it before even starting the
+//! node. This pallet uses offchain worker to set reserved nodes, if the node is not
+//! an authority, make sure to enable offchain worker with the right CLI flag. The
+//! node can be lagged with the latest block, in this case you need to disable offchain
+//! worker and manually set reserved nodes when starting it.
+
+// Ensure we're `no_std` when compiling for Wasm.
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use sp_core::OpaquePeerId as PeerId;
+use sp_std::{
+	collections::btree_set::BTreeSet,
+	iter::FromIterator,
+	prelude::*,
+};
+use codec::Decode;
+use frame_support::{
+	decl_module, decl_storage, decl_event, decl_error,
+	debug, ensure,
+	weights::{DispatchClass, Weight},
+	traits::{Get, EnsureOrigin},
+};
+use frame_system::ensure_signed;
+
+pub trait WeightInfo {
+	fn add_well_known_node() -> Weight;
+	fn remove_well_known_node() -> Weight;
+	fn swap_well_known_node() -> Weight;
+	fn reset_well_known_nodes() -> Weight;
+	fn claim_node() -> Weight;
+	fn remove_claim() -> Weight;
+	fn transfer_node() -> Weight;
+	fn add_connections() -> Weight;
+	fn remove_connections() -> Weight;
+}
+
+impl WeightInfo for () {
+	fn add_well_known_node() -> Weight { 50_000_000 }
+	fn remove_well_known_node() -> Weight { 50_000_000 }
+	fn swap_well_known_node() -> Weight { 50_000_000 }
+	fn reset_well_known_nodes() -> Weight { 50_000_000 }
+	fn claim_node() -> Weight { 50_000_000 }
+	fn remove_claim() -> Weight { 50_000_000 }
+	fn transfer_node() -> Weight { 50_000_000 }
+	fn add_connections() -> Weight { 50_000_000 }
+	fn remove_connections() -> Weight { 50_000_000 }
+}
+
+pub trait Trait: frame_system::Trait {
+	/// The event type of this module.
+	type Event: From<Event<Self>> + Into<<Self as frame_system::Trait>::Event>;
+
+	/// The maximum number of well known nodes that are allowed to set
+	type MaxWellKnownNodes: Get<u32>;
+
+	/// The maximum length in bytes of PeerId
+	type MaxPeerIdLength: Get<u32>;
+
+	/// The origin which can add a well known node.
+	type AddOrigin: EnsureOrigin<Self::Origin>;
+
+	/// The origin which can remove a well known node.
+	type RemoveOrigin: EnsureOrigin<Self::Origin>;
+
+	/// The origin which can swap the well known nodes.
+	type SwapOrigin: EnsureOrigin<Self::Origin>;
+
+	/// The origin which can reset the well known nodes.
+	type ResetOrigin: EnsureOrigin<Self::Origin>;
+
+	/// Weight information for extrinsics in this pallet.
+	type WeightInfo: WeightInfo;
+}
+
+decl_storage! {
+	trait Store for Module<T: Trait> as NodeAuthorization {
+		/// The set of well known nodes. This is stored sorted (just by value).
+		pub WellKnownNodes get(fn well_known_nodes): BTreeSet<PeerId>;
+		/// A map that maintains the ownership of each node.
+		pub Owners get(fn owners):
+			map hasher(blake2_128_concat) PeerId => T::AccountId;
+		/// The additional adapative connections of each node.
+		pub AdditionalConnections get(fn additional_connection):
+			map hasher(blake2_128_concat) PeerId => BTreeSet<PeerId>;
+	}
+	add_extra_genesis {
+		config(nodes): Vec<(PeerId, T::AccountId)>;
+		build(|config: &GenesisConfig<T>| {
+			<Module<T>>::initialize_nodes(&config.nodes)
+		})
+	}
+}
+
+decl_event! {
+	pub enum Event<T> where
+		<T as frame_system::Trait>::AccountId,
+	{
+		/// The given well known node was added.
+		NodeAdded(PeerId, AccountId),
+		/// The given well known node was removed.
+		NodeRemoved(PeerId),
+		/// The given well known node was swapped; first item was removed,
+		/// the latter was added.
+		NodeSwapped(PeerId, PeerId),
+		/// The given well known nodes were reset.
+		NodesReset(Vec<(PeerId, AccountId)>),
+		/// The given node was claimed by a user.
+		NodeClaimed(PeerId, AccountId),
+		/// The given claim was removed by its owner.
+		ClaimRemoved(PeerId, AccountId),
+		/// The node was transferred to another account.
+		NodeTransferred(PeerId, AccountId),
+		/// The allowed connections were added to a node.
+		ConnectionsAdded(PeerId, Vec<PeerId>),
+		/// The allowed connections were removed from a node.
+		ConnectionsRemoved(PeerId, Vec<PeerId>),
+	}
+}
+
+decl_error! {
+	/// Error for the node authorization module.
+	pub enum Error for Module<T: Trait> {
+		/// The PeerId is too long.
+		PeerIdTooLong,
+		/// Too many well known nodes.
+		TooManyNodes,
+		/// The node is already joined in the list.
+		AlreadyJoined,
+		/// The node doesn't exist in the list.
+		NotExist,
+		/// The node is already claimed by a user.
+		AlreadyClaimed,
+		/// The node hasn't been claimed yet.
+		NotClaimed,
+		/// You are not the owner of the node.
+		NotOwner,
+		/// No permisson to perform specific operation.
+		PermissionDenied,
+	}
+}
+
+decl_module! {
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+		/// The maximum number of authorized well known nodes
+		const MaxWellKnownNodes: u32 = T::MaxWellKnownNodes::get();
+
+		/// The maximum length in bytes of PeerId
+		const MaxPeerIdLength: u32 = T::MaxPeerIdLength::get();
+
+		type Error = Error<T>;
+
+		fn deposit_event() = default;
+
+		/// Add a node to the set of well known nodes. If the node is already claimed, the owner
+		/// will be updated and keep the existing additional connection unchanged.
+		///
+		/// May only be called from `T::AddOrigin`.
+		///
+		/// - `node`: identifier of the node.
+		#[weight = (T::WeightInfo::add_well_known_node(), DispatchClass::Operational)]
+		pub fn add_well_known_node(origin, node: PeerId, owner: T::AccountId) {
+			T::AddOrigin::ensure_origin(origin)?;
+			ensure!(node.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+
+			let mut nodes = WellKnownNodes::get();
+			ensure!(nodes.len() < T::MaxWellKnownNodes::get() as usize, Error::<T>::TooManyNodes);
+			ensure!(!nodes.contains(&node), Error::<T>::AlreadyJoined);
+
+			nodes.insert(node.clone());
+
+			WellKnownNodes::put(&nodes);
+			<Owners<T>>::insert(&node, &owner);
+
+			Self::deposit_event(RawEvent::NodeAdded(node, owner));
+		}
+
+		/// Remove a node from the set of well known nodes. The ownership and additional
+		/// connections of the node will also be removed.
+		///
+		/// May only be called from `T::RemoveOrigin`.
+		///
+		/// - `node`: identifier of the node.
+		#[weight = (T::WeightInfo::remove_well_known_node(), DispatchClass::Operational)]
+		pub fn remove_well_known_node(origin, node: PeerId) {
+			T::RemoveOrigin::ensure_origin(origin)?;
+			ensure!(node.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+
+			let mut nodes = WellKnownNodes::get();
+			ensure!(nodes.contains(&node), Error::<T>::NotExist);
+
+			nodes.remove(&node);
+
+			WellKnownNodes::put(&nodes);
+			<Owners<T>>::remove(&node);
+			AdditionalConnections::remove(&node);
+
+			Self::deposit_event(RawEvent::NodeRemoved(node));
+		}
+
+		/// Swap a well known node to another. Both the ownership and additional connections
+		/// stay untouched.
+		///
+		/// May only be called from `T::SwapOrigin`.
+		///
+		/// - `remove`: the node which will be moved out from the list.
+		/// - `add`: the node which will be put in the list.
+		#[weight = (T::WeightInfo::swap_well_known_node(), DispatchClass::Operational)]
+		pub fn swap_well_known_node(origin, remove: PeerId, add: PeerId) {
+			T::SwapOrigin::ensure_origin(origin)?;
+			ensure!(remove.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+			ensure!(add.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+
+			if remove == add { return Ok(()) }
+
+			let mut nodes = WellKnownNodes::get();
+			ensure!(nodes.contains(&remove), Error::<T>::NotExist);
+			ensure!(!nodes.contains(&add), Error::<T>::AlreadyJoined);
+
+			nodes.remove(&remove);
+			nodes.insert(add.clone());
+
+			WellKnownNodes::put(&nodes);
+			Owners::<T>::swap(&remove, &add);
+			AdditionalConnections::swap(&remove, &add);
+
+			Self::deposit_event(RawEvent::NodeSwapped(remove, add));
+		}
+
+		/// Reset all the well known nodes. This will not remove the ownership and additional
+		/// connections for the removed nodes. The node owner can perform further cleaning if
+		/// they decide to leave the network.
+		///
+		/// May only be called from `T::ResetOrigin`.
+		///
+		/// - `nodes`: the new nodes for the allow list.
+		#[weight = (T::WeightInfo::reset_well_known_nodes(), DispatchClass::Operational)]
+		pub fn reset_well_known_nodes(origin, nodes: Vec<(PeerId, T::AccountId)>) {
+			T::ResetOrigin::ensure_origin(origin)?;
+			ensure!(nodes.len() < T::MaxWellKnownNodes::get() as usize, Error::<T>::TooManyNodes);
+	
+			Self::initialize_nodes(&nodes);
+
+			Self::deposit_event(RawEvent::NodesReset(nodes));
+		}
+
+		/// A given node can be claimed by anyone. The owner should be the first to know its
+		/// PeerId, so claim it right away!
+		///
+		/// - `node`: identifier of the node.
+		#[weight = T::WeightInfo::claim_node()]
+		pub fn claim_node(origin, node: PeerId) {
+			let sender = ensure_signed(origin)?;
+			
+			ensure!(node.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+			ensure!(!Owners::<T>::contains_key(&node),Error::<T>::AlreadyClaimed);
+
+			Owners::<T>::insert(&node, &sender);
+			Self::deposit_event(RawEvent::NodeClaimed(node, sender));
+		}
+
+		/// A claim can be removed by its owner and get back the reservation. The additional
+		/// connections are also removed. You can't remove a claim on well known nodes, as it
+		/// needs to reach consensus among the network participants.
+		///
+		/// - `node`: identifier of the node.
+		#[weight = T::WeightInfo::remove_claim()]
+		pub fn remove_claim(origin, node: PeerId) {
+			let sender = ensure_signed(origin)?;
+
+			ensure!(node.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+			ensure!(Owners::<T>::contains_key(&node), Error::<T>::NotClaimed);
+			ensure!(Owners::<T>::get(&node) == sender, Error::<T>::NotOwner);
+			ensure!(!WellKnownNodes::get().contains(&node), Error::<T>::PermissionDenied);
+
+			Owners::<T>::remove(&node);
+			AdditionalConnections::remove(&node);
+
+			Self::deposit_event(RawEvent::ClaimRemoved(node, sender));
+		}
+
+		/// A node can be transferred to a new owner.
+		///
+		/// - `node`: identifier of the node.
+		/// - `owner`: new owner of the node.
+		#[weight = T::WeightInfo::transfer_node()]
+		pub fn transfer_node(origin, node: PeerId, owner: T::AccountId) {
+			let sender = ensure_signed(origin)?;
+
+			ensure!(node.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+			ensure!(Owners::<T>::contains_key(&node), Error::<T>::NotClaimed);
+			ensure!(Owners::<T>::get(&node) == sender, Error::<T>::NotOwner);
+
+			Owners::<T>::insert(&node, &owner);
+
+			Self::deposit_event(RawEvent::NodeTransferred(node, owner));
+		}
+
+		/// Add additional connections to a given node.
+		///
+		/// - `node`: identifier of the node.
+		/// - `connections`: additonal nodes from which the connections are allowed.
+		#[weight = T::WeightInfo::add_connections()]
+		pub fn add_connections(
+			origin,
+			node: PeerId,
+			connections: Vec<PeerId>
+		) {
+			let sender = ensure_signed(origin)?;
+
+			ensure!(node.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+			ensure!(Owners::<T>::contains_key(&node), Error::<T>::NotClaimed);
+			ensure!(Owners::<T>::get(&node) == sender, Error::<T>::NotOwner);
+
+			let mut nodes = AdditionalConnections::get(&node);
+
+			for add_node in connections.iter() {
+				if *add_node == node {
+					continue;
+				}
+				nodes.insert(add_node.clone());
+			}
+
+			AdditionalConnections::insert(&node, nodes);
+
+			Self::deposit_event(RawEvent::ConnectionsAdded(node, connections));
+		}
+
+		/// Remove additional connections of a given node.
+		///
+		/// - `node`: identifier of the node.
+		/// - `connections`: additonal nodes from which the connections are not allowed anymore.
+		#[weight = T::WeightInfo::remove_connections()]
+		pub fn remove_connections(
+			origin,
+			node: PeerId,
+			connections: Vec<PeerId>
+		) {
+			let sender = ensure_signed(origin)?;
+
+			ensure!(node.0.len() < T::MaxPeerIdLength::get() as usize, Error::<T>::PeerIdTooLong);
+			ensure!(Owners::<T>::contains_key(&node), Error::<T>::NotClaimed);
+			ensure!(Owners::<T>::get(&node) == sender, Error::<T>::NotOwner);
+
+			let mut nodes = AdditionalConnections::get(&node);
+
+			for remove_node in connections.iter() {
+				nodes.remove(remove_node);
+			}
+
+			AdditionalConnections::insert(&node, nodes);
+
+			Self::deposit_event(RawEvent::ConnectionsRemoved(node, connections));
+		}
+
+		/// Set reserved node every block. It may not be enabled depends on the offchain
+		/// worker settings when starting the node.
+		fn offchain_worker(now: T::BlockNumber) {
+			let network_state = sp_io::offchain::network_state();
+			match network_state {
+				Err(_) => debug::error!("Error: failed to get network state of node at {:?}", now),
+				Ok(state) => {
+					let encoded_peer = state.peer_id.0;
+					match Decode::decode(&mut &encoded_peer[..]) {
+						Err(_) => debug::error!("Error: failed to decode PeerId at {:?}", now),
+						Ok(node) => sp_io::offchain::set_authorized_nodes(
+							Self::get_authorized_nodes(&PeerId(node)),
+							true
+						)
+					}
+				}
+			}
+		}
+	}
+}
+
+impl<T: Trait> Module<T> {
+	fn initialize_nodes(nodes: &Vec<(PeerId, T::AccountId)>) {
+		let peer_ids = nodes.iter()
+			.map(|item| item.0.clone())
+			.collect::<BTreeSet<PeerId>>();
+		WellKnownNodes::put(&peer_ids);
+
+		for (node, who) in nodes.iter() {
+			Owners::<T>::insert(node, who);
+		}
+	}
+
+	fn get_authorized_nodes(node: &PeerId) -> Vec<PeerId> {
+		let mut nodes = AdditionalConnections::get(node);
+
+		let mut well_known_nodes = WellKnownNodes::get();
+		if well_known_nodes.contains(node) {
+			well_known_nodes.remove(node);
+			nodes.extend(well_known_nodes);
+		}
+
+		Vec::from_iter(nodes)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use frame_support::{
+		assert_ok, assert_noop, impl_outer_origin, weights::Weight,
+		parameter_types, ord_parameter_types,
+	};
+	use frame_system::EnsureSignedBy;
+	use sp_core::H256;
+	use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup, BadOrigin}, testing::Header};
+
+	impl_outer_origin! {
+		pub enum Origin for Test where system = frame_system {}
+	}
+
+	#[derive(Clone, Eq, PartialEq)]
+	pub struct Test;
+
+	parameter_types! {
+		pub const BlockHashCount: u64 = 250;
+		pub const MaximumBlockWeight: Weight = 1024;
+		pub const MaximumBlockLength: u32 = 2 * 1024;
+		pub const AvailableBlockRatio: Perbill = Perbill::one();
+	}
+	impl frame_system::Trait for Test {
+		type BaseCallFilter = ();
+		type Origin = Origin;
+		type Index = u64;
+		type BlockNumber = u64;
+		type Hash = H256;
+		type Call = ();
+		type Hashing = BlakeTwo256;
+		type AccountId = u64;
+		type Lookup = IdentityLookup<Self::AccountId>;
+		type Header = Header;
+		type Event = ();
+		type BlockHashCount = BlockHashCount;
+		type MaximumBlockWeight = MaximumBlockWeight;
+		type DbWeight = ();
+		type BlockExecutionWeight = ();
+		type ExtrinsicBaseWeight = ();
+		type MaximumExtrinsicWeight = MaximumBlockWeight;
+		type MaximumBlockLength = MaximumBlockLength;
+		type AvailableBlockRatio = AvailableBlockRatio;
+		type Version = ();
+		type ModuleToIndex = ();
+		type AccountData = ();
+		type OnNewAccount = ();
+		type OnKilledAccount = ();
+		type SystemWeightInfo = ();
+	}
+
+	ord_parameter_types! {
+		pub const One: u64 = 1;
+		pub const Two: u64 = 2;
+		pub const Three: u64 = 3;
+		pub const Four: u64 = 4;
+	}
+	parameter_types! {
+		pub const MaxWellKnownNodes: u32 = 4;
+		pub const MaxPeerIdLength: u32 = 2;
+	}
+	impl Trait for Test {
+		type Event = ();
+		type MaxWellKnownNodes = MaxWellKnownNodes;
+		type MaxPeerIdLength = MaxPeerIdLength;
+		type AddOrigin = EnsureSignedBy<One, u64>;
+		type RemoveOrigin = EnsureSignedBy<Two, u64>;
+		type SwapOrigin = EnsureSignedBy<Three, u64>;
+		type ResetOrigin = EnsureSignedBy<Four, u64>;
+		type WeightInfo = ();
+	}
+
+	type NodeAuthorization = Module<Test>;
+
+	fn test_node(id: u8) -> PeerId {
+		PeerId(vec![id])
+	}
+
+	fn new_test_ext() -> sp_io::TestExternalities {
+		let mut t = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();
+		GenesisConfig::<Test> {
+			nodes: vec![(test_node(10), 10), (test_node(20), 20), (test_node(30), 30)],
+		}.assimilate_storage(&mut t).unwrap();
+		t.into()
+	}
+
+	#[test]
+	fn add_well_known_node_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::add_well_known_node(Origin::signed(2), test_node(15), 15),
+				BadOrigin
+			);
+			assert_noop!(
+				NodeAuthorization::add_well_known_node(Origin::signed(1), PeerId(vec![1, 2, 3]), 15),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::add_well_known_node(Origin::signed(1), test_node(20), 20),
+				Error::<Test>::AlreadyJoined
+			);
+
+			assert_ok!(
+				NodeAuthorization::add_well_known_node(Origin::signed(1), test_node(15), 15)
+			);
+			assert_eq!(
+				WellKnownNodes::get(),
+				BTreeSet::from_iter(vec![test_node(10), test_node(15), test_node(20), test_node(30)])
+			);
+			assert_eq!(Owners::<Test>::get(test_node(10)), 10);
+			assert_eq!(Owners::<Test>::get(test_node(20)), 20);
+			assert_eq!(Owners::<Test>::get(test_node(30)), 30);
+			assert_eq!(Owners::<Test>::get(test_node(15)), 15);
+
+			assert_noop!(
+				NodeAuthorization::add_well_known_node(Origin::signed(1), test_node(25), 25),
+				Error::<Test>::TooManyNodes
+			);
+		});
+	}
+
+	#[test]
+	fn remove_well_known_node_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::remove_well_known_node(Origin::signed(3), test_node(20)),
+				BadOrigin
+			);
+			assert_noop!(
+				NodeAuthorization::remove_well_known_node(Origin::signed(2), PeerId(vec![1, 2, 3])),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::remove_well_known_node(Origin::signed(2), test_node(40)),
+				Error::<Test>::NotExist
+			);
+
+			AdditionalConnections::insert(
+				test_node(20),
+				BTreeSet::from_iter(vec![test_node(40)])
+			);
+			assert!(AdditionalConnections::contains_key(test_node(20)));
+
+			assert_ok!(
+				NodeAuthorization::remove_well_known_node(Origin::signed(2), test_node(20))
+			);
+			assert_eq!(
+				WellKnownNodes::get(),
+				BTreeSet::from_iter(vec![test_node(10), test_node(30)])
+			);
+			assert!(!Owners::<Test>::contains_key(test_node(20)));
+			assert!(!AdditionalConnections::contains_key(test_node(20)));
+		});
+	}
+
+	#[test]
+	fn swap_well_known_node_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::swap_well_known_node(
+					Origin::signed(4), test_node(20), test_node(5)
+				),
+				BadOrigin
+			);
+			assert_noop!(
+				NodeAuthorization::swap_well_known_node(
+					Origin::signed(3), PeerId(vec![1, 2, 3]), test_node(20)
+				),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::swap_well_known_node(
+					Origin::signed(3), test_node(20), PeerId(vec![1, 2, 3])
+				),
+				Error::<Test>::PeerIdTooLong
+			);
+
+			assert_ok!(
+				NodeAuthorization::swap_well_known_node(
+					Origin::signed(3), test_node(20), test_node(20)
+				)
+			);
+			assert_eq!(
+				WellKnownNodes::get(),
+				BTreeSet::from_iter(vec![test_node(10), test_node(20), test_node(30)])
+			);
+
+			assert_noop!(
+				NodeAuthorization::swap_well_known_node(
+					Origin::signed(3), test_node(15), test_node(5)
+				),
+				Error::<Test>::NotExist
+			);
+			assert_noop!(
+				NodeAuthorization::swap_well_known_node(
+					Origin::signed(3), test_node(20), test_node(30)
+				),
+				Error::<Test>::AlreadyJoined
+			);
+
+			AdditionalConnections::insert(
+				test_node(20),
+				BTreeSet::from_iter(vec![test_node(15)])
+			);
+			assert_ok!(
+				NodeAuthorization::swap_well_known_node(
+					Origin::signed(3), test_node(20), test_node(5)
+				)
+			);
+			assert_eq!(
+				WellKnownNodes::get(),
+				BTreeSet::from_iter(vec![test_node(5), test_node(10), test_node(30)])
+			);
+			assert!(!Owners::<Test>::contains_key(test_node(20)));
+			assert_eq!(Owners::<Test>::get(test_node(5)), 20);
+			assert!(!AdditionalConnections::contains_key(test_node(20)));
+			assert_eq!(
+				AdditionalConnections::get(test_node(5)),
+				BTreeSet::from_iter(vec![test_node(15)])
+			);
+		});
+	}
+
+	#[test]
+	fn reset_well_known_nodes_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::reset_well_known_nodes(
+					Origin::signed(3),
+					vec![(test_node(15), 15), (test_node(5), 5), (test_node(20), 20)]
+				),
+				BadOrigin
+			);
+			assert_noop!(
+				NodeAuthorization::reset_well_known_nodes(
+					Origin::signed(4),
+					vec![
+						(test_node(15), 15),
+						(test_node(5), 5),
+						(test_node(20), 20),
+						(test_node(25), 25),
+					]
+				),
+				Error::<Test>::TooManyNodes
+			);
+
+			assert_ok!(
+				NodeAuthorization::reset_well_known_nodes(
+					Origin::signed(4),
+					vec![(test_node(15), 15), (test_node(5), 5), (test_node(20), 20)]
+				)
+			);
+			assert_eq!(
+				WellKnownNodes::get(),
+				BTreeSet::from_iter(vec![test_node(5), test_node(15), test_node(20)])
+			);
+			assert_eq!(Owners::<Test>::get(test_node(5)), 5);
+			assert_eq!(Owners::<Test>::get(test_node(15)), 15);
+			assert_eq!(Owners::<Test>::get(test_node(20)), 20);
+		});
+	}
+
+	#[test]
+	fn claim_node_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::claim_node(Origin::signed(1), PeerId(vec![1, 2, 3])),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::claim_node(Origin::signed(1), test_node(20)),
+				Error::<Test>::AlreadyClaimed
+			);
+
+			assert_ok!(NodeAuthorization::claim_node(Origin::signed(15), test_node(15)));
+			assert_eq!(Owners::<Test>::get(test_node(15)), 15);
+		});
+	}
+
+	#[test]
+	fn remove_claim_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::remove_claim(Origin::signed(15), PeerId(vec![1, 2, 3])),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::remove_claim(Origin::signed(15), test_node(15)),
+				Error::<Test>::NotClaimed
+			);
+
+			assert_noop!(
+				NodeAuthorization::remove_claim(Origin::signed(15), test_node(20)),
+				Error::<Test>::NotOwner
+			);
+
+			assert_noop!(
+				NodeAuthorization::remove_claim(Origin::signed(20), test_node(20)),
+				Error::<Test>::PermissionDenied
+			);
+
+			Owners::<Test>::insert(test_node(15), 15);
+			AdditionalConnections::insert(
+				test_node(15),
+				BTreeSet::from_iter(vec![test_node(20)])
+			);
+			assert_ok!(NodeAuthorization::remove_claim(Origin::signed(15), test_node(15)));
+			assert!(!Owners::<Test>::contains_key(test_node(15)));
+			assert!(!AdditionalConnections::contains_key(test_node(15)));
+		});
+	}
+
+	#[test]
+	fn transfer_node_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::transfer_node(Origin::signed(15), PeerId(vec![1, 2, 3]), 10),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::transfer_node(Origin::signed(15), test_node(15), 10),
+				Error::<Test>::NotClaimed
+			);
+
+			assert_noop!(
+				NodeAuthorization::transfer_node(Origin::signed(15), test_node(20), 10),
+				Error::<Test>::NotOwner
+			);
+
+			assert_ok!(NodeAuthorization::transfer_node(Origin::signed(20), test_node(20), 15));
+			assert_eq!(Owners::<Test>::get(test_node(20)), 15);
+		});
+	}
+
+	#[test]
+	fn add_connections_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::add_connections(
+					Origin::signed(15), PeerId(vec![1, 2, 3]), vec![test_node(5)]
+				),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::add_connections(
+					Origin::signed(15), test_node(15), vec![test_node(5)]
+				),
+				Error::<Test>::NotClaimed
+			);
+
+			assert_noop!(
+				NodeAuthorization::add_connections(
+					Origin::signed(15), test_node(20), vec![test_node(5)]
+				),
+				Error::<Test>::NotOwner
+			);
+
+			assert_ok!(
+				NodeAuthorization::add_connections(
+					Origin::signed(20),
+					test_node(20),
+					vec![test_node(15), test_node(5), test_node(25), test_node(20)]
+				)
+			);
+			assert_eq!(
+				AdditionalConnections::get(test_node(20)),
+				BTreeSet::from_iter(vec![test_node(5), test_node(15), test_node(25)])
+			);
+		});
+	}
+
+	#[test]
+	fn remove_connections_works() {
+		new_test_ext().execute_with(|| {
+			assert_noop!(
+				NodeAuthorization::remove_connections(
+					Origin::signed(15), PeerId(vec![1, 2, 3]), vec![test_node(5)]
+				),
+				Error::<Test>::PeerIdTooLong
+			);
+			assert_noop!(
+				NodeAuthorization::remove_connections(
+					Origin::signed(15), test_node(15), vec![test_node(5)]
+				),
+				Error::<Test>::NotClaimed
+			);
+
+			assert_noop!(
+				NodeAuthorization::remove_connections(
+					Origin::signed(15), test_node(20), vec![test_node(5)]
+				),
+				Error::<Test>::NotOwner
+			);
+
+			AdditionalConnections::insert(
+				test_node(20),
+				BTreeSet::from_iter(vec![test_node(5), test_node(15), test_node(25)])
+			);
+			assert_ok!(
+				NodeAuthorization::remove_connections(
+					Origin::signed(20),
+					test_node(20),
+					vec![test_node(15), test_node(5)]
+				)
+			);
+			assert_eq!(
+				AdditionalConnections::get(test_node(20)),
+				BTreeSet::from_iter(vec![test_node(25)])
+			);
+		});
+	}
+
+	#[test]
+	fn get_authorized_nodes_works() {
+		new_test_ext().execute_with(|| {
+			AdditionalConnections::insert(
+				test_node(20),
+				BTreeSet::from_iter(vec![test_node(5), test_node(15), test_node(25)])
+			);
+
+			let mut authorized_nodes = Module::<Test>::get_authorized_nodes(&test_node(20));
+			authorized_nodes.sort();
+			assert_eq!(
+				authorized_nodes,
+				vec![test_node(5), test_node(10), test_node(15), test_node(25), test_node(30)]
+			);
+		});
+	}
+}
diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs
index 2a40972166e14..94f6bb2967a0b 100644
--- a/primitives/core/src/lib.rs
+++ b/primitives/core/src/lib.rs
@@ -32,6 +32,7 @@ macro_rules! map {
 	);
 }
 
+use sp_runtime_interface::pass_by::{PassByEnum, PassByInner};
 use sp_std::prelude::*;
 use sp_std::ops::Deref;
 #[cfg(feature = "std")]
@@ -176,6 +177,18 @@ impl sp_std::ops::Deref for OpaqueMetadata {
 	}
 }
 
+/// Simple blob to hold a `PeerId` without committing to its format.
+#[derive(Default, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, RuntimeDebug, PassByInner)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
+pub struct OpaquePeerId(pub Vec<u8>);
+
+impl OpaquePeerId {
+	/// Create new `OpaquePeerId`
+	pub fn new(vec: Vec<u8>) -> Self {
+		OpaquePeerId(vec)
+	}
+}
+
 /// Something that is either a native or an encoded value.
 #[cfg(feature = "std")]
 pub enum NativeOrEncoded<R> {
@@ -257,7 +270,7 @@ pub trait TypeId {
 /// A log level matching the one from `log` crate.
 ///
 /// Used internally by `sp_io::log` method.
-#[derive(Encode, Decode, sp_runtime_interface::pass_by::PassByEnum, Copy, Clone)]
+#[derive(Encode, Decode, PassByEnum, Copy, Clone)]
 pub enum LogLevel {
 	/// `Error` log level.
 	Error = 1,
diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs
index b2ff3552135ce..4768496c4a508 100644
--- a/primitives/core/src/offchain/mod.rs
+++ b/primitives/core/src/offchain/mod.rs
@@ -19,7 +19,7 @@
 
 use codec::{Encode, Decode};
 use sp_std::{prelude::{Vec, Box}, convert::TryFrom};
-use crate::RuntimeDebug;
+use crate::{OpaquePeerId, RuntimeDebug};
 use sp_runtime_interface::pass_by::{PassByCodec, PassByInner, PassByEnum};
 
 pub use crate::crypto::KeyTypeId;
@@ -184,23 +184,12 @@ impl TryFrom<u32> for HttpRequestStatus {
 #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByCodec)]
 #[cfg_attr(feature = "std", derive(Default))]
 pub struct OpaqueNetworkState {
-	/// PeerId of the local node.
+	/// PeerId of the local node in SCALE encoded.
 	pub peer_id: OpaquePeerId,
 	/// List of addresses the node knows it can be reached as.
 	pub external_addresses: Vec<OpaqueMultiaddr>,
 }
 
-/// Simple blob to hold a `PeerId` without committing to its format.
-#[derive(Default, Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
-pub struct OpaquePeerId(pub Vec<u8>);
-
-impl OpaquePeerId {
-	/// Create new `OpaquePeerId`
-	pub fn new(vec: Vec<u8>) -> Self {
-		OpaquePeerId(vec)
-	}
-}
-
 /// Simple blob to hold a `Multiaddr` without committing to its format.
 #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
 pub struct OpaqueMultiaddr(pub Vec<u8>);
@@ -277,6 +266,8 @@ pub enum Capability {
 	OffchainWorkerDbRead = 32,
 	/// Access to offchain worker DB (writes).
 	OffchainWorkerDbWrite = 64,
+	/// Manage the authorized nodes
+	NodeAuthorization = 128,
 }
 
 /// A set of capabilities
@@ -495,6 +486,18 @@ pub trait Externalities: Send {
 		buffer: &mut [u8],
 		deadline: Option<Timestamp>
 	) -> Result<usize, HttpError>;
+
+	/// Set the authorized nodes from runtime.
+	///
+	/// In a permissioned network, the connections between nodes need to reach a
+	/// consensus between participants.
+	///
+	/// - `nodes`: a set of nodes which are allowed to connect for the local node.
+	/// each one is identified with an `OpaquePeerId`, here it just use plain bytes
+	/// without any encoding. Invalid `OpaquePeerId`s are silently ignored.
+	/// - `authorized_only`: if true, only the authorized nodes are allowed to connect,
+	/// otherwise unauthorized nodes can also be connected through other mechanism.
+	fn set_authorized_nodes(&mut self, nodes: Vec<OpaquePeerId>, authorized_only: bool);
 }
 
 impl<T: Externalities + ?Sized> Externalities for Box<T> {
@@ -573,6 +576,10 @@ impl<T: Externalities + ?Sized> Externalities for Box<T> {
 	) -> Result<usize, HttpError> {
 		(&mut **self).http_response_read_body(request_id, buffer, deadline)
 	}
+
+	fn set_authorized_nodes(&mut self, nodes: Vec<OpaquePeerId>, authorized_only: bool) {
+		(&mut **self).set_authorized_nodes(nodes, authorized_only)
+	}
 }
 
 /// An `OffchainExternalities` implementation with limited capabilities.
@@ -691,6 +698,11 @@ impl<T: Externalities> Externalities for LimitedExternalities<T> {
 		self.check(Capability::Http, "http_response_read_body");
 		self.externalities.http_response_read_body(request_id, buffer, deadline)
 	}
+
+	fn set_authorized_nodes(&mut self, nodes: Vec<OpaquePeerId>, authorized_only: bool) {
+		self.check(Capability::NodeAuthorization, "set_authorized_nodes");
+		self.externalities.set_authorized_nodes(nodes, authorized_only)
+	}
 }
 
 #[cfg(feature = "std")]
diff --git a/primitives/core/src/offchain/testing.rs b/primitives/core/src/offchain/testing.rs
index c939c5cfccc14..3fe34cc0cfa7b 100644
--- a/primitives/core/src/offchain/testing.rs
+++ b/primitives/core/src/offchain/testing.rs
@@ -24,6 +24,7 @@ use std::{
 	collections::{BTreeMap, VecDeque},
 	sync::Arc,
 };
+use crate::OpaquePeerId;
 use crate::offchain::{
 	self,
 	storage::{InMemOffchainStorage, OffchainOverlayedChange, OffchainOverlayedChanges},
@@ -375,6 +376,10 @@ impl offchain::Externalities for TestOffchainExt {
 			Err(HttpError::IoError)
 		}
 	}
+
+	fn set_authorized_nodes(&mut self, _nodes: Vec<OpaquePeerId>, _authorized_only: bool) {
+		unimplemented!()
+	}
 }
 
 /// The internal state of the fake transaction pool.
diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs
index 59d1c4f37ef27..3248efaa17e50 100644
--- a/primitives/io/src/lib.rs
+++ b/primitives/io/src/lib.rs
@@ -42,7 +42,7 @@ use sp_core::{
 };
 
 use sp_core::{
-	crypto::KeyTypeId, ed25519, sr25519, ecdsa, H256, LogLevel,
+	OpaquePeerId, crypto::KeyTypeId, ed25519, sr25519, ecdsa, H256, LogLevel,
 	offchain::{
 		Timestamp, HttpRequestId, HttpRequestStatus, HttpError, StorageKind, OpaqueNetworkState,
 	},
@@ -960,6 +960,13 @@ pub trait Offchain {
 			.http_response_read_body(request_id, buffer, deadline)
 			.map(|r| r as u32)
 	}
+
+	/// Set the authorized nodes and authorized_only flag.
+	fn set_authorized_nodes(&mut self, nodes: Vec<OpaquePeerId>, authorized_only: bool) {
+		self.extension::<OffchainExt>()
+			.expect("set_authorized_nodes can be called only in the offchain worker context")
+			.set_authorized_nodes(nodes, authorized_only)
+	}
 }
 
 /// Wasm only interface that provides functions for calling into the allocator.

From 0f613523428356e239bdfaaaf512078d236f2b27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Thu, 10 Sep 2020 23:04:24 +0200
Subject: [PATCH 068/122] Rename `TRIGGER_WASM_BUILD` to `FORCE_WASM_BUILD`
 (#7080)

Because apparently I can not speak properly ;)
---
 docs/README.adoc                     | 6 +++---
 utils/wasm-builder-runner/src/lib.rs | 4 ++--
 utils/wasm-builder/README.md         | 6 +++---
 utils/wasm-builder/src/lib.rs        | 6 +++---
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/README.adoc b/docs/README.adoc
index e1ed86c2d5269..7f3d50faac7d6 100644
--- a/docs/README.adoc
+++ b/docs/README.adoc
@@ -318,9 +318,9 @@ we support multiple environment variables:
                               for `cargo check` runs.
 * `WASM_BUILD_TYPE` - Sets the build type for building WASM binaries. Supported values are `release` or `debug`.
                       By default the build type is equal to the build type used by the main build.
-* `TRIGGER_WASM_BUILD` - Can be set to trigger a WASM build. On subsequent calls the value of the variable
-                         needs to change. As WASM builder instructs `cargo` to watch for file changes
-                         this environment variable should only be required in certain circumstances.
+* `FORCE_WASM_BUILD` - Can be set to force a WASM build. On subsequent calls the value of the variable
+                       needs to change. As WASM builder instructs `cargo` to watch for file changes
+                       this environment variable should only be required in certain circumstances.
 * `WASM_TARGET_DIRECTORY` - Will copy release build WASM binary to the given directory. The path needs
                             to be absolute.
 * `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary.
diff --git a/utils/wasm-builder-runner/src/lib.rs b/utils/wasm-builder-runner/src/lib.rs
index 7990ea2bb97d1..b0b1ac479e791 100644
--- a/utils/wasm-builder-runner/src/lib.rs
+++ b/utils/wasm-builder-runner/src/lib.rs
@@ -44,7 +44,7 @@ const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD";
 const DUMMY_WASM_BINARY_ENV: &str = "BUILD_DUMMY_WASM_BINARY";
 
 /// Environment variable that makes sure the WASM build is triggered.
-const TRIGGER_WASM_BUILD_ENV: &str = "TRIGGER_WASM_BUILD";
+const FORCE_WASM_BUILD_ENV: &str = "FORCE_WASM_BUILD";
 
 /// Replace all backslashes with slashes.
 fn replace_back_slashes<T: ToString>(path: T) -> String {
@@ -476,6 +476,6 @@ fn generate_rerun_if_changed_instructions() {
 	// Make sure that the `build.rs` is called again if one of the following env variables changes.
 	println!("cargo:rerun-if-env-changed={}", SKIP_BUILD_ENV);
 	println!("cargo:rerun-if-env-changed={}", DUMMY_WASM_BINARY_ENV);
-	println!("cargo:rerun-if-env-changed={}", TRIGGER_WASM_BUILD_ENV);
+	println!("cargo:rerun-if-env-changed={}", FORCE_WASM_BUILD_ENV);
 	println!("cargo:rerun-if-env-changed={}", generate_crate_skip_build_env_name());
 }
diff --git a/utils/wasm-builder/README.md b/utils/wasm-builder/README.md
index b72e7e16d4ff4..1e24d2cebab32 100644
--- a/utils/wasm-builder/README.md
+++ b/utils/wasm-builder/README.md
@@ -51,9 +51,9 @@ By using environment variables, you can configure which Wasm binaries are built
                              for `cargo check` runs.
 - `WASM_BUILD_TYPE` - Sets the build type for building wasm binaries. Supported values are `release` or `debug`.
                       By default the build type is equal to the build type used by the main build.
-- `TRIGGER_WASM_BUILD` - Can be set to trigger a wasm build. On subsequent calls the value of the variable
-                         needs to change. As wasm builder instructs `cargo` to watch for file changes
-                         this environment variable should only be required in certain circumstances.
+- `FORCE_WASM_BUILD` - Can be set to force a wasm build. On subsequent calls the value of the variable
+                       needs to change. As wasm builder instructs `cargo` to watch for file changes
+                       this environment variable should only be required in certain circumstances.
 - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary.
 - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build.
 - `WASM_TARGET_DIRECTORY` - Will copy any build wasm binary to the given directory. The path needs
diff --git a/utils/wasm-builder/src/lib.rs b/utils/wasm-builder/src/lib.rs
index f1a1c7729a070..500025c29678e 100644
--- a/utils/wasm-builder/src/lib.rs
+++ b/utils/wasm-builder/src/lib.rs
@@ -68,9 +68,9 @@
 //!                              for `cargo check` runs.
 //! - `WASM_BUILD_TYPE` - Sets the build type for building wasm binaries. Supported values are `release` or `debug`.
 //!                       By default the build type is equal to the build type used by the main build.
-//! - `TRIGGER_WASM_BUILD` - Can be set to trigger a wasm build. On subsequent calls the value of the variable
-//!                          needs to change. As wasm builder instructs `cargo` to watch for file changes
-//!                          this environment variable should only be required in certain circumstances.
+//! - `FORCE_WASM_BUILD` - Can be set to force a wasm build. On subsequent calls the value of the variable
+//!                        needs to change. As wasm builder instructs `cargo` to watch for file changes
+//!                        this environment variable should only be required in certain circumstances.
 //! - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary.
 //! - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build.
 //! - `WASM_TARGET_DIRECTORY` - Will copy any build wasm binary to the given directory. The path needs

From 0fd0d957f2535fa01e81a8dc2ee4b02c1654bdc2 Mon Sep 17 00:00:00 2001
From: Guillaume Thiolliere <gui.thiolliere@gmail.com>
Date: Fri, 11 Sep 2020 10:39:44 +0200
Subject: [PATCH 069/122] Make decoding of `compact<perthing>` saturating
 instead of invalid (#7062)

* make decoding of cmopact<perthing> saturating

* fix stable build

* Update primitives/arithmetic/src/per_things.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update primitives/arithmetic/src/per_things.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
---
 primitives/arithmetic/src/per_things.rs | 32 ++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/primitives/arithmetic/src/per_things.rs b/primitives/arithmetic/src/per_things.rs
index cf53988b33d31..035a704ba3009 100644
--- a/primitives/arithmetic/src/per_things.rs
+++ b/primitives/arithmetic/src/per_things.rs
@@ -323,9 +323,28 @@ macro_rules! implement_per_thing {
 		///
 		#[doc = $title]
 		#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
-		#[derive(Encode, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, CompactAs)]
+		#[derive(Encode, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug)]
 		pub struct $name($type);
 
+		/// Implementation makes any compact encoding of `PerThing::Inner` valid,
+		/// when decoding it will saturate up to `PerThing::ACCURACY`.
+		impl CompactAs for $name {
+			type As = $type;
+			fn encode_as(&self) -> &Self::As {
+				&self.0
+			}
+			fn decode_from(x: Self::As) -> Self {
+				// Saturates if `x` is more than `$max` internally. 
+				Self::from_parts(x)
+			}
+		}
+
+		impl From<codec::Compact<$name>> for $name {
+			fn from(x: codec::Compact<$name>) -> $name {
+				x.0
+			}
+		}
+
 		impl PerThing for $name {
 			type Inner = $type;
 			type Upper = $upper_type;
@@ -1166,6 +1185,17 @@ macro_rules! implement_per_thing {
 				// deconstruct is also const, hence it can be called in const rhs.
 				const C5: bool = C1.deconstruct() == 0;
 			}
+
+			#[test]
+			fn compact_decoding_saturate_when_beyond_accuracy() {
+				use num_traits::Bounded;
+				use codec::Compact;
+
+				let p = Compact::<$name>::decode(&mut &Compact(<$type>::max_value()).encode()[..])
+					.unwrap();
+				assert_eq!((p.0).0, $max);
+				assert_eq!($name::from(p), $name::max_value());
+			}
 		}
 	};
 }

From 006f3f0daff8cfaf7932158a225cd3800148bcac Mon Sep 17 00:00:00 2001
From: cheme <emericchevalier.pro@gmail.com>
Date: Fri, 11 Sep 2020 12:11:25 +0200
Subject: [PATCH 070/122] state_machine no_std witness externalities (#6934)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* checkpoint before removing CT from change trie

* before trie backend without tx

* undo

* Started no transaction, but would need using a different root
calculation method, out of the scope of this pr, will roll back.

* Remove NoTransaction.

* partially address review.
dummy stats implementation for no_std.

* Remove ChangeTrieOverlay.

* modified function

* Remove witness_ext

* need noops changes root

* update from cumulus branch

* line break

* remove warning

* line break

* From review: renamings and stats active in no std (except time).

* include cache, exclude change trie cache with individual temporary bad looking
no_std check

* little test

* fuse imports and filter_map prepare_extrinsics_input_inner fold.

* put back ExtInner into Ext, awkward double proto for new function.

* Apply suggestions from code review

* Update primitives/state-machine/Cargo.toml

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
---
 Cargo.lock                                    |    3 +-
 primitives/externalities/Cargo.toml           |   17 +-
 primitives/externalities/src/extensions.rs    |    7 +-
 primitives/externalities/src/lib.rs           |    4 +-
 primitives/state-machine/Cargo.toml           |   43 +-
 primitives/state-machine/src/backend.rs       |    9 +-
 .../state-machine/src/changes_trie/build.rs   |   21 +-
 .../state-machine/src/changes_trie/mod.rs     |    3 -
 primitives/state-machine/src/error.rs         |    4 +-
 primitives/state-machine/src/ext.rs           |  150 +-
 primitives/state-machine/src/lib.rs           | 1421 +++++++++--------
 .../src/overlayed_changes/changeset.rs        |   34 +-
 .../src/overlayed_changes/mod.rs              |  126 +-
 primitives/state-machine/src/stats.rs         |    7 +-
 primitives/state-machine/src/trie_backend.rs  |    9 +-
 .../state-machine/src/trie_backend_essence.rs |   39 +-
 test-utils/runtime/Cargo.toml                 |    5 +-
 test-utils/runtime/src/lib.rs                 |   71 +-
 18 files changed, 1194 insertions(+), 779 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3c525c6c063e6..741f2ba7c48c4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8424,7 +8424,6 @@ version = "0.8.0-rc6"
 dependencies = [
  "hash-db",
  "hex-literal",
- "itertools 0.9.0",
  "log",
  "num-traits",
  "parity-scale-codec",
@@ -8436,6 +8435,7 @@ dependencies = [
  "sp-externalities",
  "sp-panic-handler",
  "sp-runtime",
+ "sp-std",
  "sp-trie",
  "trie-db",
  "trie-root",
@@ -8833,6 +8833,7 @@ dependencies = [
  "sp-consensus-aura",
  "sp-consensus-babe",
  "sp-core",
+ "sp-externalities",
  "sp-finality-grandpa",
  "sp-inherents",
  "sp-io",
diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml
index 17184ca69402f..952912bee592c 100644
--- a/primitives/externalities/Cargo.toml
+++ b/primitives/externalities/Cargo.toml
@@ -13,7 +13,16 @@ documentation = "https://docs.rs/sp-externalities"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-sp-storage = { version = "2.0.0-rc6", path = "../storage" }
-sp-std = { version = "2.0.0-rc6", path = "../std" }
-environmental = { version = "1.1.1" }
-codec = { package = "parity-scale-codec", version = "1.3.1" }
+sp-storage = { version = "2.0.0-rc6", path = "../storage", default-features = false }
+sp-std = { version = "2.0.0-rc6", path = "../std", default-features = false }
+environmental = { version = "1.1.1", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+	"codec/std",
+	"environmental/std",
+  "sp-std/std",
+  "sp-storage/std",
+]
diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs
index 08d81e46c88fc..d79f99d3344ea 100644
--- a/primitives/externalities/src/extensions.rs
+++ b/primitives/externalities/src/extensions.rs
@@ -22,7 +22,9 @@
 //!
 //! It is required that each extension implements the [`Extension`] trait.
 
-use std::{collections::HashMap, collections::hash_map::Entry, any::{Any, TypeId}, ops::DerefMut};
+use sp_std::{
+	collections::btree_map::{BTreeMap, Entry}, any::{Any, TypeId}, ops::DerefMut, boxed::Box,
+};
 use crate::Error;
 
 /// Marker trait for types that should be registered as [`Externalities`](crate::Externalities) extension.
@@ -104,9 +106,10 @@ pub trait ExtensionStore {
 /// Stores extensions that should be made available through the externalities.
 #[derive(Default)]
 pub struct Extensions {
-	extensions: HashMap<TypeId, Box<dyn Extension>>,
+	extensions: BTreeMap<TypeId, Box<dyn Extension>>,
 }
 
+#[cfg(feature = "std")]
 impl std::fmt::Debug for Extensions {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		write!(f, "Extensions: ({})", self.extensions.len())
diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs
index 01570e0bfadd3..388482964f18c 100644
--- a/primitives/externalities/src/lib.rs
+++ b/primitives/externalities/src/lib.rs
@@ -15,6 +15,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![cfg_attr(not(feature = "std"), no_std)]
+
 //! Substrate externalities abstraction
 //!
 //! The externalities mainly provide access to storage and to registered extensions. Extensions
@@ -23,7 +25,7 @@
 //!
 //! This crate exposes the main [`Externalities`] trait.
 
-use std::any::{Any, TypeId};
+use sp_std::{any::{Any, TypeId}, vec::Vec, boxed::Box};
 
 use sp_storage::{ChildInfo, TrackedStorageKey};
 
diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml
index 88d3b5a75c156..f34fabdd88908 100644
--- a/primitives/state-machine/Cargo.toml
+++ b/primitives/state-machine/Cargo.toml
@@ -13,20 +13,20 @@ documentation = "https://docs.rs/sp-state-machine"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-log = "0.4.8"
-parking_lot = "0.10.0"
-hash-db = "0.15.2"
-trie-db = "0.22.0"
-trie-root = "0.16.0"
-sp-trie = { version = "2.0.0-rc6", path = "../trie" }
-sp-core = { version = "2.0.0-rc6", path = "../core" }
-sp-panic-handler = { version = "2.0.0-rc6", path = "../panic-handler" }
-codec = { package = "parity-scale-codec", version = "1.3.1" }
-num-traits = "0.2.8"
-rand = "0.7.2"
-sp-externalities = { version = "0.8.0-rc6", path = "../externalities" }
-itertools = "0.9"
+log = { version = "0.4.8", optional = true }
+parking_lot = { version = "0.10.0", optional = true }
+hash-db = { version = "0.15.2", default-features = false }
+trie-db = { version = "0.22.0", default-features = false }
+trie-root = { version = "0.16.0", default-features = false }
+sp-trie = { version = "2.0.0-rc6", path = "../trie", default-features = false }
+sp-core = { version = "2.0.0-rc6", path = "../core", default-features = false }
+sp-panic-handler = { version = "2.0.0-rc6", path = "../panic-handler", optional = true }
+codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false }
+num-traits = { version = "0.2.8", default-features = false }
+rand = { version = "0.7.2", optional = true }
+sp-externalities = { version = "0.8.0-rc6", path = "../externalities", default-features = false }
 smallvec = "1.4.1"
+sp-std = { version = "2.0.0-rc6", default-features = false, path = "../std" }
 
 [dev-dependencies]
 hex-literal = "0.3.1"
@@ -34,4 +34,19 @@ sp-runtime = { version = "2.0.0-rc6", path = "../runtime" }
 pretty_assertions = "0.6.1"
 
 [features]
-default = []
+default = ["std"]
+std = [
+	"codec/std",
+	"hash-db/std",
+	"num-traits/std",
+  "sp-core/std",
+  "sp-externalities/std",
+  "sp-std/std",
+	"sp-trie/std",
+	"trie-db/std",
+	"trie-root/std",
+	"log",
+	"parking_lot",
+	"rand",
+  "sp-panic-handler",
+]
diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs
index 6ced5ed0e521e..360fe9a985682 100644
--- a/primitives/state-machine/src/backend.rs
+++ b/primitives/state-machine/src/backend.rs
@@ -20,7 +20,6 @@
 use hash_db::Hasher;
 use codec::{Decode, Encode};
 use sp_core::{
-	traits::RuntimeCode,
 	storage::{ChildInfo, well_known_keys, TrackedStorageKey}
 };
 use crate::{
@@ -28,12 +27,15 @@ use crate::{
 	trie_backend_essence::TrieBackendStorage,
 	UsageInfo, StorageKey, StorageValue, StorageCollection, ChildStorageCollection,
 };
+use sp_std::vec::Vec;
+#[cfg(feature = "std")]
+use sp_core::traits::RuntimeCode;
 
 /// A state backend is used to read state data and can have changes committed
 /// to it.
 ///
 /// The clone operation (if implemented) should be cheap.
-pub trait Backend<H: Hasher>: std::fmt::Debug {
+pub trait Backend<H: Hasher>: sp_std::fmt::Debug {
 	/// An error type when fetching data is not possible.
 	type Error: super::Error;
 
@@ -375,11 +377,13 @@ pub(crate) fn insert_into_memory_db<H, I>(mdb: &mut sp_trie::MemoryDB<H>, input:
 }
 
 /// Wrapper to create a [`RuntimeCode`] from a type that implements [`Backend`].
+#[cfg(feature = "std")]
 pub struct BackendRuntimeCode<'a, B, H> {
 	backend: &'a B,
 	_marker: std::marker::PhantomData<H>,
 }
 
+#[cfg(feature = "std")]
 impl<'a, B: Backend<H>, H: Hasher> sp_core::traits::FetchRuntimeCode for
 	BackendRuntimeCode<'a, B, H>
 {
@@ -388,6 +392,7 @@ impl<'a, B: Backend<H>, H: Hasher> sp_core::traits::FetchRuntimeCode for
 	}
 }
 
+#[cfg(feature = "std")]
 impl<'a, B: Backend<H>, H: Hasher> BackendRuntimeCode<'a, B, H> where H::Out: Encode {
 	/// Create a new instance.
 	pub fn new(backend: &'a B) -> Self {
diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs
index 675904578be97..b23481411ae27 100644
--- a/primitives/state-machine/src/changes_trie/build.rs
+++ b/primitives/state-machine/src/changes_trie/build.rs
@@ -140,8 +140,15 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>(
 		Number: BlockNumber,
 {
 	changes
-		.filter(|( _, v)| v.extrinsics().next().is_some())
-		.try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex<Number>, Vec<u32>)>, (k, v)| {
+		.filter_map(|(k, v)| {
+			let extrinsics = v.extrinsics();
+			if !extrinsics.is_empty() {
+				Some((k, extrinsics))
+			} else {
+				None
+			}
+		})
+		.try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex<Number>, Vec<u32>)>, (k, extrinsics)| {
 			match map.entry(k) {
 				Entry::Vacant(entry) => {
 					// ignore temporary values (values that have null value at the end of operation
@@ -161,7 +168,7 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>(
 						}
 					};
 
-					let extrinsics = v.extrinsics().cloned().collect();
+					let extrinsics = extrinsics.into_iter().collect();
 					entry.insert((ExtrinsicIndex {
 						block: block.clone(),
 						key: k.to_vec(),
@@ -170,11 +177,11 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>(
 				Entry::Occupied(mut entry) => {
 					// we do not need to check for temporary values here, because entry is Occupied
 					// AND we are checking it before insertion
-					let extrinsics = &mut entry.get_mut().1;
-					extrinsics.extend(
-						v.extrinsics().cloned()
+					let entry_extrinsics = &mut entry.get_mut().1;
+					entry_extrinsics.extend(
+						extrinsics.into_iter()
 					);
-					extrinsics.sort();
+					entry_extrinsics.sort();
 				},
 			}
 
diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs
index 04322f1d5930c..fd7b38c052f9e 100644
--- a/primitives/state-machine/src/changes_trie/mod.rs
+++ b/primitives/state-machine/src/changes_trie/mod.rs
@@ -85,9 +85,6 @@ use crate::{
 	},
 };
 
-/// Changes that are made outside of extrinsics are marked with this index;
-pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
-
 /// Requirements for block number that can be used with changes tries.
 pub trait BlockNumber:
 	Send + Sync + 'static +
diff --git a/primitives/state-machine/src/error.rs b/primitives/state-machine/src/error.rs
index 5468262f54a2c..489f6e6666001 100644
--- a/primitives/state-machine/src/error.rs
+++ b/primitives/state-machine/src/error.rs
@@ -17,7 +17,7 @@
 
 /// State Machine Errors
 
-use std::fmt;
+use sp_std::fmt;
 
 /// State Machine Error bound.
 ///
@@ -34,7 +34,7 @@ impl<T: 'static + fmt::Debug + fmt::Display + Send> Error for T {}
 #[derive(Debug, Eq, PartialEq)]
 pub enum ExecutionError {
 	/// Backend error.
-	Backend(String),
+	Backend(crate::DefaultError),
 	/// The entry `:code` doesn't exist in storage so there's no way we can execute anything.
 	CodeEntryDoesNotExist,
 	/// Backend is incompatible with execution proof generation process.
diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs
index e36964716f8c9..e9259f9a10bc1 100644
--- a/primitives/state-machine/src/ext.rs
+++ b/primitives/state-machine/src/ext.rs
@@ -18,23 +18,28 @@
 //! Concrete externalities implementation.
 
 use crate::{
-	StorageKey, StorageValue, OverlayedChanges, StorageTransactionCache,
+	StorageKey, StorageValue, OverlayedChanges,
 	backend::Backend,
-	changes_trie::State as ChangesTrieState,
 };
-
 use hash_db::Hasher;
 use sp_core::{
-	offchain::storage::OffchainOverlayedChanges,
 	storage::{well_known_keys::is_child_storage_key, ChildInfo, TrackedStorageKey},
-	traits::Externalities, hexdisplay::HexDisplay,
+	hexdisplay::HexDisplay,
 };
 use sp_trie::{trie_types::Layout, empty_child_trie_root};
-use sp_externalities::{Extensions, Extension};
+use sp_externalities::{Externalities, Extensions, Extension,
+	ExtensionStore};
 use codec::{Decode, Encode, EncodeAppend};
 
-use std::{error, fmt, any::{Any, TypeId}};
-use log::{warn, trace};
+use sp_std::{fmt, any::{Any, TypeId}, vec::Vec, vec, boxed::Box};
+use crate::{warn, trace, log_error};
+#[cfg(feature = "std")]
+use sp_core::offchain::storage::OffchainOverlayedChanges;
+#[cfg(feature = "std")]
+use crate::changes_trie::State as ChangesTrieState;
+use crate::StorageTransactionCache;
+#[cfg(feature = "std")]
+use std::error;
 
 const EXT_NOT_ALLOWED_TO_FAIL: &str = "Externalities not allowed to fail within runtime";
 const BENCHMARKING_FN: &str = "\
@@ -42,7 +47,19 @@ const BENCHMARKING_FN: &str = "\
 	For that reason client started transactions before calling into runtime are not allowed.
 	Without client transactions the loop condition garantuees the success of the tx close.";
 
+
+#[cfg(feature = "std")]
+fn guard() -> sp_panic_handler::AbortGuard {
+	sp_panic_handler::AbortGuard::force_abort()
+}
+
+#[cfg(not(feature = "std"))]
+fn guard() -> () {
+	()
+}
+
 /// Errors that can occur when interacting with the externalities.
+#[cfg(feature = "std")]
 #[derive(Debug, Copy, Clone)]
 pub enum Error<B, E> {
 	/// Failure to load state data from the backend.
@@ -53,6 +70,7 @@ pub enum Error<B, E> {
 	Executor(E),
 }
 
+#[cfg(feature = "std")]
 impl<B: fmt::Display, E: fmt::Display> fmt::Display for Error<B, E> {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		match *self {
@@ -62,6 +80,7 @@ impl<B: fmt::Display, E: fmt::Display> fmt::Display for Error<B, E> {
 	}
 }
 
+#[cfg(feature = "std")]
 impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
 	fn description(&self) -> &str {
 		match *self {
@@ -81,30 +100,49 @@ pub struct Ext<'a, H, N, B>
 	/// The overlayed changes to write to.
 	overlay: &'a mut OverlayedChanges,
 	/// The overlayed changes destined for the Offchain DB.
+	#[cfg(feature = "std")]
 	offchain_overlay: &'a mut OffchainOverlayedChanges,
 	/// The storage backend to read from.
 	backend: &'a B,
 	/// The cache for the storage transactions.
 	storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
 	/// Changes trie state to read from.
+	#[cfg(feature = "std")]
 	changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
 	/// Pseudo-unique id used for tracing.
 	pub id: u16,
 	/// Dummy usage of N arg.
-	_phantom: std::marker::PhantomData<N>,
+	_phantom: sp_std::marker::PhantomData<N>,
 	/// Extensions registered with this instance.
+	#[cfg(feature = "std")]
 	extensions: Option<&'a mut Extensions>,
 }
 
+
 impl<'a, H, N, B> Ext<'a, H, N, B>
-where
-	H: Hasher,
-	H::Out: Ord + 'static + codec::Codec,
-	B: 'a + Backend<H>,
-	N: crate::changes_trie::BlockNumber,
+	where
+		H: Hasher,
+		B: Backend<H>,
+		N: crate::changes_trie::BlockNumber,
 {
+	/// Create a new `Ext`.
+	#[cfg(not(feature = "std"))]
+	pub fn new(
+		overlay: &'a mut OverlayedChanges,
+		storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
+		backend: &'a B,
+	) -> Self {
+		Ext {
+			overlay,
+			backend,
+			id: 0,
+			storage_transaction_cache,
+			_phantom: Default::default(),
+		}
+	}
 
 	/// Create a new `Ext` from overlayed changes and read-only backend
+	#[cfg(feature = "std")]
 	pub fn new(
 		overlay: &'a mut OverlayedChanges,
 		offchain_overlay: &'a mut OffchainOverlayedChanges,
@@ -133,6 +171,7 @@ where
 	}
 
 	/// Read only accessor for the scheduled overlay changes.
+	#[cfg(feature = "std")]
 	pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges {
 		&*self.offchain_overlay
 	}
@@ -159,14 +198,14 @@ where
 	}
 }
 
-impl<'a, H, B, N> Externalities for Ext<'a, H, N, B>
+impl<'a, H, N, B> Externalities for Ext<'a, H, N, B>
 where
 	H: Hasher,
 	H::Out: Ord + 'static + codec::Codec,
-	B: 'a + Backend<H>,
+	B: Backend<H>,
 	N: crate::changes_trie::BlockNumber,
 {
-
+	#[cfg(feature = "std")]
 	fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) {
 		use ::sp_core::offchain::STORAGE_PREFIX;
 		match value {
@@ -175,8 +214,11 @@ where
 		}
 	}
 
+	#[cfg(not(feature = "std"))]
+	fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) {}
+
 	fn storage(&self, key: &[u8]) -> Option<StorageValue> {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(||
 			self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL));
 		trace!(target: "state", "{:04x}: Get {}={:?}",
@@ -188,7 +230,7 @@ where
 	}
 
 	fn storage_hash(&self, key: &[u8]) -> Option<Vec<u8>> {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		let result = self.overlay
 			.storage(key)
 			.map(|x| x.map(|x| H::hash(x)))
@@ -207,7 +249,7 @@ where
 		child_info: &ChildInfo,
 		key: &[u8],
 	) -> Option<StorageValue> {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		let result = self.overlay
 			.child_storage(child_info, key)
 			.map(|x| x.map(|x| x.to_vec()))
@@ -231,7 +273,7 @@ where
 		child_info: &ChildInfo,
 		key: &[u8],
 	) -> Option<Vec<u8>> {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		let result = self.overlay
 			.child_storage(child_info, key)
 			.map(|x| x.map(|x| H::hash(x)))
@@ -251,7 +293,7 @@ where
 	}
 
 	fn exists_storage(&self, key: &[u8]) -> bool {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		let result = match self.overlay.storage(key) {
 			Some(x) => x.is_some(),
 			_ => self.backend.exists_storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL),
@@ -271,7 +313,7 @@ where
 		child_info: &ChildInfo,
 		key: &[u8],
 	) -> bool {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 
 		let result = match self.overlay.child_storage(child_info, key) {
 			Some(x) => x.is_some(),
@@ -337,7 +379,7 @@ where
 			HexDisplay::from(&key),
 			value.as_ref().map(HexDisplay::from)
 		);
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		if is_child_storage_key(&key) {
 			warn!(target: "trie", "Refuse to directly set child storage key");
 			return;
@@ -359,7 +401,7 @@ where
 			HexDisplay::from(&key),
 			value.as_ref().map(HexDisplay::from)
 		);
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 
 		self.mark_dirty();
 		self.overlay.set_child_storage(child_info, key, value);
@@ -373,7 +415,7 @@ where
 			self.id,
 			HexDisplay::from(&child_info.storage_key()),
 		);
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 
 		self.mark_dirty();
 		self.overlay.clear_child_storage(child_info);
@@ -387,7 +429,7 @@ where
 			self.id,
 			HexDisplay::from(&prefix),
 		);
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		if is_child_storage_key(prefix) {
 			warn!(target: "trie", "Refuse to directly clear prefix that is part of child storage key");
 			return;
@@ -410,7 +452,7 @@ where
 			HexDisplay::from(&child_info.storage_key()),
 			HexDisplay::from(&prefix),
 		);
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 
 		self.mark_dirty();
 		self.overlay.clear_child_prefix(child_info, prefix);
@@ -430,7 +472,7 @@ where
 			HexDisplay::from(&value),
 		);
 
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		self.mark_dirty();
 
 		let backend = &mut self.backend;
@@ -446,7 +488,7 @@ where
 	}
 
 	fn storage_root(&mut self) -> Vec<u8> {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root {
 			trace!(target: "state", "{:04x}: Root(cached) {}",
 				self.id,
@@ -464,7 +506,7 @@ where
 		&mut self,
 		child_info: &ChildInfo,
 	) -> Vec<u8> {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		let storage_key = child_info.storage_key();
 		let prefixed_storage_key = child_info.prefixed_storage_key();
 		if self.storage_transaction_cache.transaction_storage_root.is_some() {
@@ -525,8 +567,14 @@ where
 		}
 	}
 
+	#[cfg(not(feature = "std"))]
+	fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
+		Ok(None)
+	}
+
+	#[cfg(feature = "std")]
 	fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
-		let _guard = sp_panic_handler::AbortGuard::force_abort();
+		let _guard = guard();
 		let root = self.overlay.changes_trie_root(
 			self.backend,
 			self.changes_trie_state.as_ref(),
@@ -569,6 +617,7 @@ where
 		}
 		self.overlay.drain_storage_changes(
 			&self.backend,
+			#[cfg(feature = "std")]
 			None,
 			Default::default(),
 			self.storage_transaction_cache,
@@ -586,6 +635,7 @@ where
 		}
 		let changes = self.overlay.drain_storage_changes(
 			&self.backend,
+			#[cfg(feature = "std")]
 			None,
 			Default::default(),
 			self.storage_transaction_cache,
@@ -619,7 +669,6 @@ where
 	}
 }
 
-
 /// Implement `Encode` by forwarding the stored raw vec.
 struct EncodeOpaqueValue(Vec<u8>);
 
@@ -644,12 +693,12 @@ impl<'a> StorageAppend<'a> {
 	pub fn append(&mut self, value: Vec<u8>) {
 		let value = vec![EncodeOpaqueValue(value)];
 
-		let item = std::mem::take(self.0);
+		let item = sp_std::mem::take(self.0);
 
 		*self.0 = match Vec::<EncodeOpaqueValue>::append_or_new(item, &value) {
 			Ok(item) => item,
 			Err(_) => {
-				log::error!(
+				log_error!(
 					target: "runtime",
 					"Failed to append value, resetting storage item to `[value]`.",
 				);
@@ -659,7 +708,36 @@ impl<'a> StorageAppend<'a> {
 	}
 }
 
-impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B>
+#[cfg(not(feature = "std"))]
+impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B>
+where
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
+	B: Backend<H>,
+	N: crate::changes_trie::BlockNumber,
+{
+	fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> {
+		None
+	}
+
+	fn register_extension_with_type_id(
+		&mut self,
+		_type_id: TypeId,
+		_extension: Box<dyn Extension>,
+	) -> Result<(), sp_externalities::Error> {
+		Err(sp_externalities::Error::ExtensionsAreNotSupported)
+	}
+
+	fn deregister_extension_by_type_id(
+		&mut self,
+		_type_id: TypeId,
+	) -> Result<(), sp_externalities::Error> {
+		Err(sp_externalities::Error::ExtensionsAreNotSupported)
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B>
 where
 	H: Hasher,
 	B: 'a + Backend<H>,
diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs
index ee0980f59b926..5b86640aa7d0e 100644
--- a/primitives/state-machine/src/lib.rs
+++ b/primitives/state-machine/src/lib.rs
@@ -18,747 +18,851 @@
 //! Substrate state machine implementation.
 
 #![warn(missing_docs)]
-
-use std::{fmt, result, collections::HashMap, panic::UnwindSafe};
-use log::{warn, trace};
-use hash_db::Hasher;
-use codec::{Decode, Encode, Codec};
-use sp_core::{
-	offchain::storage::OffchainOverlayedChanges,
-	storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay,
-	traits::{CodeExecutor, CallInWasmExt, RuntimeCode, SpawnNamed},
-};
-use sp_externalities::Extensions;
+#![cfg_attr(not(feature = "std"), no_std)]
 
 pub mod backend;
+#[cfg(feature = "std")]
 mod in_memory_backend;
+#[cfg(feature = "std")]
 mod changes_trie;
 mod error;
 mod ext;
+#[cfg(feature = "std")]
 mod testing;
+#[cfg(feature = "std")]
 mod basic;
 mod overlayed_changes;
+#[cfg(feature = "std")]
 mod proving_backend;
 mod trie_backend;
 mod trie_backend_essence;
 mod stats;
+#[cfg(feature = "std")]
 mod read_only;
 
-pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB};
-pub use testing::TestExternalities;
-pub use basic::BasicExternalities;
-pub use read_only::{ReadOnlyExternalities, InspectState};
-pub use ext::Ext;
-pub use backend::Backend;
-pub use changes_trie::{
-	AnchorBlockId as ChangesTrieAnchorBlockId,
-	State as ChangesTrieState,
-	Storage as ChangesTrieStorage,
-	RootsStorage as ChangesTrieRootsStorage,
-	InMemoryStorage as InMemoryChangesTrieStorage,
-	BuildCache as ChangesTrieBuildCache,
-	CacheAction as ChangesTrieCacheAction,
-	ConfigurationRange as ChangesTrieConfigurationRange,
-	key_changes, key_changes_proof,
-	key_changes_proof_check, key_changes_proof_check_with_db,
-	prune as prune_changes_tries,
-	disabled_state as disabled_changes_trie_state,
-	BlockNumber as ChangesTrieBlockNumber,
-};
-pub use overlayed_changes::{
-	OverlayedChanges, StorageChanges, StorageTransactionCache, StorageKey, StorageValue,
-	StorageCollection, ChildStorageCollection,
-};
-pub use proving_backend::{
-	create_proof_check_backend, ProofRecorder, ProvingBackend, ProvingBackendRecorder,
-};
-pub use trie_backend_essence::{TrieBackendStorage, Storage};
-pub use trie_backend::TrieBackend;
-pub use error::{Error, ExecutionError};
-pub use in_memory_backend::new_in_mem;
-pub use stats::{UsageInfo, UsageUnit, StateMachineStats};
-
-const PROOF_CLOSE_TRANSACTION: &str = "\
-	Closing a transaction that was started in this function. Client initiated transactions
-	are protected from being closed by the runtime. qed";
-
-type CallResult<R, E> = Result<NativeOrEncoded<R>, E>;
-
-/// Default handler of the execution manager.
-pub type DefaultHandler<R, E> = fn(CallResult<R, E>, CallResult<R, E>) -> CallResult<R, E>;
-
-/// Type of changes trie transaction.
-pub type ChangesTrieTransaction<H, N> = (
-	MemoryDB<H>,
-	ChangesTrieCacheAction<<H as Hasher>::Out, N>,
-);
-
-/// Trie backend with in-memory storage.
-pub type InMemoryBackend<H> = TrieBackend<MemoryDB<H>, H>;
-
-/// Strategy for executing a call into the runtime.
-#[derive(Copy, Clone, Eq, PartialEq, Debug)]
-pub enum ExecutionStrategy {
-	/// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm.
-	NativeWhenPossible,
-	/// Use the given wasm module.
-	AlwaysWasm,
-	/// Run with both the wasm and the native variant (if compatible). Report any discrepancy as an error.
-	Both,
-	/// First native, then if that fails or is not possible, wasm.
-	NativeElseWasm,
+#[cfg(feature = "std")]
+pub use std_reexport::*;
+
+#[cfg(feature = "std")]
+pub use execution::*;
+#[cfg(feature = "std")]
+pub use log::{debug, warn, trace, error as log_error};
+
+/// In no_std we skip logs for state_machine, this macro
+/// is a noops.
+#[cfg(not(feature = "std"))]
+#[macro_export]
+macro_rules! warn {
+	(target: $target:expr, $($arg:tt)+) => (
+		()
+	);
+	($($arg:tt)+) => (
+		()
+	);
 }
 
-/// Storage backend trust level.
-#[derive(Debug, Clone)]
-pub enum BackendTrustLevel {
-	/// Panics from trusted backends are considered justified, and never caught.
-	Trusted,
-	/// Panics from untrusted backend are caught and interpreted as runtime error.
-	/// Untrusted backend may be missing some parts of the trie, so panics are not considered
-	/// fatal.
-	Untrusted,
+/// In no_std we skip logs for state_machine, this macro
+/// is a noops.
+#[cfg(not(feature = "std"))]
+#[macro_export]
+macro_rules! debug {
+	(target: $target:expr, $($arg:tt)+) => (
+		()
+	);
+	($($arg:tt)+) => (
+		()
+	);
 }
 
-/// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure.
-#[derive(Clone)]
-pub enum ExecutionManager<F> {
-	/// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm.
-	NativeWhenPossible,
-	/// Use the given wasm module. The backend on which code is executed code could be
-	/// trusted to provide all storage or not (i.e. the light client cannot be trusted to provide
-	/// for all storage queries since the storage entries it has come from an external node).
-	AlwaysWasm(BackendTrustLevel),
-	/// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepancy.
-	Both(F),
-	/// First native, then if that fails or is not possible, wasm.
-	NativeElseWasm,
+/// In no_std we skip logs for state_machine, this macro
+/// is a noops.
+#[cfg(not(feature = "std"))]
+#[macro_export]
+macro_rules! trace {
+	(target: $target:expr, $($arg:tt)+) => (
+		()
+	);
+	($($arg:tt)+) => (
+		()
+	);
 }
 
-impl<'a, F> From<&'a ExecutionManager<F>> for ExecutionStrategy {
-	fn from(s: &'a ExecutionManager<F>) -> Self {
-		match *s {
-			ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible,
-			ExecutionManager::AlwaysWasm(_) => ExecutionStrategy::AlwaysWasm,
-			ExecutionManager::NativeElseWasm => ExecutionStrategy::NativeElseWasm,
-			ExecutionManager::Both(_) => ExecutionStrategy::Both,
-		}
-	}
+/// In no_std we skip logs for state_machine, this macro
+/// is a noops.
+#[cfg(not(feature = "std"))]
+#[macro_export]
+macro_rules! log_error {
+	(target: $target:expr, $($arg:tt)+) => (
+		()
+	);
+	($($arg:tt)+) => (
+		()
+	);
 }
 
-impl ExecutionStrategy {
-	/// Gets the corresponding manager for the execution strategy.
-	pub fn get_manager<E: fmt::Debug, R: Decode + Encode>(
-		self,
-	) -> ExecutionManager<DefaultHandler<R, E>> {
-		match self {
-			ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted),
-			ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible,
-			ExecutionStrategy::NativeElseWasm => ExecutionManager::NativeElseWasm,
-			ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| {
-				warn!(
-					"Consensus error between wasm {:?} and native {:?}. Using wasm.",
-					wasm_result,
-					native_result,
-				);
-				warn!("   Native result {:?}", native_result);
-				warn!("   Wasm result {:?}", wasm_result);
-				wasm_result
-			}),
-		}
+/// Default error type to use with state machine trie backend.
+#[cfg(feature = "std")]
+pub type DefaultError = String;
+/// Error type to use with state machine trie backend.
+#[cfg(not(feature = "std"))]
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)]
+pub struct DefaultError;
+
+#[cfg(not(feature = "std"))]
+impl sp_std::fmt::Display for DefaultError {
+	fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result {
+		write!(f, "DefaultError")
 	}
 }
 
-/// Evaluate to ExecutionManager::NativeElseWasm, without having to figure out the type.
-pub fn native_else_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E>> {
-	ExecutionManager::NativeElseWasm
-}
+pub use crate::overlayed_changes::{
+	OverlayedChanges, StorageKey, StorageValue,
+	StorageCollection, ChildStorageCollection,
+	StorageChanges, StorageTransactionCache,
+};
+pub use crate::backend::Backend;
+pub use crate::trie_backend_essence::{TrieBackendStorage, Storage};
+pub use crate::trie_backend::TrieBackend;
+pub use crate::stats::{UsageInfo, UsageUnit, StateMachineStats};
+pub use error::{Error, ExecutionError};
+pub use crate::ext::Ext;
 
-/// Evaluate to ExecutionManager::AlwaysWasm with trusted backend, without having to figure out the type.
-fn always_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E>> {
-	ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted)
-}
+#[cfg(not(feature = "std"))]
+mod changes_trie {
+	/// Stub for change trie block number until
+	/// change trie move to no_std.
+	pub trait BlockNumber {}
 
-/// Evaluate ExecutionManager::AlwaysWasm with untrusted backend, without having to figure out the type.
-fn always_untrusted_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E>> {
-	ExecutionManager::AlwaysWasm(BackendTrustLevel::Untrusted)
+	impl<N> BlockNumber for N {}
 }
 
-/// The substrate state machine.
-pub struct StateMachine<'a, B, H, N, Exec>
-	where
-		H: Hasher,
-		B: Backend<H>,
-		N: ChangesTrieBlockNumber,
-{
-	backend: &'a B,
-	exec: &'a Exec,
-	method: &'a str,
-	call_data: &'a [u8],
-	overlay: &'a mut OverlayedChanges,
-	offchain_overlay: &'a mut OffchainOverlayedChanges,
-	extensions: Extensions,
-	changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
-	storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
-	runtime_code: &'a RuntimeCode<'a>,
-	stats: StateMachineStats,
+#[cfg(feature = "std")]
+mod std_reexport {
+	pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB};
+	pub use crate::testing::TestExternalities;
+	pub use crate::basic::BasicExternalities;
+	pub use crate::read_only::{ReadOnlyExternalities, InspectState};
+	pub use crate::changes_trie::{
+		AnchorBlockId as ChangesTrieAnchorBlockId,
+		State as ChangesTrieState,
+		Storage as ChangesTrieStorage,
+		RootsStorage as ChangesTrieRootsStorage,
+		InMemoryStorage as InMemoryChangesTrieStorage,
+		BuildCache as ChangesTrieBuildCache,
+		CacheAction as ChangesTrieCacheAction,
+		ConfigurationRange as ChangesTrieConfigurationRange,
+		key_changes, key_changes_proof,
+		key_changes_proof_check, key_changes_proof_check_with_db,
+		prune as prune_changes_tries,
+		disabled_state as disabled_changes_trie_state,
+		BlockNumber as ChangesTrieBlockNumber,
+	};
+	pub use crate::proving_backend::{
+		create_proof_check_backend, ProofRecorder, ProvingBackend, ProvingBackendRecorder,
+	};
+	pub use crate::error::{Error, ExecutionError};
+	pub use crate::in_memory_backend::new_in_mem;
 }
 
-impl<'a, B, H, N, Exec> Drop for StateMachine<'a, B, H, N, Exec> where
-	H: Hasher,
-	B: Backend<H>,
-	N: ChangesTrieBlockNumber,
-{
-	fn drop(&mut self) {
-		self.backend.register_overlay_stats(&self.stats);
+#[cfg(feature = "std")]
+mod execution {
+	use super::*;
+	use std::{fmt, result, collections::HashMap, panic::UnwindSafe};
+	use log::{warn, trace};
+	use hash_db::Hasher;
+	use codec::{Decode, Encode, Codec};
+	use sp_core::{
+		offchain::storage::OffchainOverlayedChanges,
+		storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay,
+		traits::{CodeExecutor, CallInWasmExt, RuntimeCode, SpawnNamed},
+	};
+	use sp_externalities::Extensions;
+
+
+	const PROOF_CLOSE_TRANSACTION: &str = "\
+		Closing a transaction that was started in this function. Client initiated transactions
+		are protected from being closed by the runtime. qed";
+
+	pub(crate) type CallResult<R, E> = Result<NativeOrEncoded<R>, E>;
+
+	/// Default handler of the execution manager.
+	pub type DefaultHandler<R, E> = fn(CallResult<R, E>, CallResult<R, E>) -> CallResult<R, E>;
+
+	/// Type of changes trie transaction.
+	pub type ChangesTrieTransaction<H, N> = (
+		MemoryDB<H>,
+		ChangesTrieCacheAction<<H as Hasher>::Out, N>,
+	);
+
+	/// Trie backend with in-memory storage.
+	pub type InMemoryBackend<H> = TrieBackend<MemoryDB<H>, H>;
+
+	/// Strategy for executing a call into the runtime.
+	#[derive(Copy, Clone, Eq, PartialEq, Debug)]
+	pub enum ExecutionStrategy {
+		/// Execute with the native equivalent if it is compatible with the given wasm module;
+		/// otherwise fall back to the wasm.
+		NativeWhenPossible,
+		/// Use the given wasm module.
+		AlwaysWasm,
+		/// Run with both the wasm and the native variant (if compatible). Report any discrepancy as an error.
+		Both,
+		/// First native, then if that fails or is not possible, wasm.
+		NativeElseWasm,
 	}
-}
 
-impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where
-	H: Hasher,
-	H::Out: Ord + 'static + codec::Codec,
-	Exec: CodeExecutor + Clone + 'static,
-	B: Backend<H>,
-	N: crate::changes_trie::BlockNumber,
-{
-	/// Creates new substrate state machine.
-	pub fn new(
-		backend: &'a B,
-		changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
-		overlay: &'a mut OverlayedChanges,
-		offchain_overlay: &'a mut OffchainOverlayedChanges,
-		exec: &'a Exec,
-		method: &'a str,
-		call_data: &'a [u8],
-		mut extensions: Extensions,
-		runtime_code: &'a RuntimeCode,
-		spawn_handle: impl SpawnNamed + Send + 'static,
-	) -> Self {
-		extensions.register(CallInWasmExt::new(exec.clone()));
-		extensions.register(sp_core::traits::TaskExecutorExt::new(spawn_handle));
-
-		Self {
-			backend,
-			exec,
-			method,
-			call_data,
-			extensions,
-			overlay,
-			offchain_overlay,
-			changes_trie_state,
-			storage_transaction_cache: None,
-			runtime_code,
-			stats: StateMachineStats::default(),
-		}
+	/// Storage backend trust level.
+	#[derive(Debug, Clone)]
+	pub enum BackendTrustLevel {
+		/// Panics from trusted backends are considered justified, and never caught.
+		Trusted,
+		/// Panics from untrusted backend are caught and interpreted as runtime error.
+		/// Untrusted backend may be missing some parts of the trie, so panics are not considered
+		/// fatal.
+		Untrusted,
 	}
 
-	/// Use given `cache` as storage transaction cache.
-	///
-	/// The cache will be used to cache storage transactions that can be build while executing a
-	/// function in the runtime. For example, when calculating the storage root a transaction is
-	/// build that will be cached.
-	pub fn with_storage_transaction_cache(
-		mut self,
-		cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
-	) -> Self {
-		self.storage_transaction_cache = cache;
-		self
+	/// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure.
+	#[derive(Clone)]
+	pub enum ExecutionManager<F> {
+		/// Execute with the native equivalent if it is compatible with the given wasm module;
+		/// otherwise fall back to the wasm.
+		NativeWhenPossible,
+		/// Use the given wasm module. The backend on which code is executed code could be
+		/// trusted to provide all storage or not (i.e. the light client cannot be trusted to provide
+		/// for all storage queries since the storage entries it has come from an external node).
+		AlwaysWasm(BackendTrustLevel),
+		/// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepancy.
+		Both(F),
+		/// First native, then if that fails or is not possible, wasm.
+		NativeElseWasm,
 	}
 
-	/// Execute a call using the given state backend, overlayed changes, and call executor.
-	///
-	/// On an error, no prospective changes are written to the overlay.
-	///
-	/// Note: changes to code will be in place if this call is made again. For running partial
-	/// blocks (e.g. a transaction at a time), ensure a different method is used.
-	///
-	/// Returns the SCALE encoded result of the executed function.
-	pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result<Vec<u8>, Box<dyn Error>> {
-		// We are not giving a native call and thus we are sure that the result can never be a native
-		// value.
-		self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
-			strategy.get_manager(),
-			None,
-		).map(NativeOrEncoded::into_encoded)
+	impl<'a, F> From<&'a ExecutionManager<F>> for ExecutionStrategy {
+		fn from(s: &'a ExecutionManager<F>) -> Self {
+			match *s {
+				ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible,
+				ExecutionManager::AlwaysWasm(_) => ExecutionStrategy::AlwaysWasm,
+				ExecutionManager::NativeElseWasm => ExecutionStrategy::NativeElseWasm,
+				ExecutionManager::Both(_) => ExecutionStrategy::Both,
+			}
+		}
 	}
 
-	fn execute_aux<R, NC>(
-		&mut self,
-		use_native: bool,
-		native_call: Option<NC>,
-	) -> (
-		CallResult<R, Exec::Error>,
-		bool,
-	) where
-		R: Decode + Encode + PartialEq,
-		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
-	{
-		let mut cache = StorageTransactionCache::default();
+	impl ExecutionStrategy {
+		/// Gets the corresponding manager for the execution strategy.
+		pub fn get_manager<E: fmt::Debug, R: Decode + Encode>(
+			self,
+		) -> ExecutionManager<DefaultHandler<R, E>> {
+			match self {
+				ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted),
+				ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible,
+				ExecutionStrategy::NativeElseWasm => ExecutionManager::NativeElseWasm,
+				ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| {
+					warn!(
+						"Consensus error between wasm {:?} and native {:?}. Using wasm.",
+						wasm_result,
+						native_result,
+					);
+					warn!("   Native result {:?}", native_result);
+					warn!("   Wasm result {:?}", wasm_result);
+					wasm_result
+				}),
+			}
+		}
+	}
 
-		let cache = match self.storage_transaction_cache.as_mut() {
-			Some(cache) => cache,
-			None => &mut cache,
-		};
+	/// Evaluate to ExecutionManager::NativeElseWasm, without having to figure out the type.
+	pub fn native_else_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E>> {
+		ExecutionManager::NativeElseWasm
+	}
 
-		self.overlay.enter_runtime().expect("StateMachine is never called from the runtime; qed");
+	/// Evaluate to ExecutionManager::AlwaysWasm with trusted backend, without having to figure out the type.
+	fn always_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E>> {
+		ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted)
+	}
 
-		let mut ext = Ext::new(
-			self.overlay,
-			self.offchain_overlay,
-			cache,
-			self.backend,
-			self.changes_trie_state.clone(),
-			Some(&mut self.extensions),
-		);
+	/// Evaluate ExecutionManager::AlwaysWasm with untrusted backend, without having to figure out the type.
+	fn always_untrusted_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E>> {
+		ExecutionManager::AlwaysWasm(BackendTrustLevel::Untrusted)
+	}
 
-		let id = ext.id;
-		trace!(
-			target: "state", "{:04x}: Call {} at {:?}. Input={:?}",
-			id,
-			self.method,
-			self.backend,
-			HexDisplay::from(&self.call_data),
-		);
+	/// The substrate state machine.
+	pub struct StateMachine<'a, B, H, N, Exec>
+		where
+			H: Hasher,
+			B: Backend<H>,
+			N: ChangesTrieBlockNumber,
+	{
+		backend: &'a B,
+		exec: &'a Exec,
+		method: &'a str,
+		call_data: &'a [u8],
+		overlay: &'a mut OverlayedChanges,
+		offchain_overlay: &'a mut OffchainOverlayedChanges,
+		extensions: Extensions,
+		changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
+		storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
+		runtime_code: &'a RuntimeCode<'a>,
+		stats: StateMachineStats,
+	}
 
-		let (result, was_native) = self.exec.call(
-			&mut ext,
-			self.runtime_code,
-			self.method,
-			self.call_data,
-			use_native,
-			native_call,
-		);
+	impl<'a, B, H, N, Exec> Drop for StateMachine<'a, B, H, N, Exec> where
+		H: Hasher,
+		B: Backend<H>,
+		N: ChangesTrieBlockNumber,
+	{
+		fn drop(&mut self) {
+			self.backend.register_overlay_stats(&self.stats);
+		}
+	}
 
-		self.overlay.exit_runtime()
-			.expect("Runtime is not able to call this function in the overlay; qed");
+	impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where
+		H: Hasher,
+		H::Out: Ord + 'static + codec::Codec,
+		Exec: CodeExecutor + Clone + 'static,
+		B: Backend<H>,
+		N: crate::changes_trie::BlockNumber,
+	{
+		/// Creates new substrate state machine.
+		pub fn new(
+			backend: &'a B,
+			changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
+			overlay: &'a mut OverlayedChanges,
+			offchain_overlay: &'a mut OffchainOverlayedChanges,
+			exec: &'a Exec,
+			method: &'a str,
+			call_data: &'a [u8],
+			mut extensions: Extensions,
+			runtime_code: &'a RuntimeCode,
+			spawn_handle: impl SpawnNamed + Send + 'static,
+		) -> Self {
+			extensions.register(CallInWasmExt::new(exec.clone()));
+			extensions.register(sp_core::traits::TaskExecutorExt::new(spawn_handle));
+
+			Self {
+				backend,
+				exec,
+				method,
+				call_data,
+				extensions,
+				overlay,
+				offchain_overlay,
+				changes_trie_state,
+				storage_transaction_cache: None,
+				runtime_code,
+				stats: StateMachineStats::default(),
+			}
+		}
 
-		trace!(
-			target: "state", "{:04x}: Return. Native={:?}, Result={:?}",
-			id,
-			was_native,
-			result,
-		);
+		/// Use given `cache` as storage transaction cache.
+		///
+		/// The cache will be used to cache storage transactions that can be build while executing a
+		/// function in the runtime. For example, when calculating the storage root a transaction is
+		/// build that will be cached.
+		pub fn with_storage_transaction_cache(
+			mut self,
+			cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
+		) -> Self {
+			self.storage_transaction_cache = cache;
+			self
+		}
 
-		(result, was_native)
-	}
+		/// Execute a call using the given state backend, overlayed changes, and call executor.
+		///
+		/// On an error, no prospective changes are written to the overlay.
+		///
+		/// Note: changes to code will be in place if this call is made again. For running partial
+		/// blocks (e.g. a transaction at a time), ensure a different method is used.
+		///
+		/// Returns the SCALE encoded result of the executed function.
+		pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result<Vec<u8>, Box<dyn Error>> {
+			// We are not giving a native call and thus we are sure that the result can never be a native
+			// value.
+			self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
+				strategy.get_manager(),
+				None,
+			).map(NativeOrEncoded::into_encoded)
+		}
 
-	fn execute_call_with_both_strategy<Handler, R, NC>(
-		&mut self,
-		mut native_call: Option<NC>,
-		on_consensus_failure: Handler,
-	) -> CallResult<R, Exec::Error>
-		where
+		fn execute_aux<R, NC>(
+			&mut self,
+			use_native: bool,
+			native_call: Option<NC>,
+		) -> (
+			CallResult<R, Exec::Error>,
+			bool,
+		) where
 			R: Decode + Encode + PartialEq,
 			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
-			Handler: FnOnce(
-				CallResult<R, Exec::Error>,
-				CallResult<R, Exec::Error>,
-			) -> CallResult<R, Exec::Error>
-	{
-		self.overlay.start_transaction();
-		let (result, was_native) = self.execute_aux(true, native_call.take());
+		{
+			let mut cache = StorageTransactionCache::default();
 
-		if was_native {
-			self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION);
-			let (wasm_result, _) = self.execute_aux(
-				false,
+			let cache = match self.storage_transaction_cache.as_mut() {
+				Some(cache) => cache,
+				None => &mut cache,
+			};
+
+			self.overlay.enter_runtime().expect("StateMachine is never called from the runtime; qed");
+
+			let mut ext = Ext::new(
+				self.overlay,
+				self.offchain_overlay,
+				cache,
+				self.backend,
+				self.changes_trie_state.clone(),
+				Some(&mut self.extensions),
+			);
+
+			let id = ext.id;
+			trace!(
+				target: "state", "{:04x}: Call {} at {:?}. Input={:?}",
+				id,
+				self.method,
+				self.backend,
+				HexDisplay::from(&self.call_data),
+			);
+
+			let (result, was_native) = self.exec.call(
+				&mut ext,
+				self.runtime_code,
+				self.method,
+				self.call_data,
+				use_native,
 				native_call,
 			);
 
-			if (result.is_ok() && wasm_result.is_ok()
-				&& result.as_ref().ok() == wasm_result.as_ref().ok())
-				|| result.is_err() && wasm_result.is_err()
-			{
+			self.overlay.exit_runtime()
+				.expect("Runtime is not able to call this function in the overlay; qed");
+
+			trace!(
+				target: "state", "{:04x}: Return. Native={:?}, Result={:?}",
+				id,
+				was_native,
+				result,
+			);
+
+			(result, was_native)
+		}
+
+		fn execute_call_with_both_strategy<Handler, R, NC>(
+			&mut self,
+			mut native_call: Option<NC>,
+			on_consensus_failure: Handler,
+		) -> CallResult<R, Exec::Error>
+			where
+				R: Decode + Encode + PartialEq,
+				NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+				Handler: FnOnce(
+					CallResult<R, Exec::Error>,
+					CallResult<R, Exec::Error>,
+				) -> CallResult<R, Exec::Error>
+		{
+			self.overlay.start_transaction();
+			let (result, was_native) = self.execute_aux(true, native_call.take());
+
+			if was_native {
+				self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION);
+				let (wasm_result, _) = self.execute_aux(
+					false,
+					native_call,
+				);
+
+				if (result.is_ok() && wasm_result.is_ok()
+					&& result.as_ref().ok() == wasm_result.as_ref().ok())
+					|| result.is_err() && wasm_result.is_err()
+				{
+					result
+				} else {
+					on_consensus_failure(wasm_result, result)
+				}
+			} else {
+				self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION);
+				result
+			}
+		}
+
+		fn execute_call_with_native_else_wasm_strategy<R, NC>(
+			&mut self,
+			mut native_call: Option<NC>,
+		) -> CallResult<R, Exec::Error>
+			where
+				R: Decode + Encode + PartialEq,
+				NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+		{
+			self.overlay.start_transaction();
+			let (result, was_native) = self.execute_aux(
+				true,
+				native_call.take(),
+			);
+
+			if !was_native || result.is_ok() {
+				self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION);
 				result
 			} else {
-				on_consensus_failure(wasm_result, result)
+				self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION);
+				let (wasm_result, _) = self.execute_aux(
+					false,
+					native_call,
+				);
+				wasm_result
 			}
-		} else {
-			self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION);
-			result
+		}
+
+		/// Execute a call using the given state backend, overlayed changes, and call executor.
+		///
+		/// On an error, no prospective changes are written to the overlay.
+		///
+		/// Note: changes to code will be in place if this call is made again. For running partial
+		/// blocks (e.g. a transaction at a time), ensure a different method is used.
+		///
+		/// Returns the result of the executed function either in native representation `R` or
+		/// in SCALE encoded representation.
+		pub fn execute_using_consensus_failure_handler<Handler, R, NC>(
+			&mut self,
+			manager: ExecutionManager<Handler>,
+			mut native_call: Option<NC>,
+		) -> Result<NativeOrEncoded<R>, Box<dyn Error>>
+			where
+				R: Decode + Encode + PartialEq,
+				NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+				Handler: FnOnce(
+					CallResult<R, Exec::Error>,
+					CallResult<R, Exec::Error>,
+				) -> CallResult<R, Exec::Error>
+		{
+			let changes_tries_enabled = self.changes_trie_state.is_some();
+			self.overlay.set_collect_extrinsics(changes_tries_enabled);
+
+			let result = {
+				match manager {
+					ExecutionManager::Both(on_consensus_failure) => {
+						self.execute_call_with_both_strategy(
+							native_call.take(),
+							on_consensus_failure,
+						)
+					},
+					ExecutionManager::NativeElseWasm => {
+						self.execute_call_with_native_else_wasm_strategy(
+							native_call.take(),
+						)
+					},
+					ExecutionManager::AlwaysWasm(trust_level) => {
+						let _abort_guard = match trust_level {
+							BackendTrustLevel::Trusted => None,
+							BackendTrustLevel::Untrusted => Some(sp_panic_handler::AbortGuard::never_abort()),
+						};
+						self.execute_aux(false, native_call).0
+					},
+					ExecutionManager::NativeWhenPossible => {
+						self.execute_aux(true, native_call).0
+					},
+				}
+			};
+
+			result.map_err(|e| Box::new(e) as _)
 		}
 	}
 
-	fn execute_call_with_native_else_wasm_strategy<R, NC>(
-		&mut self,
-		mut native_call: Option<NC>,
-	) -> CallResult<R, Exec::Error>
-		where
-			R: Decode + Encode + PartialEq,
-			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	/// Prove execution using the given state backend, overlayed changes, and call executor.
+	pub fn prove_execution<B, H, N, Exec, Spawn>(
+		mut backend: B,
+		overlay: &mut OverlayedChanges,
+		exec: &Exec,
+		spawn_handle: Spawn,
+		method: &str,
+		call_data: &[u8],
+		runtime_code: &RuntimeCode,
+	) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
+	where
+		B: Backend<H>,
+		H: Hasher,
+		H::Out: Ord + 'static + codec::Codec,
+		Exec: CodeExecutor + Clone + 'static,
+		N: crate::changes_trie::BlockNumber,
+		Spawn: SpawnNamed + Send + 'static,
 	{
-		self.overlay.start_transaction();
-		let (result, was_native) = self.execute_aux(
-			true,
-			native_call.take(),
-		);
-
-		if !was_native || result.is_ok() {
-			self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION);
-			result
-		} else {
-			self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION);
-			let (wasm_result, _) = self.execute_aux(
-				false,
-				native_call,
-			);
-			wasm_result
-		}
+		let trie_backend = backend.as_trie_backend()
+			.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
+		prove_execution_on_trie_backend::<_, _, N, _, _>(
+			trie_backend,
+			overlay,
+			exec,
+			spawn_handle,
+			method,
+			call_data,
+			runtime_code,
+		)
 	}
 
-	/// Execute a call using the given state backend, overlayed changes, and call executor.
+	/// Prove execution using the given trie backend, overlayed changes, and call executor.
+	/// Produces a state-backend-specific "transaction" which can be used to apply the changes
+	/// to the backing store, such as the disk.
+	/// Execution proof is the set of all 'touched' storage DBValues from the backend.
 	///
 	/// On an error, no prospective changes are written to the overlay.
 	///
 	/// Note: changes to code will be in place if this call is made again. For running partial
 	/// blocks (e.g. a transaction at a time), ensure a different method is used.
-	///
-	/// Returns the result of the executed function either in native representation `R` or
-	/// in SCALE encoded representation.
-	pub fn execute_using_consensus_failure_handler<Handler, R, NC>(
-		&mut self,
-		manager: ExecutionManager<Handler>,
-		mut native_call: Option<NC>,
-	) -> Result<NativeOrEncoded<R>, Box<dyn Error>>
-		where
-			R: Decode + Encode + PartialEq,
-			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
-			Handler: FnOnce(
-				CallResult<R, Exec::Error>,
-				CallResult<R, Exec::Error>,
-			) -> CallResult<R, Exec::Error>
+	pub fn prove_execution_on_trie_backend<S, H, N, Exec, Spawn>(
+		trie_backend: &TrieBackend<S, H>,
+		overlay: &mut OverlayedChanges,
+		exec: &Exec,
+		spawn_handle: Spawn,
+		method: &str,
+		call_data: &[u8],
+		runtime_code: &RuntimeCode,
+	) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
+	where
+		S: trie_backend_essence::TrieBackendStorage<H>,
+		H: Hasher,
+		H::Out: Ord + 'static + codec::Codec,
+		Exec: CodeExecutor + 'static + Clone,
+		N: crate::changes_trie::BlockNumber,
+		Spawn: SpawnNamed + Send + 'static,
 	{
-		let changes_tries_enabled = self.changes_trie_state.is_some();
-		self.overlay.set_collect_extrinsics(changes_tries_enabled);
-
-		let result = {
-			match manager {
-				ExecutionManager::Both(on_consensus_failure) => {
-					self.execute_call_with_both_strategy(
-						native_call.take(),
-						on_consensus_failure,
-					)
-				},
-				ExecutionManager::NativeElseWasm => {
-					self.execute_call_with_native_else_wasm_strategy(
-						native_call.take(),
-					)
-				},
-				ExecutionManager::AlwaysWasm(trust_level) => {
-					let _abort_guard = match trust_level {
-						BackendTrustLevel::Trusted => None,
-						BackendTrustLevel::Untrusted => Some(sp_panic_handler::AbortGuard::never_abort()),
-					};
-					self.execute_aux(false, native_call).0
-				},
-				ExecutionManager::NativeWhenPossible => {
-					self.execute_aux(true, native_call).0
-				},
-			}
-		};
+		let mut offchain_overlay = OffchainOverlayedChanges::default();
+		let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
+		let mut sm = StateMachine::<_, H, N, Exec>::new(
+			&proving_backend,
+			None,
+			overlay,
+			&mut offchain_overlay,
+			exec,
+			method,
+			call_data,
+			Extensions::default(),
+			runtime_code,
+			spawn_handle,
+		);
 
-		result.map_err(|e| Box::new(e) as _)
+		let result = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
+			always_wasm(),
+			None,
+		)?;
+		let proof = sm.backend.extract_proof();
+		Ok((result.into_encoded(), proof))
 	}
-}
 
-/// Prove execution using the given state backend, overlayed changes, and call executor.
-pub fn prove_execution<B, H, N, Exec, Spawn>(
-	mut backend: B,
-	overlay: &mut OverlayedChanges,
-	exec: &Exec,
-	spawn_handle: Spawn,
-	method: &str,
-	call_data: &[u8],
-	runtime_code: &RuntimeCode,
-) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
-where
-	B: Backend<H>,
-	H: Hasher,
-	H::Out: Ord + 'static + codec::Codec,
-	Exec: CodeExecutor + Clone + 'static,
-	N: crate::changes_trie::BlockNumber,
-	Spawn: SpawnNamed + Send + 'static,
-{
-	let trie_backend = backend.as_trie_backend()
-		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
-	prove_execution_on_trie_backend::<_, _, N, _, _>(
-		trie_backend,
-		overlay,
-		exec,
-		spawn_handle,
-		method,
-		call_data,
-		runtime_code,
-	)
-}
-
-/// Prove execution using the given trie backend, overlayed changes, and call executor.
-/// Produces a state-backend-specific "transaction" which can be used to apply the changes
-/// to the backing store, such as the disk.
-/// Execution proof is the set of all 'touched' storage DBValues from the backend.
-///
-/// On an error, no prospective changes are written to the overlay.
-///
-/// Note: changes to code will be in place if this call is made again. For running partial
-/// blocks (e.g. a transaction at a time), ensure a different method is used.
-pub fn prove_execution_on_trie_backend<S, H, N, Exec, Spawn>(
-	trie_backend: &TrieBackend<S, H>,
-	overlay: &mut OverlayedChanges,
-	exec: &Exec,
-	spawn_handle: Spawn,
-	method: &str,
-	call_data: &[u8],
-	runtime_code: &RuntimeCode,
-) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
-where
-	S: trie_backend_essence::TrieBackendStorage<H>,
-	H: Hasher,
-	H::Out: Ord + 'static + codec::Codec,
-	Exec: CodeExecutor + 'static + Clone,
-	N: crate::changes_trie::BlockNumber,
-	Spawn: SpawnNamed + Send + 'static,
-{
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
-	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
-	let mut sm = StateMachine::<_, H, N, Exec>::new(
-		&proving_backend,
-		None,
-		overlay,
-		&mut offchain_overlay,
-		exec,
-		method,
-		call_data,
-		Extensions::default(),
-		runtime_code,
-		spawn_handle,
-	);
-
-	let result = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
-		always_wasm(),
-		None,
-	)?;
-	let proof = sm.backend.extract_proof();
-	Ok((result.into_encoded(), proof))
-}
-
-/// Check execution proof, generated by `prove_execution` call.
-pub fn execution_proof_check<H, N, Exec, Spawn>(
-	root: H::Out,
-	proof: StorageProof,
-	overlay: &mut OverlayedChanges,
-	exec: &Exec,
-	spawn_handle: Spawn,
-	method: &str,
-	call_data: &[u8],
-	runtime_code: &RuntimeCode,
-) -> Result<Vec<u8>, Box<dyn Error>>
-where
-	H: Hasher,
-	Exec: CodeExecutor + Clone + 'static,
-	H::Out: Ord + 'static + codec::Codec,
-	N: crate::changes_trie::BlockNumber,
-	Spawn: SpawnNamed + Send + 'static,
-{
-	let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
-	execution_proof_check_on_trie_backend::<_, N, _, _>(
-		&trie_backend,
-		overlay,
-		exec,
-		spawn_handle,
-		method,
-		call_data,
-		runtime_code,
-	)
-}
+	/// Check execution proof, generated by `prove_execution` call.
+	pub fn execution_proof_check<H, N, Exec, Spawn>(
+		root: H::Out,
+		proof: StorageProof,
+		overlay: &mut OverlayedChanges,
+		exec: &Exec,
+		spawn_handle: Spawn,
+		method: &str,
+		call_data: &[u8],
+		runtime_code: &RuntimeCode,
+	) -> Result<Vec<u8>, Box<dyn Error>>
+	where
+		H: Hasher,
+		Exec: CodeExecutor + Clone + 'static,
+		H::Out: Ord + 'static + codec::Codec,
+		N: crate::changes_trie::BlockNumber,
+		Spawn: SpawnNamed + Send + 'static,
+	{
+		let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
+		execution_proof_check_on_trie_backend::<_, N, _, _>(
+			&trie_backend,
+			overlay,
+			exec,
+			spawn_handle,
+			method,
+			call_data,
+			runtime_code,
+		)
+	}
 
-/// Check execution proof on proving backend, generated by `prove_execution` call.
-pub fn execution_proof_check_on_trie_backend<H, N, Exec, Spawn>(
-	trie_backend: &TrieBackend<MemoryDB<H>, H>,
-	overlay: &mut OverlayedChanges,
-	exec: &Exec,
-	spawn_handle: Spawn,
-	method: &str,
-	call_data: &[u8],
-	runtime_code: &RuntimeCode,
-) -> Result<Vec<u8>, Box<dyn Error>>
-where
-	H: Hasher,
-	H::Out: Ord + 'static + codec::Codec,
-	Exec: CodeExecutor + Clone + 'static,
-	N: crate::changes_trie::BlockNumber,
-	Spawn: SpawnNamed + Send + 'static,
-{
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
-	let mut sm = StateMachine::<_, H, N, Exec>::new(
-		trie_backend,
-		None,
-		overlay,
-		&mut offchain_overlay,
-		exec,
-		method,
-		call_data,
-		Extensions::default(),
-		runtime_code,
-		spawn_handle,
-	);
+	/// Check execution proof on proving backend, generated by `prove_execution` call.
+	pub fn execution_proof_check_on_trie_backend<H, N, Exec, Spawn>(
+		trie_backend: &TrieBackend<MemoryDB<H>, H>,
+		overlay: &mut OverlayedChanges,
+		exec: &Exec,
+		spawn_handle: Spawn,
+		method: &str,
+		call_data: &[u8],
+		runtime_code: &RuntimeCode,
+	) -> Result<Vec<u8>, Box<dyn Error>>
+	where
+		H: Hasher,
+		H::Out: Ord + 'static + codec::Codec,
+		Exec: CodeExecutor + Clone + 'static,
+		N: crate::changes_trie::BlockNumber,
+		Spawn: SpawnNamed + Send + 'static,
+	{
+		let mut offchain_overlay = OffchainOverlayedChanges::default();
+		let mut sm = StateMachine::<_, H, N, Exec>::new(
+			trie_backend,
+			None,
+			overlay,
+			&mut offchain_overlay,
+			exec,
+			method,
+			call_data,
+			Extensions::default(),
+			runtime_code,
+			spawn_handle,
+		);
 
-	sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
-		always_untrusted_wasm(),
-		None,
-	).map(NativeOrEncoded::into_encoded)
-}
+		sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
+			always_untrusted_wasm(),
+			None,
+		).map(NativeOrEncoded::into_encoded)
+	}
 
-/// Generate storage read proof.
-pub fn prove_read<B, H, I>(
-	mut backend: B,
-	keys: I,
-) -> Result<StorageProof, Box<dyn Error>>
-where
-	B: Backend<H>,
-	H: Hasher,
-	H::Out: Ord + Codec,
-	I: IntoIterator,
-	I::Item: AsRef<[u8]>,
-{
-	let trie_backend = backend.as_trie_backend()
-		.ok_or_else(
-			|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>
-		)?;
-	prove_read_on_trie_backend(trie_backend, keys)
-}
+	/// Generate storage read proof.
+	pub fn prove_read<B, H, I>(
+		mut backend: B,
+		keys: I,
+	) -> Result<StorageProof, Box<dyn Error>>
+	where
+		B: Backend<H>,
+		H: Hasher,
+		H::Out: Ord + Codec,
+		I: IntoIterator,
+		I::Item: AsRef<[u8]>,
+	{
+		let trie_backend = backend.as_trie_backend()
+			.ok_or_else(
+				|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>
+			)?;
+		prove_read_on_trie_backend(trie_backend, keys)
+	}
 
-/// Generate child storage read proof.
-pub fn prove_child_read<B, H, I>(
-	mut backend: B,
-	child_info: &ChildInfo,
-	keys: I,
-) -> Result<StorageProof, Box<dyn Error>>
-where
-	B: Backend<H>,
-	H: Hasher,
-	H::Out: Ord + Codec,
-	I: IntoIterator,
-	I::Item: AsRef<[u8]>,
-{
-	let trie_backend = backend.as_trie_backend()
-		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
-	prove_child_read_on_trie_backend(trie_backend, child_info, keys)
-}
+	/// Generate child storage read proof.
+	pub fn prove_child_read<B, H, I>(
+		mut backend: B,
+		child_info: &ChildInfo,
+		keys: I,
+	) -> Result<StorageProof, Box<dyn Error>>
+	where
+		B: Backend<H>,
+		H: Hasher,
+		H::Out: Ord + Codec,
+		I: IntoIterator,
+		I::Item: AsRef<[u8]>,
+	{
+		let trie_backend = backend.as_trie_backend()
+			.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
+		prove_child_read_on_trie_backend(trie_backend, child_info, keys)
+	}
 
-/// Generate storage read proof on pre-created trie backend.
-pub fn prove_read_on_trie_backend<S, H, I>(
-	trie_backend: &TrieBackend<S, H>,
-	keys: I,
-) -> Result<StorageProof, Box<dyn Error>>
-where
-	S: trie_backend_essence::TrieBackendStorage<H>,
-	H: Hasher,
-	H::Out: Ord + Codec,
-	I: IntoIterator,
-	I::Item: AsRef<[u8]>,
-{
-	let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend);
-	for key in keys.into_iter() {
-		proving_backend
-			.storage(key.as_ref())
-			.map_err(|e| Box::new(e) as Box<dyn Error>)?;
+	/// Generate storage read proof on pre-created trie backend.
+	pub fn prove_read_on_trie_backend<S, H, I>(
+		trie_backend: &TrieBackend<S, H>,
+		keys: I,
+	) -> Result<StorageProof, Box<dyn Error>>
+	where
+		S: trie_backend_essence::TrieBackendStorage<H>,
+		H: Hasher,
+		H::Out: Ord + Codec,
+		I: IntoIterator,
+		I::Item: AsRef<[u8]>,
+	{
+		let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend);
+		for key in keys.into_iter() {
+			proving_backend
+				.storage(key.as_ref())
+				.map_err(|e| Box::new(e) as Box<dyn Error>)?;
+		}
+		Ok(proving_backend.extract_proof())
 	}
-	Ok(proving_backend.extract_proof())
-}
 
-/// Generate storage read proof on pre-created trie backend.
-pub fn prove_child_read_on_trie_backend<S, H, I>(
-	trie_backend: &TrieBackend<S, H>,
-	child_info: &ChildInfo,
-	keys: I,
-) -> Result<StorageProof, Box<dyn Error>>
-where
-	S: trie_backend_essence::TrieBackendStorage<H>,
-	H: Hasher,
-	H::Out: Ord + Codec,
-	I: IntoIterator,
-	I::Item: AsRef<[u8]>,
-{
-	let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend);
-	for key in keys.into_iter() {
-		proving_backend
-			.child_storage(child_info, key.as_ref())
-			.map_err(|e| Box::new(e) as Box<dyn Error>)?;
+	/// Generate storage read proof on pre-created trie backend.
+	pub fn prove_child_read_on_trie_backend<S, H, I>(
+		trie_backend: &TrieBackend<S, H>,
+		child_info: &ChildInfo,
+		keys: I,
+	) -> Result<StorageProof, Box<dyn Error>>
+	where
+		S: trie_backend_essence::TrieBackendStorage<H>,
+		H: Hasher,
+		H::Out: Ord + Codec,
+		I: IntoIterator,
+		I::Item: AsRef<[u8]>,
+	{
+		let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend);
+		for key in keys.into_iter() {
+			proving_backend
+				.child_storage(child_info, key.as_ref())
+				.map_err(|e| Box::new(e) as Box<dyn Error>)?;
+		}
+		Ok(proving_backend.extract_proof())
 	}
-	Ok(proving_backend.extract_proof())
-}
 
-/// Check storage read proof, generated by `prove_read` call.
-pub fn read_proof_check<H, I>(
-	root: H::Out,
-	proof: StorageProof,
-	keys: I,
-) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, Box<dyn Error>>
-where
-	H: Hasher,
-	H::Out: Ord + Codec,
-	I: IntoIterator,
-	I::Item: AsRef<[u8]>,
-{
-	let proving_backend = create_proof_check_backend::<H>(root, proof)?;
-	let mut result = HashMap::new();
-	for key in keys.into_iter() {
-		let value = read_proof_check_on_proving_backend(&proving_backend, key.as_ref())?;
-		result.insert(key.as_ref().to_vec(), value);
+	/// Check storage read proof, generated by `prove_read` call.
+	pub fn read_proof_check<H, I>(
+		root: H::Out,
+		proof: StorageProof,
+		keys: I,
+	) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, Box<dyn Error>>
+	where
+		H: Hasher,
+		H::Out: Ord + Codec,
+		I: IntoIterator,
+		I::Item: AsRef<[u8]>,
+	{
+		let proving_backend = create_proof_check_backend::<H>(root, proof)?;
+		let mut result = HashMap::new();
+		for key in keys.into_iter() {
+			let value = read_proof_check_on_proving_backend(&proving_backend, key.as_ref())?;
+			result.insert(key.as_ref().to_vec(), value);
+		}
+		Ok(result)
 	}
-	Ok(result)
-}
 
-/// Check child storage read proof, generated by `prove_child_read` call.
-pub fn read_child_proof_check<H, I>(
-	root: H::Out,
-	proof: StorageProof,
-	child_info: &ChildInfo,
-	keys: I,
-) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, Box<dyn Error>>
-where
-	H: Hasher,
-	H::Out: Ord + Codec,
-	I: IntoIterator,
-	I::Item: AsRef<[u8]>,
-{
-	let proving_backend = create_proof_check_backend::<H>(root, proof)?;
-	let mut result = HashMap::new();
-	for key in keys.into_iter() {
-		let value = read_child_proof_check_on_proving_backend(
-			&proving_backend,
-			child_info,
-			key.as_ref(),
-		)?;
-		result.insert(key.as_ref().to_vec(), value);
+	/// Check child storage read proof, generated by `prove_child_read` call.
+	pub fn read_child_proof_check<H, I>(
+		root: H::Out,
+		proof: StorageProof,
+		child_info: &ChildInfo,
+		keys: I,
+	) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, Box<dyn Error>>
+	where
+		H: Hasher,
+		H::Out: Ord + Codec,
+		I: IntoIterator,
+		I::Item: AsRef<[u8]>,
+	{
+		let proving_backend = create_proof_check_backend::<H>(root, proof)?;
+		let mut result = HashMap::new();
+		for key in keys.into_iter() {
+			let value = read_child_proof_check_on_proving_backend(
+				&proving_backend,
+				child_info,
+				key.as_ref(),
+			)?;
+			result.insert(key.as_ref().to_vec(), value);
+		}
+		Ok(result)
 	}
-	Ok(result)
-}
 
-/// Check storage read proof on pre-created proving backend.
-pub fn read_proof_check_on_proving_backend<H>(
-	proving_backend: &TrieBackend<MemoryDB<H>, H>,
-	key: &[u8],
-) -> Result<Option<Vec<u8>>, Box<dyn Error>>
-where
-	H: Hasher,
-	H::Out: Ord + Codec,
-{
-	proving_backend.storage(key).map_err(|e| Box::new(e) as Box<dyn Error>)
-}
+	/// Check storage read proof on pre-created proving backend.
+	pub fn read_proof_check_on_proving_backend<H>(
+		proving_backend: &TrieBackend<MemoryDB<H>, H>,
+		key: &[u8],
+	) -> Result<Option<Vec<u8>>, Box<dyn Error>>
+	where
+		H: Hasher,
+		H::Out: Ord + Codec,
+	{
+		proving_backend.storage(key).map_err(|e| Box::new(e) as Box<dyn Error>)
+	}
 
-/// Check child storage read proof on pre-created proving backend.
-pub fn read_child_proof_check_on_proving_backend<H>(
-	proving_backend: &TrieBackend<MemoryDB<H>, H>,
-	child_info: &ChildInfo,
-	key: &[u8],
-) -> Result<Option<Vec<u8>>, Box<dyn Error>>
-where
-	H: Hasher,
-	H::Out: Ord + Codec,
-{
-	proving_backend.child_storage(child_info, key)
-		.map_err(|e| Box::new(e) as Box<dyn Error>)
+	/// Check child storage read proof on pre-created proving backend.
+	pub fn read_child_proof_check_on_proving_backend<H>(
+		proving_backend: &TrieBackend<MemoryDB<H>, H>,
+		child_info: &ChildInfo,
+		key: &[u8],
+	) -> Result<Option<Vec<u8>>, Box<dyn Error>>
+	where
+		H: Hasher,
+		H::Out: Ord + Codec,
+	{
+		proving_backend.child_storage(child_info, key)
+			.map_err(|e| Box::new(e) as Box<dyn Error>)
+	}
 }
 
 #[cfg(test)]
@@ -772,6 +876,15 @@ mod tests {
 		map, traits::{Externalities, RuntimeCode}, testing::TaskExecutor,
 	};
 	use sp_runtime::traits::BlakeTwo256;
+	use std::{result, collections::HashMap};
+	use codec::Decode;
+	use sp_core::{
+		offchain::storage::OffchainOverlayedChanges,
+		storage::ChildInfo, NativeOrEncoded, NeverNativeValue,
+		traits::CodeExecutor,
+	};
+	use crate::execution::CallResult;
+
 
 	#[derive(Clone)]
 	struct DummyCodeExecutor {
diff --git a/primitives/state-machine/src/overlayed_changes/changeset.rs b/primitives/state-machine/src/overlayed_changes/changeset.rs
index fe43c0ea99d89..5e4fd77c68563 100644
--- a/primitives/state-machine/src/overlayed_changes/changeset.rs
+++ b/primitives/state-machine/src/overlayed_changes/changeset.rs
@@ -17,18 +17,22 @@
 
 //! Houses the code that implements the transactional overlay storage.
 
-use super::{StorageKey, StorageValue};
+use super::{StorageKey, StorageValue, Extrinsics};
 
-use itertools::Itertools;
-use std::collections::{HashSet, BTreeMap, BTreeSet};
+#[cfg(feature = "std")]
+use std::collections::HashSet as Set;
+#[cfg(not(feature = "std"))]
+use sp_std::collections::btree_set::BTreeSet as Set;
+
+use sp_std::collections::{btree_map::BTreeMap, btree_set::BTreeSet};
 use smallvec::SmallVec;
-use log::warn;
+use crate::warn;
 
 const PROOF_OVERLAY_NON_EMPTY: &str = "\
 	An OverlayValue is always created with at least one transaction and dropped as soon
 	as the last transaction is removed; qed";
 
-type DirtyKeysSets = SmallVec<[HashSet<StorageKey>; 5]>;
+type DirtyKeysSets = SmallVec<[Set<StorageKey>; 5]>;
 type Transactions = SmallVec<[InnerValue; 5]>;
 
 /// Error returned when trying to commit or rollback while no transaction is open or
@@ -63,7 +67,7 @@ struct InnerValue {
 	value: Option<StorageValue>,
 	/// The set of extrinsic indices where the values has been changed.
 	/// Is filled only if runtime has announced changes trie support.
-	extrinsics: BTreeSet<u32>,
+	extrinsics: Extrinsics,
 }
 
 /// An overlay that contains all versions of a value for a specific key.
@@ -105,8 +109,10 @@ impl OverlayedValue {
 	}
 
 	/// Unique list of extrinsic indices which modified the value.
-	pub fn extrinsics(&self) -> impl Iterator<Item=&u32> {
-		self.transactions.iter().flat_map(|t| t.extrinsics.iter()).unique()
+	pub fn extrinsics(&self) -> BTreeSet<u32> {
+		let mut set = BTreeSet::new();
+		self.transactions.iter().for_each(|t| t.extrinsics.copy_extrinsics_into(&mut set));
+		set
 	}
 
 	/// Mutable reference to the most recent version.
@@ -120,7 +126,7 @@ impl OverlayedValue {
 	}
 
 	/// Mutable reference to the set which holds the indices for the **current transaction only**.
-	fn transaction_extrinsics_mut(&mut self) -> &mut BTreeSet<u32> {
+	fn transaction_extrinsics_mut(&mut self) -> &mut Extrinsics {
 		&mut self.transactions.last_mut().expect(PROOF_OVERLAY_NON_EMPTY).extrinsics
 	}
 
@@ -163,9 +169,9 @@ impl OverlayedChangeSet {
 	/// This changeset might be created when there are already open transactions.
 	/// We need to catch up here so that the child is at the same transaction depth.
 	pub fn spawn_child(&self) -> Self {
-		use std::iter::repeat;
+		use sp_std::iter::repeat;
 		Self {
-			dirty_keys: repeat(HashSet::new()).take(self.transaction_depth()).collect(),
+			dirty_keys: repeat(Set::new()).take(self.transaction_depth()).collect(),
 			num_client_transactions: self.num_client_transactions,
 			execution_mode: self.execution_mode,
 			.. Default::default()
@@ -232,7 +238,7 @@ impl OverlayedChangeSet {
 		at_extrinsic: Option<u32>,
 	) {
 		for (key, val) in self.changes.iter_mut().filter(|(k, v)| predicate(k, v)) {
-			val.set(None, insert_dirty(&mut self.dirty_keys, key.to_owned()), at_extrinsic);
+			val.set(None, insert_dirty(&mut self.dirty_keys, key.clone()), at_extrinsic);
 		}
 	}
 
@@ -243,7 +249,7 @@ impl OverlayedChangeSet {
 
 	/// Get the change that is next to the supplied key.
 	pub fn next_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> {
-		use std::ops::Bound;
+		use sp_std::ops::Bound;
 		let range = (Bound::Excluded(key), Bound::Unbounded);
 		self.changes.range::<[u8], _>(range).next().map(|(k, v)| (&k[..], v))
 	}
@@ -388,7 +394,7 @@ mod test {
 
 	fn assert_changes(is: &OverlayedChangeSet, expected: &Changes) {
 		let is: Changes = is.changes().map(|(k, v)| {
-			(k.as_ref(), (v.value().map(AsRef::as_ref), v.extrinsics().cloned().collect()))
+			(k.as_ref(), (v.value().map(AsRef::as_ref), v.extrinsics().into_iter().collect()))
 		}).collect();
 		assert_eq!(&is, expected);
 	}
diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs
index 9a2b1c4197310..992f7b3519299 100644
--- a/primitives/state-machine/src/overlayed_changes/mod.rs
+++ b/primitives/state-machine/src/overlayed_changes/mod.rs
@@ -20,23 +20,38 @@
 mod changeset;
 
 use crate::{
-	backend::Backend, ChangesTrieTransaction,
-	changes_trie::{
-		NO_EXTRINSIC_INDEX, BlockNumber, build_changes_trie,
-		State as ChangesTrieState,
-	},
+	backend::Backend,
 	stats::StateMachineStats,
 };
+use sp_std::vec::Vec;
 use self::changeset::OverlayedChangeSet;
 
-use std::collections::HashMap;
+#[cfg(feature = "std")]
+use crate::{
+	ChangesTrieTransaction,
+	changes_trie::{
+		build_changes_trie,
+		State as ChangesTrieState,
+	},
+};
+use crate::changes_trie::BlockNumber;
+#[cfg(feature = "std")]
+use std::collections::HashMap as Map;
+#[cfg(not(feature = "std"))]
+use sp_std::collections::btree_map::BTreeMap as Map;
+use sp_std::collections::btree_set::BTreeSet;
 use codec::{Decode, Encode};
 use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo};
+#[cfg(feature = "std")]
 use sp_core::offchain::storage::OffchainOverlayedChanges;
 use hash_db::Hasher;
+use crate::DefaultError;
 
 pub use self::changeset::{OverlayedValue, NoOpenTransaction, AlreadyInRuntime, NotInRuntime};
 
+/// Changes that are made outside of extrinsics are marked with this index;
+pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
+
 /// Storage key.
 pub type StorageKey = Vec<u8>;
 
@@ -49,6 +64,29 @@ pub type StorageCollection = Vec<(StorageKey, Option<StorageValue>)>;
 /// In memory arrays of storage values for multiple child tries.
 pub type ChildStorageCollection = Vec<(StorageKey, StorageCollection)>;
 
+/// Keep trace of extrinsics index for a modified value.
+#[derive(Debug, Default, Eq, PartialEq, Clone)]
+pub struct Extrinsics(Vec<u32>);
+
+impl Extrinsics {
+	/// Extracts extrinsics into a `BTreeSets`.
+	fn copy_extrinsics_into(&self, dest: &mut BTreeSet<u32>) {
+		dest.extend(self.0.iter())
+	}
+
+	/// Add an extrinsics.
+	fn insert(&mut self, ext: u32) {
+		if Some(&ext) != self.0.last() {
+			self.0.push(ext);
+		}
+	}
+
+	/// Extend `self` with `other`.
+	fn extend(&mut self, other: Self) {
+		self.0.extend(other.0.into_iter());
+	}
+}
+
 /// The set of changes that are overlaid onto the backend.
 ///
 /// It allows changes to be modified using nestable transactions.
@@ -57,7 +95,7 @@ pub struct OverlayedChanges {
 	/// Top level storage changes.
 	top: OverlayedChangeSet,
 	/// Child storage changes. The map key is the child storage key without the common prefix.
-	children: HashMap<StorageKey, (OverlayedChangeSet, ChildInfo)>,
+	children: Map<StorageKey, (OverlayedChangeSet, ChildInfo)>,
 	/// True if extrinsics stats must be collected.
 	collect_extrinsics: bool,
 	/// Collect statistic on this execution.
@@ -76,6 +114,7 @@ pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
 	/// All changes to the child storages.
 	pub child_storage_changes: ChildStorageCollection,
 	/// Offchain state changes to write to the offchain database.
+	#[cfg(feature = "std")]
 	pub offchain_storage_changes: OffchainOverlayedChanges,
 	/// A transaction for the backend that contains all changes from
 	/// [`main_storage_changes`](StorageChanges::main_storage_changes) and from
@@ -87,9 +126,14 @@ pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
 	/// Contains the transaction for the backend for the changes trie.
 	///
 	/// If changes trie is disabled the value is set to `None`.
+	#[cfg(feature = "std")]
 	pub changes_trie_transaction: Option<ChangesTrieTransaction<H, N>>,
+	/// Phantom data for block number until change trie support no_std.
+	#[cfg(not(feature = "std"))]
+	pub _ph: sp_std::marker::PhantomData<N>,
 }
 
+#[cfg(feature = "std")]
 impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
 	/// Deconstruct into the inner values
 	pub fn into_inner(self) -> (
@@ -120,9 +164,14 @@ pub struct StorageTransactionCache<Transaction, H: Hasher, N: BlockNumber> {
 	/// The storage root after applying the transaction.
 	pub(crate) transaction_storage_root: Option<H::Out>,
 	/// Contains the changes trie transaction.
+	#[cfg(feature = "std")]
 	pub(crate) changes_trie_transaction: Option<Option<ChangesTrieTransaction<H, N>>>,
 	/// The storage root after applying the changes trie transaction.
+	#[cfg(feature = "std")]
 	pub(crate) changes_trie_transaction_storage_root: Option<Option<H::Out>>,
+	/// Phantom data for block number until change trie support no_std.
+	#[cfg(not(feature = "std"))]
+	pub(crate) _ph: sp_std::marker::PhantomData<N>,
 }
 
 impl<Transaction, H: Hasher, N: BlockNumber> StorageTransactionCache<Transaction, H, N> {
@@ -137,8 +186,12 @@ impl<Transaction, H: Hasher, N: BlockNumber> Default for StorageTransactionCache
 		Self {
 			transaction: None,
 			transaction_storage_root: None,
+			#[cfg(feature = "std")]
 			changes_trie_transaction: None,
+			#[cfg(feature = "std")]
 			changes_trie_transaction_storage_root: None,
+			#[cfg(not(feature = "std"))]
+			_ph: Default::default(),
 		}
 	}
 }
@@ -148,10 +201,14 @@ impl<Transaction: Default, H: Hasher, N: BlockNumber> Default for StorageChanges
 		Self {
 			main_storage_changes: Default::default(),
 			child_storage_changes: Default::default(),
+			#[cfg(feature = "std")]
 			offchain_storage_changes: Default::default(),
 			transaction: Default::default(),
 			transaction_storage_root: Default::default(),
+			#[cfg(feature = "std")]
 			changes_trie_transaction: None,
+			#[cfg(not(feature = "std"))]
+			_ph: Default::default(),
 		}
 	}
 }
@@ -190,7 +247,7 @@ impl OverlayedChanges {
 		key: &[u8],
 		init: impl Fn() -> StorageValue,
 	) -> &mut StorageValue {
-		let value = self.top.modify(key.to_owned(), init, self.extrinsic_index());
+		let value = self.top.modify(key.to_vec(), init, self.extrinsic_index());
 
 		// if the value was deleted initialise it back with an empty vec
 		value.get_or_insert_with(StorageValue::default)
@@ -235,7 +292,7 @@ impl OverlayedChanges {
 		let (changeset, info) = self.children.entry(storage_key).or_insert_with(||
 			(
 				top.spawn_child(),
-				child_info.to_owned()
+				child_info.clone()
 			)
 		);
 		let updatable = info.try_update(child_info);
@@ -256,7 +313,7 @@ impl OverlayedChanges {
 		let (changeset, info) = self.children.entry(storage_key).or_insert_with(||
 			(
 				top.spawn_child(),
-				child_info.to_owned()
+				child_info.clone()
 			)
 		);
 		let updatable = info.try_update(child_info);
@@ -285,7 +342,7 @@ impl OverlayedChanges {
 		let (changeset, info) = self.children.entry(storage_key).or_insert_with(||
 			(
 				top.spawn_child(),
-				child_info.to_owned()
+				child_info.clone()
 			)
 		);
 		let updatable = info.try_update(child_info);
@@ -322,7 +379,7 @@ impl OverlayedChanges {
 	/// there is no open transaction that can be rolled back.
 	pub fn rollback_transaction(&mut self) -> Result<(), NoOpenTransaction> {
 		self.top.rollback_transaction()?;
-		self.children.retain(|_, (changeset, _)| {
+		retain_map(&mut self.children, |_, (changeset, _)| {
 			changeset.rollback_transaction()
 				.expect("Top and children changesets are started in lockstep; qed");
 			!changeset.is_empty()
@@ -379,7 +436,7 @@ impl OverlayedChanges {
 		impl Iterator<Item=(StorageKey, Option<StorageValue>)>,
 		impl Iterator<Item=(StorageKey, (impl Iterator<Item=(StorageKey, Option<StorageValue>)>, ChildInfo))>,
 	) {
-		use std::mem::take;
+		use sp_std::mem::take;
 		(
 			take(&mut self.top).drain_commited(),
 			take(&mut self.children).into_iter()
@@ -409,6 +466,7 @@ impl OverlayedChanges {
 	}
 
 	/// Convert this instance with all changes into a [`StorageChanges`] instance.
+	#[cfg(feature = "std")]
 	pub fn into_storage_changes<
 		B: Backend<H>, H: Hasher, N: BlockNumber
 	>(
@@ -417,7 +475,8 @@ impl OverlayedChanges {
 		changes_trie_state: Option<&ChangesTrieState<H, N>>,
 		parent_hash: H::Out,
 		mut cache: StorageTransactionCache<B::Transaction, H, N>,
-	) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
+	) -> Result<StorageChanges<B::Transaction, H, N>, DefaultError>
+		where H::Out: Ord + Encode + 'static {
 		self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache)
 	}
 
@@ -425,10 +484,12 @@ impl OverlayedChanges {
 	pub fn drain_storage_changes<B: Backend<H>, H: Hasher, N: BlockNumber>(
 		&mut self,
 		backend: &B,
+		#[cfg(feature = "std")]
 		changes_trie_state: Option<&ChangesTrieState<H, N>>,
 		parent_hash: H::Out,
 		mut cache: &mut StorageTransactionCache<B::Transaction, H, N>,
-	) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
+	) -> Result<StorageChanges<B::Transaction, H, N>, DefaultError>
+		where H::Out: Ord + Encode + 'static {
 		// If the transaction does not exist, we generate it.
 		if cache.transaction.is_none() {
 			self.storage_root(backend, &mut cache);
@@ -439,6 +500,7 @@ impl OverlayedChanges {
 			.expect("Transaction was be generated as part of `storage_root`; qed");
 
 		// If the transaction does not exist, we generate it.
+		#[cfg(feature = "std")]
 		if cache.changes_trie_transaction.is_none() {
 			self.changes_trie_root(
 				backend,
@@ -449,20 +511,24 @@ impl OverlayedChanges {
 			).map_err(|_| "Failed to generate changes trie transaction")?;
 		}
 
+		#[cfg(feature = "std")]
 		let changes_trie_transaction = cache.changes_trie_transaction
 			.take()
 			.expect("Changes trie transaction was generated by `changes_trie_root`; qed");
 
-		let offchain_storage_changes = Default::default();
 		let (main_storage_changes, child_storage_changes) = self.drain_committed();
 
 		Ok(StorageChanges {
 			main_storage_changes: main_storage_changes.collect(),
 			child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
-			offchain_storage_changes,
+			#[cfg(feature = "std")]
+			offchain_storage_changes: Default::default(),
 			transaction,
 			transaction_storage_root,
+			#[cfg(feature = "std")]
 			changes_trie_transaction,
+			#[cfg(not(feature = "std"))]
+			_ph: Default::default(),
 		})
 	}
 
@@ -520,6 +586,7 @@ impl OverlayedChanges {
 	/// # Panics
 	///
 	/// Panics on storage error, when `panic_on_storage_error` is set.
+	#[cfg(feature = "std")]
 	pub fn changes_trie_root<'a, H: Hasher, N: BlockNumber, B: Backend<H>>(
 		&self,
 		backend: &B,
@@ -563,6 +630,29 @@ impl OverlayedChanges {
 	}
 }
 
+#[cfg(feature = "std")]
+fn retain_map<K, V, F>(map: &mut Map<K, V>, f: F)
+	where
+		K: std::cmp::Eq + std::hash::Hash,
+		F: FnMut(&K, &mut V) -> bool,
+{
+	map.retain(f);
+}
+ 
+#[cfg(not(feature = "std"))]
+fn retain_map<K, V, F>(map: &mut Map<K, V>, mut f: F)
+	where
+		K: Ord,
+		F: FnMut(&K, &mut V) -> bool,
+{
+	let old = sp_std::mem::replace(map, Map::default());
+	for (k, mut v) in old.into_iter() {
+		if f(&k, &mut v) {
+			map.insert(k, v);
+		}
+	}
+}
+ 
 #[cfg(test)]
 mod tests {
 	use hex_literal::hex;
@@ -578,7 +668,7 @@ mod tests {
 		expected: Vec<u32>,
 	) {
 		assert_eq!(
-			overlay.get(key.as_ref()).unwrap().extrinsics().cloned().collect::<Vec<_>>(),
+			overlay.get(key.as_ref()).unwrap().extrinsics().into_iter().collect::<Vec<_>>(),
 			expected
 		)
 	}
diff --git a/primitives/state-machine/src/stats.rs b/primitives/state-machine/src/stats.rs
index a8ca5a3b416f4..f84de6a5bad07 100644
--- a/primitives/state-machine/src/stats.rs
+++ b/primitives/state-machine/src/stats.rs
@@ -17,8 +17,9 @@
 
 //! Usage statistics for state db
 
+#[cfg(feature = "std")]
 use std::time::{Instant, Duration};
-use std::cell::RefCell;
+use sp_std::cell::RefCell;
 
 /// Measured count of operations and total bytes.
 #[derive(Clone, Debug, Default)]
@@ -50,8 +51,10 @@ pub struct UsageInfo {
 	/// Memory used.
 	pub memory: usize,
 
+	#[cfg(feature = "std")]
 	/// Moment at which current statistics has been started being collected.
 	pub started: Instant,
+	#[cfg(feature = "std")]
 	/// Timespan of the statistics.
 	pub span: Duration,
 }
@@ -99,7 +102,9 @@ impl UsageInfo {
 			cache_reads: UsageUnit::default(),
 			modified_reads: UsageUnit::default(),
 			memory: 0,
+			#[cfg(feature = "std")]
 			started: Instant::now(),
+			#[cfg(feature = "std")]
 			span: Default::default(),
 		}
 	}
diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs
index e0a86bbd193a1..4eaa0870baed0 100644
--- a/primitives/state-machine/src/trie_backend.rs
+++ b/primitives/state-machine/src/trie_backend.rs
@@ -17,7 +17,7 @@
 
 //! Trie-based state machine backend.
 
-use log::{warn, debug};
+use crate::{warn, debug};
 use hash_db::Hasher;
 use sp_trie::{Trie, delta_trie_root, empty_child_trie_root, child_delta_trie_root};
 use sp_trie::trie_types::{TrieDB, TrieError, Layout};
@@ -27,6 +27,7 @@ use crate::{
 	StorageKey, StorageValue, Backend,
 	trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral},
 };
+use sp_std::{boxed::Box, vec::Vec};
 
 /// Patricia trie-based backend. Transaction type is an overlay of changes to commit.
 pub struct TrieBackend<S: TrieBackendStorage<H>, H: Hasher> {
@@ -67,8 +68,8 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackend<S, H> where H::Out: Codec
 	}
 }
 
-impl<S: TrieBackendStorage<H>, H: Hasher> std::fmt::Debug for TrieBackend<S, H> {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl<S: TrieBackendStorage<H>, H: Hasher> sp_std::fmt::Debug for TrieBackend<S, H> {
+	fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result {
 		write!(f, "TrieBackend")
 	}
 }
@@ -76,7 +77,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> std::fmt::Debug for TrieBackend<S, H>
 impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
 	H::Out: Ord + Codec,
 {
-	type Error = String;
+	type Error = crate::DefaultError;
 	type Transaction = S::Overlay;
 	type TrieBackendStorage = S;
 
diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs
index 72864e312b6ab..37bbbb7cf9822 100644
--- a/primitives/state-machine/src/trie_backend_essence.rs
+++ b/primitives/state-machine/src/trie_backend_essence.rs
@@ -18,9 +18,10 @@
 //! Trie-based state machine backend essence used to read values
 //! from storage.
 
-use std::ops::Deref;
+#[cfg(feature = "std")]
 use std::sync::Arc;
-use log::{debug, warn};
+use sp_std::{ops::Deref, boxed::Box, vec::Vec};
+use crate::{warn, debug};
 use hash_db::{self, Hasher, Prefix};
 use sp_trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue,
 	empty_child_trie_root, read_trie_value, read_child_trie_value,
@@ -30,10 +31,19 @@ use crate::{backend::Consolidate, StorageKey, StorageValue};
 use sp_core::storage::ChildInfo;
 use codec::Encode;
 
+#[cfg(not(feature = "std"))]
+macro_rules! format {
+	($($arg:tt)+) => (
+		crate::DefaultError
+	);
+}
+
+type Result<V> = sp_std::result::Result<V, crate::DefaultError>;
+
 /// Patricia trie-based storage trait.
 pub trait Storage<H: Hasher>: Send + Sync {
 	/// Get a trie node.
-	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>;
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>>;
 }
 
 /// Patricia trie-based pairs storage essence.
@@ -80,12 +90,12 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 
 	/// Return the next key in the trie i.e. the minimum key that is strictly superior to `key` in
 	/// lexicographic order.
-	pub fn next_storage_key(&self, key: &[u8]) -> Result<Option<StorageKey>, String> {
+	pub fn next_storage_key(&self, key: &[u8]) -> Result<Option<StorageKey>> {
 		self.next_storage_key_from_root(&self.root, None, key)
 	}
 
 	/// Access the root of the child storage in its parent trie
-	fn child_root(&self, child_info: &ChildInfo) -> Result<Option<StorageValue>, String> {
+	fn child_root(&self, child_info: &ChildInfo) -> Result<Option<StorageValue>> {
 		self.storage(child_info.prefixed_storage_key().as_slice())
 	}
 
@@ -95,7 +105,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 		&self,
 		child_info: &ChildInfo,
 		key: &[u8],
-	) -> Result<Option<StorageKey>, String> {
+	) -> Result<Option<StorageKey>> {
 		let child_root = match self.child_root(child_info)? {
 			Some(child_root) => child_root,
 			None => return Ok(None),
@@ -118,7 +128,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 		root: &H::Out,
 		child_info: Option<&ChildInfo>,
 		key: &[u8],
-	) -> Result<Option<StorageKey>, String> {
+	) -> Result<Option<StorageKey>> {
 		let dyn_eph: &dyn hash_db::HashDBRef<_, _>;
 		let keyspace_eph;
 		if let Some(child_info) = child_info.as_ref() {
@@ -158,7 +168,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 	}
 
 	/// Get the value of storage at given key.
-	pub fn storage(&self, key: &[u8]) -> Result<Option<StorageValue>, String> {
+	pub fn storage(&self, key: &[u8]) -> Result<Option<StorageValue>> {
 		let map_e = |e| format!("Trie lookup error: {}", e);
 
 		read_trie_value::<Layout<H>, _>(self, &self.root, key).map_err(map_e)
@@ -169,7 +179,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 		&self,
 		child_info: &ChildInfo,
 		key: &[u8],
-	) -> Result<Option<StorageValue>, String> {
+	) -> Result<Option<StorageValue>> {
 		let root = self.child_root(child_info)?
 			.unwrap_or_else(|| empty_child_trie_root::<Layout<H>>().encode());
 
@@ -234,7 +244,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 		mut f: F,
 		child_info: Option<&ChildInfo>,
 	) {
-		let mut iter = move |db| -> Result<(), Box<TrieError<H::Out>>> {
+		let mut iter = move |db| -> sp_std::result::Result<(), Box<TrieError<H::Out>>> {
 			let trie = TrieDB::<H>::new(db, root)?;
 
 			for x in TrieDBIterator::new_prefixed(&trie, prefix)? {
@@ -337,14 +347,15 @@ pub trait TrieBackendStorage<H: Hasher>: Send + Sync {
 	/// Type of in-memory overlay.
 	type Overlay: hash_db::HashDB<H, DBValue> + Default + Consolidate;
 	/// Get the value stored at key.
-	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>;
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>>;
 }
 
 // This implementation is used by normal storage trie clients.
+#[cfg(feature = "std")]
 impl<H: Hasher> TrieBackendStorage<H> for Arc<dyn Storage<H>> {
 	type Overlay = PrefixedMemoryDB<H>;
 
-	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>> {
 		Storage::<H>::get(self.deref(), key, prefix)
 	}
 }
@@ -353,7 +364,7 @@ impl<H: Hasher> TrieBackendStorage<H> for Arc<dyn Storage<H>> {
 impl<H: Hasher> TrieBackendStorage<H> for PrefixedMemoryDB<H> {
 	type Overlay = PrefixedMemoryDB<H>;
 
-	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>> {
 		Ok(hash_db::HashDB::get(self, key, prefix))
 	}
 }
@@ -361,7 +372,7 @@ impl<H: Hasher> TrieBackendStorage<H> for PrefixedMemoryDB<H> {
 impl<H: Hasher> TrieBackendStorage<H> for MemoryDB<H> {
 	type Overlay = MemoryDB<H>;
 
-	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>> {
 		Ok(hash_db::HashDB::get(self, key, prefix))
 	}
 }
diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml
index 6b354f5f6e9f4..4f4cdb7d527b5 100644
--- a/test-utils/runtime/Cargo.toml
+++ b/test-utils/runtime/Cargo.toml
@@ -42,6 +42,8 @@ sp-transaction-pool = { version = "2.0.0-rc6", default-features = false, path =
 trie-db = { version = "0.22.0", default-features = false }
 parity-util-mem = { version = "0.7.0", default-features = false, features = ["primitive-types"] }
 sc-service = { version = "0.8.0-rc6", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" }
+sp-state-machine = { version = "0.8.0-rc6", default-features = false, path = "../../primitives/state-machine" }
+sp-externalities = { version = "0.8.0-rc6", default-features = false, path = "../../primitives/externalities" }
 
 # 3rd party
 cfg-if = "0.1.10"
@@ -52,7 +54,6 @@ serde = { version = "1.0.101", optional = true, features = ["derive"] }
 sc-block-builder = { version = "0.8.0-rc6", path = "../../client/block-builder" }
 sc-executor = { version = "0.8.0-rc6", path = "../../client/executor" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "./client" }
-sp-state-machine = { version = "0.8.0-rc6", path = "../../primitives/state-machine" }
 
 [build-dependencies]
 wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../utils/wasm-builder-runner" }
@@ -84,6 +85,8 @@ std = [
 	"sp-session/std",
 	"sp-api/std",
 	"sp-runtime/std",
+	"sp-externalities/std",
+	"sp-state-machine/std",
 	"pallet-babe/std",
 	"frame-system-rpc-runtime-api/std",
 	"frame-system/std",
diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs
index f5e30de838a68..a7c1c261b5541 100644
--- a/test-utils/runtime/src/lib.rs
+++ b/test-utils/runtime/src/lib.rs
@@ -29,7 +29,7 @@ use codec::{Encode, Decode, Input, Error};
 use sp_core::{offchain::KeyTypeId, ChangesTrieConfiguration, OpaqueMetadata, RuntimeDebug};
 use sp_application_crypto::{ed25519, sr25519, ecdsa, RuntimeAppPublic};
 use trie_db::{TrieMut, Trie};
-use sp_trie::PrefixedMemoryDB;
+use sp_trie::{PrefixedMemoryDB, StorageProof};
 use sp_trie::trie_types::{TrieDB, TrieDBMut};
 
 use sp_api::{decl_runtime_apis, impl_runtime_apis};
@@ -335,6 +335,8 @@ cfg_if! {
 				fn test_ecdsa_crypto() -> (ecdsa::AppSignature, ecdsa::AppPublic);
 				/// Run various tests against storage.
 				fn test_storage();
+				/// Check a witness.
+				fn test_witness(proof: StorageProof, root: crate::Hash);
 				/// Test that ensures that we can call a function that takes multiple
 				/// arguments.
 				fn test_multiple_arguments(data: Vec<u8>, other: Vec<u8>, num: u32);
@@ -384,6 +386,8 @@ cfg_if! {
 				fn test_ecdsa_crypto() -> (ecdsa::AppSignature, ecdsa::AppPublic);
 				/// Run various tests against storage.
 				fn test_storage();
+				/// Check a witness.
+				fn test_witness(proof: StorageProof, root: crate::Hash);
 				/// Test that ensures that we can call a function that takes multiple
 				/// arguments.
 				fn test_multiple_arguments(data: Vec<u8>, other: Vec<u8>, num: u32);
@@ -684,6 +688,10 @@ cfg_if! {
 					test_read_child_storage();
 				}
 
+				fn test_witness(proof: StorageProof, root: crate::Hash) {
+					test_witness(proof, root);
+				}
+
 				fn test_multiple_arguments(data: Vec<u8>, other: Vec<u8>, num: u32) {
 					assert_eq!(&data[..], &other[..]);
 					assert_eq!(data.len(), num as usize);
@@ -926,6 +934,10 @@ cfg_if! {
 					test_read_child_storage();
 				}
 
+				fn test_witness(proof: StorageProof, root: crate::Hash) {
+					test_witness(proof, root);
+				}
+
 				fn test_multiple_arguments(data: Vec<u8>, other: Vec<u8>, num: u32) {
 					assert_eq!(&data[..], &other[..]);
 					assert_eq!(data.len(), num as usize);
@@ -1099,6 +1111,34 @@ fn test_read_child_storage() {
 	assert_eq!(&v, &[0, 0, 0, 0]);
 }
 
+fn test_witness(proof: StorageProof, root: crate::Hash) {
+	use sp_externalities::Externalities;
+	let db: sp_trie::MemoryDB<crate::Hashing> = proof.into_memory_db();
+	let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(
+		db,
+		root,
+	);
+	let mut overlay = sp_state_machine::OverlayedChanges::default();
+	#[cfg(feature = "std")]
+	let mut offchain_overlay = Default::default();
+	let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default();
+	let mut ext = sp_state_machine::Ext::new(
+		&mut overlay,
+		#[cfg(feature = "std")]
+		&mut offchain_overlay,
+		&mut cache,
+		&backend,
+		#[cfg(feature = "std")]
+		None,
+		#[cfg(feature = "std")]
+		None,
+	);
+	assert!(ext.storage(b"value3").is_some());
+	assert!(ext.storage_root().as_slice() == &root[..]);
+	ext.place_storage(vec![0], Some(vec![1]));
+	assert!(ext.storage_root().as_slice() != &root[..]);
+}
+
 #[cfg(test)]
 mod tests {
 	use substrate_test_runtime_client::{
@@ -1157,4 +1197,33 @@ mod tests {
 
 		runtime_api.test_storage(&block_id).unwrap();
 	}
+
+	fn witness_backend() -> (sp_trie::MemoryDB<crate::Hashing>, crate::Hash) {
+		use sp_trie::TrieMut;
+		let mut root = crate::Hash::default();
+		let mut mdb = sp_trie::MemoryDB::<crate::Hashing>::default();
+		{
+			let mut trie = sp_trie::trie_types::TrieDBMut::new(&mut mdb, &mut root);
+			trie.insert(b"value3", &[142]).expect("insert failed");
+			trie.insert(b"value4", &[124]).expect("insert failed");
+		};
+		(mdb, root)
+	}
+
+	#[test]
+	fn witness_backend_works() {
+		let (db, root) = witness_backend();
+		let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(
+			db,
+			root,
+		);
+		let proof = sp_state_machine::prove_read(backend, vec![b"value3"]).unwrap();
+		let client = TestClientBuilder::new()
+			.set_execution_strategy(ExecutionStrategy::Both)
+			.build();
+		let runtime_api = client.runtime_api();
+		let block_id = BlockId::Number(client.chain_info().best_number);
+
+		runtime_api.test_witness(&block_id, proof, root).unwrap();
+	}
 }

From cc3040d75f5c09493c21aebabe9f12e287b99754 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Fri, 11 Sep 2020 14:05:43 +0200
Subject: [PATCH 071/122] Support hex encoded secret key for `--node-key`
 (#7052)

* Support hex encoded secret key for `--node-key`

Adds support for reading a hex encoded secret key when being passed as
file via `--node-key`.

* Make the key loading uniform

* Switch to `hex::decode`
---
 client/cli/src/params/node_key_params.rs | 66 +++++++++++++-----------
 client/network/src/config.rs             | 22 ++++++--
 2 files changed, 55 insertions(+), 33 deletions(-)

diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs
index 689cc6c681c83..875411fbfb620 100644
--- a/client/cli/src/params/node_key_params.rs
+++ b/client/cli/src/params/node_key_params.rs
@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use sc_network::config::NodeKeyConfig;
+use sc_network::{config::identity::ed25519, config::NodeKeyConfig};
 use sp_core::H256;
 use std::{path::PathBuf, str::FromStr};
 use structopt::StructOpt;
@@ -83,7 +83,7 @@ pub struct NodeKeyParams {
 	/// as follows:
 	///
 	///   `ed25519`:
-	///   The file must contain an unencoded 32 byte Ed25519 secret key.
+	///   The file must contain an unencoded 32 byte or hex encoded Ed25519 secret key.
 	///
 	/// If the file does not exist, it is created with a newly generated secret key of
 	/// the chosen type.
@@ -100,12 +100,11 @@ impl NodeKeyParams {
 				let secret = if let Some(node_key) = self.node_key.as_ref() {
 					parse_ed25519_secret(node_key)?
 				} else {
-					let path = self
-						.node_key_file
-						.clone()
-						.unwrap_or_else(|| net_config_dir.join(NODE_KEY_ED25519_FILE));
-
-					sc_network::config::Secret::File(path)
+					sc_network::config::Secret::File(
+						self.node_key_file
+							.clone()
+							.unwrap_or_else(|| net_config_dir.join(NODE_KEY_ED25519_FILE))
+					)
 				};
 
 				NodeKeyConfig::Ed25519(secret)
@@ -124,7 +123,7 @@ fn parse_ed25519_secret(hex: &str) -> error::Result<sc_network::config::Ed25519S
 	H256::from_str(&hex)
 		.map_err(invalid_node_key)
 		.and_then(|bytes| {
-			sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes)
+			ed25519::SecretKey::from_bytes(bytes)
 				.map(sc_network::config::Secret::Input)
 				.map_err(invalid_node_key)
 		})
@@ -133,7 +132,8 @@ fn parse_ed25519_secret(hex: &str) -> error::Result<sc_network::config::Ed25519S
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use sc_network::config::identity::ed25519;
+	use sc_network::config::identity::{ed25519, Keypair};
+	use std::fs;
 
 	#[test]
 	fn test_node_key_config_input() {
@@ -164,28 +164,34 @@ mod tests {
 
 	#[test]
 	fn test_node_key_config_file() {
-		fn secret_file(net_config_dir: &PathBuf) -> error::Result<()> {
-			NodeKeyType::variants().iter().try_for_each(|t| {
-				let node_key_type = NodeKeyType::from_str(t).unwrap();
-				let tmp = tempfile::Builder::new().prefix("alice").tempdir()?;
-				let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf();
-				let params = NodeKeyParams {
-					node_key_type,
-					node_key: None,
-					node_key_file: Some(file.clone()),
-				};
-				params.node_key(net_config_dir).and_then(|c| match c {
-					NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f))
-						if node_key_type == NodeKeyType::Ed25519 && f == &file =>
-					{
-						Ok(())
-					}
-					_ => Err(error::Error::Input("Unexpected node key config".into())),
-				})
-			})
+		fn check_key(file: PathBuf, key: &ed25519::SecretKey) {
+			let params = NodeKeyParams {
+				node_key_type: NodeKeyType::Ed25519,
+				node_key: None,
+				node_key_file: Some(file),
+			};
+
+			let node_key = params.node_key(&PathBuf::from("not-used"))
+				.expect("Creates node key config")
+				.into_keypair()
+				.expect("Creates node key pair");
+
+			match node_key {
+				Keypair::Ed25519(ref pair)
+					if pair.secret().as_ref() == key.as_ref() => {}
+				_ => panic!("Invalid key"),
+			}
 		}
 
-		assert!(secret_file(&PathBuf::from_str("x").unwrap()).is_ok());
+		let tmp = tempfile::Builder::new().prefix("alice").tempdir().expect("Creates tempfile");
+		let file = tmp.path().join("mysecret").to_path_buf();
+		let key = ed25519::SecretKey::generate();
+
+		fs::write(&file, hex::encode(key.as_ref())).expect("Writes secret key");
+		check_key(file.clone(), &key);
+
+		fs::write(&file, &key).expect("Writes secret key");
+		check_key(file.clone(), &key);
 	}
 
 	#[test]
diff --git a/client/network/src/config.rs b/client/network/src/config.rs
index cf1f8393f380d..4949af031f085 100644
--- a/client/network/src/config.rs
+++ b/client/network/src/config.rs
@@ -625,10 +625,26 @@ impl NodeKeyConfig {
 				Ok(Keypair::Ed25519(k.into())),
 
 			Ed25519(Secret::File(f)) =>
-				get_secret(f,
-					|mut b| ed25519::SecretKey::from_bytes(&mut b),
+				get_secret(
+					f,
+					|mut b| {
+						match String::from_utf8(b.to_vec())
+							.ok()
+							.and_then(|s|{
+								if s.len() == 64 {
+									hex::decode(&s).ok()
+								} else {
+									None
+								}}
+							)
+						{
+							Some(s) => ed25519::SecretKey::from_bytes(s),
+							_ => ed25519::SecretKey::from_bytes(&mut b),
+						}
+					},
 					ed25519::SecretKey::generate,
-					|b| b.as_ref().to_vec())
+					|b| b.as_ref().to_vec()
+				)
 				.map(ed25519::Keypair::from)
 				.map(Keypair::Ed25519),
 		}

From 6db5fe3103e77075f26e6f79369749f4b7c08494 Mon Sep 17 00:00:00 2001
From: Ashley <ashley.ruglys@gmail.com>
Date: Fri, 11 Sep 2020 14:50:12 +0200
Subject: [PATCH 072/122] Add a `build-sync-spec` subcommand and remove the CHT
 roots from the light sync state. (#6999)

* Move subcommands from sc-cli to nodes

* Add --build-sync-spec subcommand

* Remove CHTs from snapshots

* Keep ProvideChtRoots
---
 bin/node/cli/src/chain_spec.rs                |   7 +-
 bin/node/cli/src/cli.rs                       |   3 +
 bin/node/cli/src/command.rs                   |  13 +-
 bin/node/cli/src/service.rs                   |  52 ++++----
 client/chain-spec/src/chain_spec.rs           |   7 --
 .../cli/src/commands/build_sync_spec_cmd.rs   | 113 ++++++++++++++++++
 client/cli/src/commands/mod.rs                |   2 +
 .../{build_spec.rs => build_sync_spec.rs}     |  33 +----
 client/service/src/chain_ops/mod.rs           |   4 +-
 9 files changed, 171 insertions(+), 63 deletions(-)
 create mode 100644 client/cli/src/commands/build_sync_spec_cmd.rs
 rename client/service/src/chain_ops/{build_spec.rs => build_sync_spec.rs} (52%)

diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs
index e323f7956f169..90824a5572f12 100644
--- a/bin/node/cli/src/chain_spec.rs
+++ b/bin/node/cli/src/chain_spec.rs
@@ -380,7 +380,7 @@ pub fn local_testnet_config() -> ChainSpec {
 #[cfg(test)]
 pub(crate) mod tests {
 	use super::*;
-	use crate::service::{new_full_base, new_light_base};
+	use crate::service::{new_full_base, new_light_base, NewFullBase};
 	use sc_service_test;
 	use sp_runtime::BuildStorage;
 
@@ -431,8 +431,9 @@ pub(crate) mod tests {
 		sc_service_test::connectivity(
 			integration_test_config_with_two_authorities(),
 			|config| {
-				let (keep_alive, _, client, network, transaction_pool) = new_full_base(config,|_, _| ())?;
-				Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool))
+				let NewFullBase { task_manager, client, network, transaction_pool, .. }
+					= new_full_base(config,|_, _| ())?;
+				Ok(sc_service_test::TestNetComponents::new(task_manager, client, network, transaction_pool))
 			},
 			|config| {
 				let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?;
diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs
index 2130ff1e4b106..6e51dae93793f 100644
--- a/bin/node/cli/src/cli.rs
+++ b/bin/node/cli/src/cli.rs
@@ -59,6 +59,9 @@ pub enum Subcommand {
 	/// Build a chain specification.
 	BuildSpec(sc_cli::BuildSpecCmd),
 
+	/// Build a chain specification with a light client sync state.
+	BuildSyncSpec(sc_cli::BuildSyncSpecCmd),
+
 	/// Validate blocks.
 	CheckBlock(sc_cli::CheckBlockCmd),
 
diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs
index a715b2ecaa091..4772d6e4be605 100644
--- a/bin/node/cli/src/command.rs
+++ b/bin/node/cli/src/command.rs
@@ -21,7 +21,7 @@ use node_executor::Executor;
 use node_runtime::{Block, RuntimeApi};
 use sc_cli::{Result, SubstrateCli, RuntimeVersion, Role, ChainSpec};
 use sc_service::PartialComponents;
-use crate::service::new_partial;
+use crate::service::{new_partial, new_full_base, NewFullBase};
 
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
@@ -101,6 +101,17 @@ pub fn run() -> Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.sync_run(|config| cmd.run(config.chain_spec, config.network))
 		},
+		Some(Subcommand::BuildSyncSpec(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let chain_spec = config.chain_spec.cloned_box();
+				let network_config = config.network.clone();
+				let NewFullBase { task_manager, client, network_status_sinks, .. }
+					= new_full_base(config, |_, _| ())?;
+
+				Ok((cmd.run(chain_spec, network_config, client, network_status_sinks), task_manager))
+			})
+		},
 		Some(Subcommand::CheckBlock(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {
diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs
index d91696ab7d6bc..51232df9b82b1 100644
--- a/bin/node/cli/src/service.rs
+++ b/bin/node/cli/src/service.rs
@@ -151,6 +151,15 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 	})
 }
 
+pub struct NewFullBase {
+	pub task_manager: TaskManager,
+	pub inherent_data_providers: InherentDataProviders,
+	pub client: Arc<FullClient>,
+	pub network: Arc<NetworkService<Block, <Block as BlockT>::Hash>>,
+	pub network_status_sinks: sc_service::NetworkStatusSinks<Block>,
+	pub transaction_pool: Arc<sc_transaction_pool::FullPool<Block, FullClient>>,
+}
+
 /// Creates a full service from the configuration.
 pub fn new_full_base(
 	config: Configuration,
@@ -158,11 +167,7 @@ pub fn new_full_base(
 		&sc_consensus_babe::BabeBlockImport<Block, FullClient, FullGrandpaBlockImport>,
 		&sc_consensus_babe::BabeLink<Block>,
 	)
-) -> Result<(
-	TaskManager, InherentDataProviders, Arc<FullClient>,
-	Arc<NetworkService<Block, <Block as BlockT>::Hash>>,
-	Arc<sc_transaction_pool::FullPool<Block, FullClient>>,
-), ServiceError> {
+) -> Result<NewFullBase, ServiceError> {
 	let sc_service::PartialComponents {
 		client, backend, mut task_manager, import_queue, keystore, select_chain, transaction_pool,
 		inherent_data_providers,
@@ -210,7 +215,7 @@ pub fn new_full_base(
 		on_demand: None,
 		remote_blockchain: None,
 		telemetry_connection_sinks: telemetry_connection_sinks.clone(),
-		network_status_sinks,
+		network_status_sinks: network_status_sinks.clone(),
 		system_rpc_tx,
 	})?;
 
@@ -330,13 +335,16 @@ pub fn new_full_base(
 	}
 
 	network_starter.start_network();
-	Ok((task_manager, inherent_data_providers, client, network, transaction_pool))
+	Ok(NewFullBase {
+		task_manager, inherent_data_providers, client, network, network_status_sinks,
+		transaction_pool,
+	})
 }
 
 /// Builds a new service for a full client.
 pub fn new_full(config: Configuration)
 -> Result<TaskManager, ServiceError> {
-	new_full_base(config, |_, _| ()).map(|(task_manager, _, _, _, _)| {
+	new_full_base(config, |_, _| ()).map(|NewFullBase { task_manager, .. }| {
 		task_manager
 	})
 }
@@ -467,7 +475,7 @@ mod tests {
 	use sp_finality_tracker;
 	use sp_keyring::AccountKeyring;
 	use sc_service_test::TestNetNode;
-	use crate::service::{new_full_base, new_light_base};
+	use crate::service::{new_full_base, new_light_base, NewFullBase};
 	use sp_runtime::traits::IdentifyAccount;
 	use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent};
 	use sc_client_api::BlockBackend;
@@ -499,18 +507,19 @@ mod tests {
 			chain_spec,
 			|config| {
 				let mut setup_handles = None;
-				let (keep_alive, inherent_data_providers, client, network, transaction_pool) =
-					new_full_base(config,
-						|
-							block_import: &sc_consensus_babe::BabeBlockImport<Block, _, _>,
-							babe_link: &sc_consensus_babe::BabeLink<Block>,
-						| {
-							setup_handles = Some((block_import.clone(), babe_link.clone()));
-						}
-					)?;
+				let NewFullBase {
+					task_manager, inherent_data_providers, client, network, transaction_pool, ..
+				} = new_full_base(config,
+					|
+						block_import: &sc_consensus_babe::BabeBlockImport<Block, _, _>,
+						babe_link: &sc_consensus_babe::BabeLink<Block>,
+					| {
+						setup_handles = Some((block_import.clone(), babe_link.clone()));
+					}
+				)?;
 
 				let node = sc_service_test::TestNetComponents::new(
-					keep_alive, client, network, transaction_pool
+					task_manager, client, network, transaction_pool
 				);
 				Ok((node, (inherent_data_providers, setup_handles.unwrap())))
 			},
@@ -661,8 +670,9 @@ mod tests {
 		sc_service_test::consensus(
 			crate::chain_spec::tests::integration_test_config_with_two_authorities(),
 			|config| {
-				let (keep_alive, _, client, network, transaction_pool) = new_full_base(config, |_, _| ())?;
-				Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool))
+				let NewFullBase { task_manager, client, network, transaction_pool, .. }
+					= new_full_base(config,|_, _| ())?;
+				Ok(sc_service_test::TestNetComponents::new(task_manager, client, network, transaction_pool))
 			},
 			|config| {
 				let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?;
diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs
index 20811394c56d7..1fbf0419e2001 100644
--- a/client/chain-spec/src/chain_spec.rs
+++ b/client/chain-spec/src/chain_spec.rs
@@ -401,8 +401,6 @@ where
 pub struct LightSyncState<Block: BlockT> {
 	/// The header of the best finalized block.
 	pub header: <Block as BlockT>::Header,
-	/// A list of all CHTs in the chain.
-	pub chts: Vec<<Block as BlockT>::Hash>,
 }
 
 impl<Block: BlockT> LightSyncState<Block> {
@@ -412,7 +410,6 @@ impl<Block: BlockT> LightSyncState<Block> {
 
 		SerializableLightSyncState {
 			header: StorageData(self.header.encode()),
-			chts: self.chts.iter().map(|hash| StorageData(hash.encode())).collect(),
 		}
 	}
 
@@ -420,9 +417,6 @@ impl<Block: BlockT> LightSyncState<Block> {
 	pub fn from_serializable(serialized: &SerializableLightSyncState) -> Result<Self, codec::Error> {
 		Ok(Self {
 			header: codec::Decode::decode(&mut &serialized.header.0[..])?,
-			chts: serialized.chts.iter()
-				.map(|cht| codec::Decode::decode(&mut &cht.0[..]))
-				.collect::<Result<_, _>>()?,
 		})
 	}
 }
@@ -433,7 +427,6 @@ impl<Block: BlockT> LightSyncState<Block> {
 #[serde(deny_unknown_fields)]
 pub struct SerializableLightSyncState {
 	header: StorageData,
-	chts: Vec<StorageData>,
 }
 
 #[cfg(test)]
diff --git a/client/cli/src/commands/build_sync_spec_cmd.rs b/client/cli/src/commands/build_sync_spec_cmd.rs
new file mode 100644
index 0000000000000..4d87e2b063ac6
--- /dev/null
+++ b/client/cli/src/commands/build_sync_spec_cmd.rs
@@ -0,0 +1,113 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2018-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::error;
+use crate::params::{SharedParams, NetworkParams};
+use crate::CliConfiguration;
+use log::info;
+use sc_network::config::build_multiaddr;
+use sc_service::{config::{MultiaddrWithPeerId, NetworkConfiguration}, ChainSpec};
+use structopt::StructOpt;
+use std::io::Write;
+use std::sync::Arc;
+use sp_runtime::traits::Block as BlockT;
+use sc_service::chain_ops::build_light_sync_state;
+use sc_service::NetworkStatusSinks;
+use futures::{FutureExt, StreamExt};
+use futures::future::ready;
+
+/// The `build-sync-spec` command used to build a chain spec that contains a light client state
+/// so that light clients can sync faster.
+#[derive(Debug, StructOpt)]
+pub struct BuildSyncSpecCmd {
+	/// Force raw genesis storage output.
+	#[structopt(long = "raw")]
+	pub raw: bool,
+
+	/// Sync the chain using a full client first.
+	#[structopt(long)]
+	pub sync_first: bool,
+
+	/// Disable adding the default bootnode to the specification.
+	///
+	/// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the
+	/// specification when no bootnode exists.
+	#[structopt(long = "disable-default-bootnode")]
+	pub disable_default_bootnode: bool,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: SharedParams,
+
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub network_params: NetworkParams,
+}
+
+impl BuildSyncSpecCmd {
+	/// Run the build-sync-spec command
+	pub async fn run<B, CL>(
+		&self,
+		mut spec: Box<dyn ChainSpec>,
+		network_config: NetworkConfiguration,
+		client: Arc<CL>,
+		network_status_sinks: NetworkStatusSinks<B>,
+	) -> error::Result<()>
+		where
+			B: BlockT,
+			CL: sp_blockchain::HeaderBackend<B>,
+	{
+        if self.sync_first {
+            network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| {
+                ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0)
+            }).into_future().map(drop).await;
+        }
+
+        let light_sync_state = build_light_sync_state(client)?;
+        spec.set_light_sync_state(light_sync_state.to_serializable());
+
+		info!("Building chain spec");
+		let raw_output = self.raw;
+
+		if spec.boot_nodes().is_empty() && !self.disable_default_bootnode {
+			let keys = network_config.node_key.into_keypair()?;
+			let peer_id = keys.public().into_peer_id();
+			let addr = MultiaddrWithPeerId {
+				multiaddr: build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(30333u16)],
+				peer_id,
+			};
+			spec.add_boot_node(addr)
+		}
+
+		let json = sc_service::chain_ops::build_spec(&*spec, raw_output)?;
+		if std::io::stdout().write_all(json.as_bytes()).is_err() {
+			let _ = std::io::stderr().write_all(b"Error writing to stdout\n");
+		}
+		Ok(())
+	}
+}
+
+impl CliConfiguration for BuildSyncSpecCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+
+	fn network_params(&self) -> Option<&NetworkParams> {
+		Some(&self.network_params)
+	}
+}
diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs
index 7b740d1003238..899abf0c3d437 100644
--- a/client/cli/src/commands/mod.rs
+++ b/client/cli/src/commands/mod.rs
@@ -16,6 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 mod build_spec_cmd;
+mod build_sync_spec_cmd;
 mod check_block_cmd;
 mod export_blocks_cmd;
 mod export_state_cmd;
@@ -36,6 +37,7 @@ pub mod utils;
 
 pub use self::{
 	build_spec_cmd::BuildSpecCmd,
+	build_sync_spec_cmd::BuildSyncSpecCmd,
 	check_block_cmd::CheckBlockCmd,
 	export_blocks_cmd::ExportBlocksCmd,
 	export_state_cmd::ExportStateCmd,
diff --git a/client/service/src/chain_ops/build_spec.rs b/client/service/src/chain_ops/build_sync_spec.rs
similarity index 52%
rename from client/service/src/chain_ops/build_spec.rs
rename to client/service/src/chain_ops/build_sync_spec.rs
index 40d591d81f02b..9553ea21a6965 100644
--- a/client/service/src/chain_ops/build_spec.rs
+++ b/client/service/src/chain_ops/build_sync_spec.rs
@@ -14,48 +14,23 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One};
+use sp_runtime::traits::Block as BlockT;
 use sp_blockchain::HeaderBackend;
 use std::sync::Arc;
 use sp_runtime::generic::BlockId;
-use sc_client_api::ProvideChtRoots;
 
 /// Build a `LightSyncState` from the CHT roots stored in a backend.
-pub fn build_light_sync_state<TBl, TCl, TBackend>(
+pub fn build_light_sync_state<TBl, TCl>(
 	client: Arc<TCl>,
-	backend: Arc<TBackend>,
 ) -> Result<sc_chain_spec::LightSyncState<TBl>, sp_blockchain::Error>
 	where
 		TBl: BlockT,
 		TCl: HeaderBackend<TBl>,
-		TBackend: sc_client_api::Backend<TBl>,
-		<TBackend as sc_client_api::Backend<TBl>>::Blockchain: ProvideChtRoots<TBl>,
 {
-	let cht_root_provider = backend.blockchain();
-
 	let finalized_hash = client.info().finalized_hash;
-	let finalized_number = client.info().finalized_number;
-
-	use sc_client_api::cht;
-
-	let mut chts = Vec::new();
-
-	// We can't fetch a CHT root later than `finalized_number - 2 * cht_size`.
-	let cht_size_x_2 = cht::size::<NumberFor::<TBl>>() * NumberFor::<TBl>::from(2);
-
-	let mut number = NumberFor::<TBl>::one();
-
-	while number <= finalized_number.saturating_sub(cht_size_x_2) {
-		match cht_root_provider.header_cht_root(cht::size(), number)? {
-			Some(cht_root) => chts.push(cht_root),
-			None => log::error!("No CHT found for block {}", number),
-		}
-
-		number += cht::size();
-	}
+	let header = client.header(BlockId::Hash(finalized_hash))?.unwrap();
 
 	Ok(sc_chain_spec::LightSyncState {
-		header: client.header(BlockId::Hash(finalized_hash))?.unwrap(),
-		chts,
+		header
 	})
 }
diff --git a/client/service/src/chain_ops/mod.rs b/client/service/src/chain_ops/mod.rs
index 19f5e346820aa..e6b2fdfb8e0e6 100644
--- a/client/service/src/chain_ops/mod.rs
+++ b/client/service/src/chain_ops/mod.rs
@@ -21,11 +21,11 @@ mod export_blocks;
 mod export_raw_state;
 mod import_blocks;
 mod revert_chain;
-mod build_spec;
+mod build_sync_spec;
 
 pub use check_block::*;
 pub use export_blocks::*;
 pub use export_raw_state::*;
 pub use import_blocks::*;
 pub use revert_chain::*;
-pub use build_spec::*;
+pub use build_sync_spec::*;

From bb7052d0426de7a12057c5e0b741d77bc6b5168e Mon Sep 17 00:00:00 2001
From: Ashley <ashley.ruglys@gmail.com>
Date: Fri, 11 Sep 2020 15:54:03 +0200
Subject: [PATCH 073/122] Fix build sync spec (#7086)

---
 client/cli/src/commands/build_sync_spec_cmd.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/cli/src/commands/build_sync_spec_cmd.rs b/client/cli/src/commands/build_sync_spec_cmd.rs
index 4d87e2b063ac6..3f1bfce6a3290 100644
--- a/client/cli/src/commands/build_sync_spec_cmd.rs
+++ b/client/cli/src/commands/build_sync_spec_cmd.rs
@@ -73,7 +73,7 @@ impl BuildSyncSpecCmd {
 			CL: sp_blockchain::HeaderBackend<B>,
 	{
         if self.sync_first {
-            network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| {
+            network_status_sinks.status_stream(std::time::Duration::from_secs(1)).filter(|status| {
                 ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0)
             }).into_future().map(drop).await;
         }

From a9a3be3796ef321a5784cd267e61db5ab04cbf45 Mon Sep 17 00:00:00 2001
From: Denis Pisarev <denis.pisarev@parity.io>
Date: Fri, 11 Sep 2020 17:39:16 +0200
Subject: [PATCH 074/122] Fail docs on warnings (#5923)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* change (ci): docs job optimized; runs every commit; fails on warnings

* change (ci): rename jobs; temporary allow failing

* change (ci): better warnings filtering

* fix (ci): hotfix Docker release

* test (ci): run docs job with flags

* test (ci): pwd fails

* change (ci): pass just //doc dir as an artifact; debug

* change (ci): return to the previous structure; undebug

* change (ci): typo

* rebase on upstream 2

* fix the jobname

* Fix some warnings (#7079)

* Partial fix for transaction priority (#7034)

* Partial fix for priority stuff.

* Small fix

* Fix tests.

* Update frame/transaction-payment/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Better doc

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* What happens if we remove wat? (#7056)

* What happens if we remove wat?

* Update Cargo.lock

* Make SlashingSpans Public (#6961)

* Make SlashingSpans Public

Offchain Applications will often need to inspect this type because it is directly used in staking election, thus worthy of being `pub`. Rest of the slashing api can remain private, only this and the `fn last_non_zero_slash()` of `SlashingSpans` are of interest.

* Update frame/staking/src/lib.rs

* client/authority-discovery/src/service: Improve docs (#7059)

* Decrease poll interval (#7063)

* Remove unused code (#7027)

Signed-off-by: Jimmy Chu <jimmychu0807@gmail.com>

* Disambiguate `BlockNumber` type in `decl_module` (#7061)

* Disambiguate `BlockNumber` type in `decl_module`

* fix `frame-support-tests`

* fix ui tests

* fix trait order

* Implement `FromStr` for `Ss58AddressFormat` (#7068)

* Implement `FromStr` for `Ss58AddressFormat`

* Update primitives/core/src/crypto.rs

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

* Set reserved nodes with offchain worker. (#6996)

* add offchain worker api to set reserved nodes.

* new offchain api to get node public key.

* node public key from converter

* refactor set reserved nodes ocw api.

* new ndoe authorization pallet

* remove unnecessary clone and more.

* more

* tests for node authorization pallet

* remove dependency

* fix build

* more tests.

* refactor

* Update primitives/core/src/offchain/testing.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update frame/node-authorization/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update frame/node-authorization/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update frame/node-authorization/src/lib.rs

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* format code

* expose NetworkService

* remove NetworkStateInfo in offchain

* replace NodePublicKey with PeerId.

* set max length of peer id.

* clear more

* use BTreeSet for set of peers.

* decode opaque peer id.

* extract NetworkProvider for client offchain.

* use OpaquePeerId in node authorization pallet.

* fix test

* better documentation

* fix test

* doc

* more fix

* Update primitives/core/src/offchain/mod.rs

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Update client/offchain/src/api.rs

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* derive serialize and deserialize

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Fix some warnings

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Sergei Shulepov <sergei@parity.io>
Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: s3krit <pugh@s3kr.it>
Co-authored-by: Jimmy Chu <jimmy@parity.io>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: kaichao <kaichaosuna@gmail.com>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Fix more doc errors

* More doc fixes

* Remove subdb to make `rustdoc` happy

* Make the line length check happy

* Fix compilation error

* Another try

* Allow unused

Co-authored-by: Dan Forbes <dan@danforbes.dev>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Sergei Shulepov <sergei@parity.io>
Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: s3krit <pugh@s3kr.it>
Co-authored-by: Jimmy Chu <jimmy@parity.io>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: kaichao <kaichaosuna@gmail.com>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
Co-authored-by: Bastian Köcher <git@kchr.de>
---
 .gitlab-ci.yml                           | 18 ++---
 client/cli/src/arg_enums.rs              |  2 -
 client/cli/src/config.rs                 |  3 -
 client/db/Cargo.toml                     |  1 -
 client/db/src/lib.rs                     | 10 ---
 client/db/src/subdb.rs                   | 88 ------------------------
 client/db/src/utils.rs                   | 20 ++----
 client/network/src/lib.rs                |  5 +-
 client/network/src/request_responses.rs  |  6 +-
 client/network/src/service.rs            | 14 ++--
 frame/assets/src/lib.rs                  |  6 +-
 frame/atomic-swap/src/lib.rs             |  6 +-
 frame/balances/src/lib.rs                | 16 ++---
 frame/collective/src/lib.rs              | 14 ++--
 frame/contracts/src/lib.rs               | 12 ++--
 frame/democracy/src/lib.rs               | 34 ++++-----
 frame/elections-phragmen/src/lib.rs      | 12 ++--
 frame/elections/src/lib.rs               |  8 +--
 frame/evm/src/lib.rs                     | 12 ++--
 frame/example-offchain-worker/src/lib.rs |  2 +-
 frame/generic-asset/src/lib.rs           | 10 +--
 frame/grandpa/src/lib.rs                 |  2 +-
 frame/identity/src/lib.rs                | 20 +++---
 frame/im-online/src/lib.rs               |  4 +-
 frame/indices/src/lib.rs                 |  6 +-
 frame/multisig/src/lib.rs                |  9 +--
 frame/nicks/src/lib.rs                   | 10 +--
 frame/offences/src/lib.rs                |  2 +-
 frame/proxy/src/lib.rs                   |  6 +-
 frame/recovery/src/lib.rs                | 12 ++--
 frame/scheduler/src/lib.rs               |  6 +-
 frame/session/src/lib.rs                 |  2 +-
 frame/society/src/lib.rs                 | 32 ++++-----
 frame/staking/src/lib.rs                 | 18 ++---
 frame/sudo/src/lib.rs                    |  6 +-
 frame/support/src/dispatch.rs            |  2 +-
 frame/system/src/lib.rs                  |  8 +--
 frame/treasury/src/lib.rs                | 23 ++++---
 frame/utility/src/lib.rs                 |  2 +-
 frame/vesting/src/lib.rs                 |  4 +-
 40 files changed, 185 insertions(+), 288 deletions(-)
 delete mode 100644 client/db/src/subdb.rs

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d20a65b4df54d..56ac4c7f9487d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,7 +13,7 @@
 #   image:                           paritytech/tools:latest # Any docker image (required)
 #   allow_failure:                   true # Allow the pipeline to continue if this job fails (default: false)
 #   dependencies:
-#     - build-rust-doc-release # Any jobs that are required to run before this job (optional)
+#     - build-rust-doc # Any jobs that are required to run before this job (optional)
 #   variables:
 #     MY_ENVIRONMENT_VARIABLE:       "some useful value" # Environment variables passed to the job (optional)
 #   script:
@@ -476,23 +476,25 @@ build-macos-subkey:
   tags:
     - osx
 
-build-rust-doc-release:
+build-rust-doc:
   stage:                           build
   <<:                              *docker-env
   <<:                              *docker-env-only
   allow_failure:                   true
+  variables:
+    <<:                            *default-vars
+    RUSTFLAGS:                     -Dwarnings
   artifacts:
     name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
     when:                          on_success
     expire_in:                     7 days
     paths:
-    - ./crate-docs
-  <<:                              *build-only
+    - ./crate-docs/
   script:
     - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds
-    - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
-        time cargo +nightly doc --release --all --verbose
-    - cp -R ./target/doc ./crate-docs
+    - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"  
+        time cargo +nightly doc --no-deps --workspace --all-features --verbose
+    - mv ./target/doc ./crate-docs
     - echo "<meta http-equiv=refresh content=0;url=sc_service/index.html>" > ./crate-docs/index.html
     - sccache -s
 
@@ -670,7 +672,7 @@ publish-s3-doc:
   image:                           paritytech/awscli:latest
   allow_failure:                   true
   needs:
-    - job:                         build-rust-doc-release
+    - job:                         build-rust-doc
       artifacts:                   true
   <<:                              *build-only
   <<:                              *kubernetes-build
diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs
index 4ba76d7a06377..85400f2a27759 100644
--- a/client/cli/src/arg_enums.rs
+++ b/client/cli/src/arg_enums.rs
@@ -172,8 +172,6 @@ arg_enum! {
 	pub enum Database {
 		// Facebooks RocksDB
 		RocksDb,
-		// Subdb. https://github.com/paritytech/subdb/
-		SubDb,
 		// ParityDb. https://github.com/paritytech/parity-db/
 		ParityDb,
 	}
diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs
index 5da49fefd7acf..6acb786cc1c23 100644
--- a/client/cli/src/config.rs
+++ b/client/cli/src/config.rs
@@ -222,9 +222,6 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 				path: base_path.join("db"),
 				cache_size,
 			},
-			Database::SubDb => DatabaseConfig::SubDb {
-				path: base_path.join("subdb"),
-			},
 			Database::ParityDb => DatabaseConfig::ParityDb {
 				path: base_path.join("paritydb"),
 			},
diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml
index 004a7753e42d3..bbe6f83f4c1d2 100644
--- a/client/db/Cargo.toml
+++ b/client/db/Cargo.toml
@@ -50,4 +50,3 @@ default = []
 test-helpers = []
 with-kvdb-rocksdb = ["kvdb-rocksdb"]
 with-parity-db = ["parity-db"]
-with-subdb = []
diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs
index bd438f4dd71b2..927df1c0a7d25 100644
--- a/client/db/src/lib.rs
+++ b/client/db/src/lib.rs
@@ -44,8 +44,6 @@ mod utils;
 mod stats;
 #[cfg(feature = "with-parity-db")]
 mod parity_db;
-#[cfg(feature = "with-subdb")]
-mod subdb;
 
 use std::sync::Arc;
 use std::path::{Path, PathBuf};
@@ -287,12 +285,6 @@ pub enum DatabaseSettingsSrc {
 		path: PathBuf,
 	},
 
-	/// Load a Subdb database from a given path.
-	SubDb {
-		/// Path to the database.
-		path: PathBuf,
-	},
-
 	/// Use a custom already-open database.
 	Custom(Arc<dyn Database<DbHash>>),
 }
@@ -303,7 +295,6 @@ impl DatabaseSettingsSrc {
 		match self {
 			DatabaseSettingsSrc::RocksDb { path, .. } => Some(path.as_path()),
 			DatabaseSettingsSrc::ParityDb { path, .. } => Some(path.as_path()),
-			DatabaseSettingsSrc::SubDb { path, .. } => Some(path.as_path()),
 			DatabaseSettingsSrc::Custom(_) => None,
 		}
 	}
@@ -321,7 +312,6 @@ impl std::fmt::Display for DatabaseSettingsSrc {
 		let name = match self {
 			DatabaseSettingsSrc::RocksDb { .. } => "RocksDb",
 			DatabaseSettingsSrc::ParityDb { .. } => "ParityDb",
-			DatabaseSettingsSrc::SubDb { .. } => "SubDb",
 			DatabaseSettingsSrc::Custom(_) => "Custom",
 		};
 		write!(f, "{}", name)
diff --git a/client/db/src/subdb.rs b/client/db/src/subdb.rs
deleted file mode 100644
index 2f72632b04562..0000000000000
--- a/client/db/src/subdb.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-// This file is part of Substrate.
-
-// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
-// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-/// A `Database` adapter for subdb.
-
-use sp_database::{self, ColumnId};
-use parking_lot::RwLock;
-use blake2_rfc::blake2b::blake2b;
-use codec::Encode;
-use subdb::{Database, KeyType};
-
-/// A database hidden behind an RwLock, so that it implements Send + Sync.
-///
-/// Construct by creating a `Database` and then using `.into()`.
-pub struct DbAdapter<H: KeyType>(RwLock<Database<H>>);
-
-/// Wrap RocksDb database into a trait object that implements `sp_database::Database`
-pub fn open<H: KeyType + 'static>(
-	path: &std::path::Path,
-	_num_columns: u32,
-) -> Result<std::sync::Arc<dyn sp_database::Database<H>>, subdb::Error> {
-	let db = subdb::Options::from_path(path.into()).open()?;
-	Ok(std::sync::Arc::new(DbAdapter(RwLock::new(db))))
-}
-
-impl<H: KeyType> sp_database::Database<H> for DbAdapter<H> {
-	fn get(&self, col: ColumnId, key: &[u8]) -> Option<Vec<u8>> {
-		let mut hash = H::default();
-		(col, key).using_encoded(|d|
-			hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes())
-		);
-		self.0.read().get(&hash)
-	}
-
-	fn with_get(&self, col: ColumnId, key: &[u8], f: &mut dyn FnMut(&[u8])) {
-		let mut hash = H::default();
-		(col, key).using_encoded(|d|
-			hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes())
-		);
-		let _ = self.0.read().get_ref(&hash).map(|d| f(d.as_ref()));
-	}
-
-	fn set(&self, col: ColumnId, key: &[u8], value: &[u8]) {
-		let mut hash = H::default();
-		(col, key).using_encoded(|d|
-			hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes())
-		);
-		self.0.write().insert(&value, &hash);
-	}
-
-	fn remove(&self, col: ColumnId, key: &[u8]) {
-		let mut hash = H::default();
-		(col, key).using_encoded(|d|
-			hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes())
-		);
-		let _ = self.0.write().remove(&hash);
-	}
-
-	fn lookup(&self, hash: &H) -> Option<Vec<u8>> {
-		self.0.read().get(hash)
-	}
-
-	fn with_lookup(&self, hash: &H, f: &mut dyn FnMut(&[u8])) {
-		let _ = self.0.read().get_ref(hash).map(|d| f(d.as_ref()));
-	}
-
-	fn store(&self, hash: &H, preimage: &[u8]) {
-		self.0.write().insert(preimage, hash);
-	}
-
-	fn release(&self, hash: &H) {
-		let _ = self.0.write().remove(hash);
-	}
-}
diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs
index 168ab9bbb71f6..3ad6c421135d0 100644
--- a/client/db/src/utils.rs
+++ b/client/db/src/utils.rs
@@ -212,11 +212,12 @@ pub fn open_database<Block: BlockT>(
 	config: &DatabaseSettings,
 	db_type: DatabaseType,
 ) -> sp_blockchain::Result<Arc<dyn Database<DbHash>>> {
-	let db_open_error = |feat| Err(
+	#[allow(unused)]
+	fn db_open_error(feat: &'static str) -> sp_blockchain::Error {
 		sp_blockchain::Error::Backend(
 			format!("`{}` feature not enabled, database can not be opened", feat),
-		),
-	);
+		)
+	}
 
 	let db: Arc<dyn Database<DbHash>> = match &config.source {
 		#[cfg(any(feature = "with-kvdb-rocksdb", test))]
@@ -257,16 +258,7 @@ pub fn open_database<Block: BlockT>(
 		},
 		#[cfg(not(any(feature = "with-kvdb-rocksdb", test)))]
 		DatabaseSettingsSrc::RocksDb { .. } => {
-			return db_open_error("with-kvdb-rocksdb");
-		},
-		#[cfg(feature = "with-subdb")]
-		DatabaseSettingsSrc::SubDb { path } => {
-			crate::subdb::open(&path, NUM_COLUMNS)
-				.map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))?
-		},
-		#[cfg(not(feature = "with-subdb"))]
-		DatabaseSettingsSrc::SubDb { .. } => {
-			return db_open_error("with-subdb");
+			return Err(db_open_error("with-kvdb-rocksdb"));
 		},
 		#[cfg(feature = "with-parity-db")]
 		DatabaseSettingsSrc::ParityDb { path } => {
@@ -275,7 +267,7 @@ pub fn open_database<Block: BlockT>(
 		},
 		#[cfg(not(feature = "with-parity-db"))]
 		DatabaseSettingsSrc::ParityDb { .. } => {
-			return db_open_error("with-parity-db");
+			return Err(db_open_error("with-parity-db"))
 		},
 		DatabaseSettingsSrc::Custom(db) => db.clone(),
 	};
diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs
index 326d73c372110..3fd01c33dcf5f 100644
--- a/client/network/src/lib.rs
+++ b/client/network/src/lib.rs
@@ -267,7 +267,10 @@ pub mod network_state;
 #[doc(inline)]
 pub use libp2p::{multiaddr, Multiaddr, PeerId};
 pub use protocol::{event::{DhtEvent, Event, ObservedRole}, sync::SyncState, PeerInfo};
-pub use service::{NetworkService, NetworkWorker, RequestFailure, OutboundFailure};
+pub use service::{
+	NetworkService, NetworkWorker, RequestFailure, OutboundFailure, NotificationSender,
+	NotificationSenderReady,
+};
 
 pub use sc_peerset::ReputationChange;
 use sp_runtime::traits::{Block as BlockT, NumberFor};
diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs
index 92233c77d6bd1..3065d83286137 100644
--- a/client/network/src/request_responses.rs
+++ b/client/network/src/request_responses.rs
@@ -16,7 +16,7 @@
 
 //! Collection of request-response protocols.
 //!
-//! The [`RequestResponses`] struct defined in this module provides support for zero or more
+//! The [`RequestResponse`] struct defined in this module provides support for zero or more
 //! so-called "request-response" protocols.
 //!
 //! A request-response protocol works in the following way:
@@ -29,7 +29,7 @@
 //! - Requests have a certain time limit before they time out. This time includes the time it
 //! takes to send/receive the request and response.
 //!
-//! - If provided, a ["requests processing"](RequestResponseConfig::inbound_queue) channel
+//! - If provided, a ["requests processing"](ProtocolConfig::inbound_queue) channel
 //! is used to handle incoming requests.
 //!
 
@@ -108,7 +108,7 @@ pub struct IncomingRequest {
 	pub peer: PeerId,
 
 	/// Request sent by the remote. Will always be smaller than
-	/// [`RequestResponseConfig::max_request_size`].
+	/// [`ProtocolConfig::max_request_size`].
 	pub payload: Vec<u8>,
 
 	/// Channel to send back the response to.
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index d1248057cc79f..f9f877030fe10 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -639,7 +639,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// >			preventing the message from being delivered.
 	///
 	/// The protocol must have been registered with `register_notifications_protocol` or
-	/// `NetworkConfiguration::notifications_protocols`.
+	/// [`NetworkConfiguration::notifications_protocols`](crate::config::NetworkConfiguration::notifications_protocols).
 	///
 	pub fn write_notification(&self, target: PeerId, engine_id: ConsensusEngineId, message: Vec<u8>) {
 		// We clone the `NotificationsSink` in order to be able to unlock the network-wide
@@ -682,10 +682,9 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// 2.  [`NotificationSenderReady::send`] enqueues the notification for sending. This operation
 	/// can only fail if the underlying notification substream or connection has suddenly closed.
 	///
-	/// An error is returned either by `notification_sender`, by [`NotificationSender::wait`],
-	/// or by [`NotificationSenderReady::send`] if there exists no open notifications substream
-	/// with that combination of peer and protocol, or if the remote has asked to close the
-	/// notifications substream. If that happens, it is guaranteed that an
+	/// An error is returned by [`NotificationSenderReady::send`] if there exists no open
+	/// notifications substream with that combination of peer and protocol, or if the remote
+	/// has asked to close the notifications substream. If that happens, it is guaranteed that an
 	/// [`Event::NotificationStreamClosed`] has been generated on the stream returned by
 	/// [`NetworkService::event_stream`].
 	///
@@ -696,7 +695,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// in which case enqueued notifications will be lost.
 	///
 	/// The protocol must have been registered with `register_notifications_protocol` or
-	/// `NetworkConfiguration::notifications_protocols`.
+	/// [`NetworkConfiguration::notifications_protocols`](crate::config::NetworkConfiguration::notifications_protocols).
 	///
 	/// # Usage
 	///
@@ -801,7 +800,8 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// Such restrictions, if desired, need to be enforced at the call site(s).
 	///
 	/// The protocol must have been registered through
-	/// [`NetworkConfiguration::request_response_protocols`].
+	/// [`NetworkConfiguration::request_response_protocols`](
+	/// crate::config::NetworkConfiguration::request_response_protocols).
 	pub async fn request(
 		&self,
 		target: PeerId,
diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs
index 79bc9136ef4a7..e1303fcd03b0d 100644
--- a/frame/assets/src/lib.rs
+++ b/frame/assets/src/lib.rs
@@ -230,11 +230,11 @@ decl_event! {
 		<T as Trait>::Balance,
 		<T as Trait>::AssetId,
 	{
-		/// Some assets were issued. [asset_id, owner, total_supply]
+		/// Some assets were issued. \[asset_id, owner, total_supply\]
 		Issued(AssetId, AccountId, Balance),
-		/// Some assets were transferred. [asset_id, from, to, amount]
+		/// Some assets were transferred. \[asset_id, from, to, amount\]
 		Transferred(AssetId, AccountId, AccountId, Balance),
-		/// Some assets were destroyed. [asset_id, owner, balance]
+		/// Some assets were destroyed. \[asset_id, owner, balance\]
 		Destroyed(AssetId, AccountId, Balance),
 	}
 }
diff --git a/frame/atomic-swap/src/lib.rs b/frame/atomic-swap/src/lib.rs
index 65794792d0aa4..31f0c0f426525 100644
--- a/frame/atomic-swap/src/lib.rs
+++ b/frame/atomic-swap/src/lib.rs
@@ -189,12 +189,12 @@ decl_event!(
 		AccountId = <T as system::Trait>::AccountId,
 		PendingSwap = PendingSwap<T>,
 	{
-		/// Swap created. [account, proof, swap]
+		/// Swap created. \[account, proof, swap\]
 		NewSwap(AccountId, HashedProof, PendingSwap),
 		/// Swap claimed. The last parameter indicates whether the execution succeeds. 
-		/// [account, proof, success]
+		/// \[account, proof, success\]
 		SwapClaimed(AccountId, HashedProof, bool),
-		/// Swap cancelled. [account, proof]
+		/// Swap cancelled. \[account, proof\]
 		SwapCancelled(AccountId, HashedProof),
 	}
 );
diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs
index f65ed6b99a6d1..331c5a27dfa74 100644
--- a/frame/balances/src/lib.rs
+++ b/frame/balances/src/lib.rs
@@ -235,24 +235,24 @@ decl_event!(
 		<T as frame_system::Trait>::AccountId,
 		<T as Trait<I>>::Balance
 	{
-		/// An account was created with some free balance. [account, free_balance]
+		/// An account was created with some free balance. \[account, free_balance\]
 		Endowed(AccountId, Balance),
 		/// An account was removed whose balance was non-zero but below ExistentialDeposit,
-		/// resulting in an outright loss. [account, balance]
+		/// resulting in an outright loss. \[account, balance\]
 		DustLost(AccountId, Balance),
-		/// Transfer succeeded. [from, to, value]
+		/// Transfer succeeded. \[from, to, value\]
 		Transfer(AccountId, AccountId, Balance),
-		/// A balance was set by root. [who, free, reserved]
+		/// A balance was set by root. \[who, free, reserved\]
 		BalanceSet(AccountId, Balance, Balance),
-		/// Some amount was deposited (e.g. for transaction fees). [who, deposit]
+		/// Some amount was deposited (e.g. for transaction fees). \[who, deposit\]
 		Deposit(AccountId, Balance),
-		/// Some balance was reserved (moved from free to reserved). [who, value]
+		/// Some balance was reserved (moved from free to reserved). \[who, value\]
 		Reserved(AccountId, Balance),
-		/// Some balance was unreserved (moved from reserved to free). [who, value]
+		/// Some balance was unreserved (moved from reserved to free). \[who, value\]
 		Unreserved(AccountId, Balance),
 		/// Some balance was moved from the reserve of the first account to the second account.
 		/// Final argument indicates the destination balance type.
-		/// [from, to, balance, destination_status]
+		/// \[from, to, balance, destination_status\]
 		ReserveRepatriated(AccountId, AccountId, Balance, Status),
 	}
 );
diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs
index 949484a5957b4..20c701e3f0491 100644
--- a/frame/collective/src/lib.rs
+++ b/frame/collective/src/lib.rs
@@ -175,26 +175,26 @@ decl_event! {
 	{
 		/// A motion (given hash) has been proposed (by given account) with a threshold (given
 		/// `MemberCount`).
-		/// [account, proposal_index, proposal_hash, threshold]
+		/// \[account, proposal_index, proposal_hash, threshold\]
 		Proposed(AccountId, ProposalIndex, Hash, MemberCount),
 		/// A motion (given hash) has been voted on by given account, leaving
 		/// a tally (yes votes and no votes given respectively as `MemberCount`).
-		/// [account, proposal_hash, voted, yes, no]
+		/// \[account, proposal_hash, voted, yes, no\]
 		Voted(AccountId, Hash, bool, MemberCount, MemberCount),
 		/// A motion was approved by the required threshold.
-		/// [proposal_hash]
+		/// \[proposal_hash\]
 		Approved(Hash),
 		/// A motion was not approved by the required threshold.
-		/// [proposal_hash]
+		/// \[proposal_hash\]
 		Disapproved(Hash),
 		/// A motion was executed; result will be `Ok` if it returned without error.
-		/// [proposal_hash, result]
+		/// \[proposal_hash, result\]
 		Executed(Hash, DispatchResult),
 		/// A single member did some action; result will be `Ok` if it returned without error.
-		/// [proposal_hash, result]
+		/// \[proposal_hash, result\]
 		MemberExecuted(Hash, DispatchResult),
 		/// A proposal was closed because its threshold was reached or after its duration was up.
-		/// [proposal_hash, yes, no]
+		/// \[proposal_hash, yes, no\]
 		Closed(Hash, MemberCount, MemberCount),
 	}
 }
diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs
index 138c8e995a0a2..4755573783af7 100644
--- a/frame/contracts/src/lib.rs
+++ b/frame/contracts/src/lib.rs
@@ -686,11 +686,11 @@ decl_event! {
 		<T as frame_system::Trait>::AccountId,
 		<T as frame_system::Trait>::Hash
 	{
-		/// Contract deployed by address at the specified address. [owner, contract]
+		/// Contract deployed by address at the specified address. \[owner, contract\]
 		Instantiated(AccountId, AccountId),
 
 		/// Contract has been evicted and is now in tombstone state.
-		/// [contract, tombstone]
+		/// \[contract, tombstone\]
 		/// 
 		/// # Params
 		///
@@ -699,7 +699,7 @@ decl_event! {
 		Evicted(AccountId, bool),
 
 		/// Restoration for a contract has been successful.
-		/// [donor, dest, code_hash, rent_allowance]
+		/// \[donor, dest, code_hash, rent_allowance\]
 		/// 
 		/// # Params
 		///
@@ -710,14 +710,14 @@ decl_event! {
 		Restored(AccountId, AccountId, Hash, Balance),
 
 		/// Code with the specified hash has been stored.
-		/// [code_hash]
+		/// \[code_hash\]
 		CodeStored(Hash),
 
-		/// Triggered when the current [schedule] is updated.
+		/// Triggered when the current \[schedule\] is updated.
 		ScheduleUpdated(u32),
 
 		/// An event deposited upon execution of a contract from the account.
-		/// [account, data]
+		/// \[account, data\]
 		ContractExecution(AccountId, Vec<u8>),
 	}
 }
diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs
index e298b1e4508c2..9ed732d3234ea 100644
--- a/frame/democracy/src/lib.rs
+++ b/frame/democracy/src/lib.rs
@@ -434,41 +434,43 @@ decl_event! {
 		<T as frame_system::Trait>::Hash,
 		<T as frame_system::Trait>::BlockNumber,
 	{
-		/// A motion has been proposed by a public account. [proposal_index, deposit]
+		/// A motion has been proposed by a public account. \[proposal_index, deposit\]
 		Proposed(PropIndex, Balance),
-		/// A public proposal has been tabled for referendum vote. [proposal_index, deposit, depositors]
+		/// A public proposal has been tabled for referendum vote. \[proposal_index, deposit, depositors\]
 		Tabled(PropIndex, Balance, Vec<AccountId>),
 		/// An external proposal has been tabled.
 		ExternalTabled,
-		/// A referendum has begun. [ref_index, threshold]
+		/// A referendum has begun. \[ref_index, threshold\]
 		Started(ReferendumIndex, VoteThreshold),
-		/// A proposal has been approved by referendum. [ref_index]
+		/// A proposal has been approved by referendum. \[ref_index\]
 		Passed(ReferendumIndex),
-		/// A proposal has been rejected by referendum. [ref_index]
+		/// A proposal has been rejected by referendum. \[ref_index\]
 		NotPassed(ReferendumIndex),
-		/// A referendum has been cancelled. [ref_index]
+		/// A referendum has been cancelled. \[ref_index\]
 		Cancelled(ReferendumIndex),
-		/// A proposal has been enacted. [ref_index, is_ok]
+		/// A proposal has been enacted. \[ref_index, is_ok\]
 		Executed(ReferendumIndex, bool),
-		/// An account has delegated their vote to another account. [who, target]
+		/// An account has delegated their vote to another account. \[who, target\]
 		Delegated(AccountId, AccountId),
-		/// An [account] has cancelled a previous delegation operation.
+		/// An \[account\] has cancelled a previous delegation operation.
 		Undelegated(AccountId),
-		/// An external proposal has been vetoed. [who, proposal_hash, until]
+		/// An external proposal has been vetoed. \[who, proposal_hash, until\]
 		Vetoed(AccountId, Hash, BlockNumber),
-		/// A proposal's preimage was noted, and the deposit taken. [proposal_hash, who, deposit]
+		/// A proposal's preimage was noted, and the deposit taken. \[proposal_hash, who, deposit\]
 		PreimageNoted(Hash, AccountId, Balance),
 		/// A proposal preimage was removed and used (the deposit was returned).
-		/// [proposal_hash, provider, deposit]
+		/// \[proposal_hash, provider, deposit\]
 		PreimageUsed(Hash, AccountId, Balance),
-		/// A proposal could not be executed because its preimage was invalid. [proposal_hash, ref_index]
+		/// A proposal could not be executed because its preimage was invalid.
+		/// \[proposal_hash, ref_index\]
 		PreimageInvalid(Hash, ReferendumIndex),
-		/// A proposal could not be executed because its preimage was missing. [proposal_hash, ref_index]
+		/// A proposal could not be executed because its preimage was missing.
+		/// \[proposal_hash, ref_index\]
 		PreimageMissing(Hash, ReferendumIndex),
 		/// A registered preimage was removed and the deposit collected by the reaper.
-		/// [proposal_hash, provider, deposit, reaper]
+		/// \[proposal_hash, provider, deposit, reaper\]
 		PreimageReaped(Hash, AccountId, Balance, AccountId),
-		/// An [account] has been unlocked successfully.
+		/// An \[account\] has been unlocked successfully.
 		Unlocked(AccountId),
 	}
 }
diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs
index 9d1922576ad42..0b93dd6c13b9c 100644
--- a/frame/elections-phragmen/src/lib.rs
+++ b/frame/elections-phragmen/src/lib.rs
@@ -709,21 +709,21 @@ decl_event!(
 		Balance = BalanceOf<T>,
 		<T as frame_system::Trait>::AccountId,
 	{
-		/// A new term with [new_members]. This indicates that enough candidates existed to run the
+		/// A new term with \[new_members\]. This indicates that enough candidates existed to run the
 		/// election, not that enough have has been elected. The inner value must be examined for
-		/// this purpose. A `NewTerm([])` indicates that some candidates got their bond slashed and
+		/// this purpose. A `NewTerm(\[\])` indicates that some candidates got their bond slashed and
 		/// none were elected, whilst `EmptyTerm` means that no candidates existed to begin with.
 		NewTerm(Vec<(AccountId, Balance)>),
 		/// No (or not enough) candidates existed for this round. This is different from
-		/// `NewTerm([])`. See the description of `NewTerm`.
+		/// `NewTerm(\[\])`. See the description of `NewTerm`.
 		EmptyTerm,
-		/// A [member] has been removed. This should always be followed by either `NewTerm` ot
+		/// A \[member\] has been removed. This should always be followed by either `NewTerm` ot
 		/// `EmptyTerm`.
 		MemberKicked(AccountId),
-		/// A [member] has renounced their candidacy.
+		/// A \[member\] has renounced their candidacy.
 		MemberRenounced(AccountId),
 		/// A voter was reported with the the report being successful or not.
-		/// [voter, reporter, success]
+		/// \[voter, reporter, success\]
 		VoterReported(AccountId, AccountId, bool),
 	}
 );
diff --git a/frame/elections/src/lib.rs b/frame/elections/src/lib.rs
index 1453e2f0fd9fc..a5c6d0eb2ba2d 100644
--- a/frame/elections/src/lib.rs
+++ b/frame/elections/src/lib.rs
@@ -700,14 +700,14 @@ decl_module! {
 
 decl_event!(
 	pub enum Event<T> where <T as frame_system::Trait>::AccountId {
-		/// Reaped [voter, reaper].
+		/// Reaped \[voter, reaper\].
 		VoterReaped(AccountId, AccountId),
-		/// Slashed [reaper].
+		/// Slashed \[reaper\].
 		BadReaperSlashed(AccountId),
-		/// A tally (for approval votes of [seats]) has started.
+		/// A tally (for approval votes of \[seats\]) has started.
 		TallyStarted(u32),
 		/// A tally (for approval votes of seat(s)) has ended (with one or more new members). 
-		/// [incoming, outgoing]
+		/// \[incoming, outgoing\]
 		TallyFinalized(Vec<AccountId>, Vec<AccountId>),
 	}
 );
diff --git a/frame/evm/src/lib.rs b/frame/evm/src/lib.rs
index 7719f5fb7efa0..a94ffe9535888 100644
--- a/frame/evm/src/lib.rs
+++ b/frame/evm/src/lib.rs
@@ -261,17 +261,17 @@ decl_event! {
 	{
 		/// Ethereum events from contracts.
 		Log(Log),
-		/// A contract has been created at given [address].
+		/// A contract has been created at given \[address\].
 		Created(H160),
-		/// A [contract] was attempted to be created, but the execution failed.
+		/// A \[contract\] was attempted to be created, but the execution failed.
 		CreatedFailed(H160),
-		/// A [contract] has been executed successfully with states applied.
+		/// A \[contract\] has been executed successfully with states applied.
 		Executed(H160),
-		/// A [contract] has been executed with errors. States are reverted with only gas fees applied.
+		/// A \[contract\] has been executed with errors. States are reverted with only gas fees applied.
 		ExecutedFailed(H160),
-		/// A deposit has been made at a given address. [sender, address, value]
+		/// A deposit has been made at a given address. \[sender, address, value\]
 		BalanceDeposit(AccountId, H160, U256),
-		/// A withdrawal has been made from a given address. [sender, address, value]
+		/// A withdrawal has been made from a given address. \[sender, address, value\]
 		BalanceWithdraw(AccountId, H160, U256),
 	}
 }
diff --git a/frame/example-offchain-worker/src/lib.rs b/frame/example-offchain-worker/src/lib.rs
index b9ee6d3d8b5ed..8e02a09484ef5 100644
--- a/frame/example-offchain-worker/src/lib.rs
+++ b/frame/example-offchain-worker/src/lib.rs
@@ -166,7 +166,7 @@ decl_event!(
 	/// Events generated by the module.
 	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId {
 		/// Event generated when new price is accepted to contribute to the average.
-		/// [price, who]
+		/// \[price, who\]
 		NewPrice(u32, AccountId),
 	}
 );
diff --git a/frame/generic-asset/src/lib.rs b/frame/generic-asset/src/lib.rs
index 881d89439ec7b..534a97cf5372a 100644
--- a/frame/generic-asset/src/lib.rs
+++ b/frame/generic-asset/src/lib.rs
@@ -493,15 +493,15 @@ decl_event!(
 		<T as Trait>::AssetId,
 		AssetOptions = AssetOptions<<T as Trait>::Balance, <T as frame_system::Trait>::AccountId>
 	{
-		/// Asset created. [asset_id, creator, asset_options]
+		/// Asset created. \[asset_id, creator, asset_options\]
 		Created(AssetId, AccountId, AssetOptions),
-		/// Asset transfer succeeded. [asset_id, from, to, amount]
+		/// Asset transfer succeeded. \[asset_id, from, to, amount\]
 		Transferred(AssetId, AccountId, AccountId, Balance),
-		/// Asset permission updated. [asset_id, new_permissions]
+		/// Asset permission updated. \[asset_id, new_permissions\]
 		PermissionUpdated(AssetId, PermissionLatest<AccountId>),
-		/// New asset minted. [asset_id, account, amount]
+		/// New asset minted. \[asset_id, account, amount\]
 		Minted(AssetId, AccountId, Balance),
-		/// Asset burned. [asset_id, account, amount]
+		/// Asset burned. \[asset_id, account, amount\]
 		Burned(AssetId, AccountId, Balance),
 	}
 );
diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs
index e0f2d7beda2a6..893bfc0dd5b27 100644
--- a/frame/grandpa/src/lib.rs
+++ b/frame/grandpa/src/lib.rs
@@ -170,7 +170,7 @@ pub enum StoredState<N> {
 
 decl_event! {
 	pub enum Event {
-		/// New authority set has been applied. [authority_set]
+		/// New authority set has been applied. \[authority_set\]
 		NewAuthorities(AuthorityList),
 		/// Current authority set has been paused.
 		Paused,
diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs
index 1607835f2414b..65f1597622c56 100644
--- a/frame/identity/src/lib.rs
+++ b/frame/identity/src/lib.rs
@@ -462,27 +462,27 @@ decl_storage! {
 
 decl_event!(
 	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId, Balance = BalanceOf<T> {
-		/// A name was set or reset (which will remove all judgements). [who]
+		/// A name was set or reset (which will remove all judgements). \[who\]
 		IdentitySet(AccountId),
-		/// A name was cleared, and the given balance returned. [who, deposit]
+		/// A name was cleared, and the given balance returned. \[who, deposit\]
 		IdentityCleared(AccountId, Balance),
-		/// A name was removed and the given balance slashed. [who, deposit]
+		/// A name was removed and the given balance slashed. \[who, deposit\]
 		IdentityKilled(AccountId, Balance),
-		/// A judgement was asked from a registrar. [who, registrar_index]
+		/// A judgement was asked from a registrar. \[who, registrar_index\]
 		JudgementRequested(AccountId, RegistrarIndex),
-		/// A judgement request was retracted. [who, registrar_index]
+		/// A judgement request was retracted. \[who, registrar_index\]
 		JudgementUnrequested(AccountId, RegistrarIndex),
-		/// A judgement was given by a registrar. [target, registrar_index]
+		/// A judgement was given by a registrar. \[target, registrar_index\]
 		JudgementGiven(AccountId, RegistrarIndex),
-		/// A registrar was added. [registrar_index]
+		/// A registrar was added. \[registrar_index\]
 		RegistrarAdded(RegistrarIndex),
-		/// A sub-identity was added to an identity and the deposit paid. [sub, main, deposit]
+		/// A sub-identity was added to an identity and the deposit paid. \[sub, main, deposit\]
 		SubIdentityAdded(AccountId, AccountId, Balance),
 		/// A sub-identity was removed from an identity and the deposit freed.
-		/// [sub, main, deposit]
+		/// \[sub, main, deposit\]
 		SubIdentityRemoved(AccountId, AccountId, Balance),
 		/// A sub-identity was cleared, and the given deposit repatriated from the
-		/// main identity account to the sub-identity account. [sub, main, deposit]
+		/// main identity account to the sub-identity account. \[sub, main, deposit\]
 		SubIdentityRevoked(AccountId, AccountId, Balance),
 	}
 );
diff --git a/frame/im-online/src/lib.rs b/frame/im-online/src/lib.rs
index 01b7b999dd004..7856ecfd5aa46 100644
--- a/frame/im-online/src/lib.rs
+++ b/frame/im-online/src/lib.rs
@@ -276,11 +276,11 @@ decl_event!(
 		<T as Trait>::AuthorityId,
 		IdentificationTuple = IdentificationTuple<T>,
 	{
-		/// A new heartbeat was received from `AuthorityId` [authority_id]
+		/// A new heartbeat was received from `AuthorityId` \[authority_id\]
 		HeartbeatReceived(AuthorityId),
 		/// At the end of the session, no offence was committed.
 		AllGood,
-		/// At the end of the session, at least one validator was found to be [offline].
+		/// At the end of the session, at least one validator was found to be \[offline\].
 		SomeOffline(Vec<IdentificationTuple>),
 	}
 );
diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs
index e03cf4f1eea4d..3dc0cec9d94bc 100644
--- a/frame/indices/src/lib.rs
+++ b/frame/indices/src/lib.rs
@@ -95,11 +95,11 @@ decl_event!(
 		<T as frame_system::Trait>::AccountId,
 		<T as Trait>::AccountIndex
 	{
-		/// A account index was assigned. [who, index]
+		/// A account index was assigned. \[who, index\]
 		IndexAssigned(AccountId, AccountIndex),
-		/// A account index has been freed up (unassigned). [index]
+		/// A account index has been freed up (unassigned). \[index\]
 		IndexFreed(AccountIndex),
-		/// A account index has been frozen to its current account ID. [who, index]
+		/// A account index has been frozen to its current account ID. \[who, index\]
 		IndexFrozen(AccountIndex, AccountId),
 	}
 );
diff --git a/frame/multisig/src/lib.rs b/frame/multisig/src/lib.rs
index 72a0f7cd070a2..06f91f8d0fd7b 100644
--- a/frame/multisig/src/lib.rs
+++ b/frame/multisig/src/lib.rs
@@ -197,13 +197,14 @@ decl_event! {
 		BlockNumber = <T as system::Trait>::BlockNumber,
 		CallHash = [u8; 32]
 	{
-		/// A new multisig operation has begun. [approving, multisig, call_hash]
+		/// A new multisig operation has begun. \[approving, multisig, call_hash\]
 		NewMultisig(AccountId, AccountId, CallHash),
-		/// A multisig operation has been approved by someone. [approving, timepoint, multisig, call_hash]
+		/// A multisig operation has been approved by someone.
+		/// \[approving, timepoint, multisig, call_hash\]
 		MultisigApproval(AccountId, Timepoint<BlockNumber>, AccountId, CallHash),
-		/// A multisig operation has been executed. [approving, timepoint, multisig, call_hash]
+		/// A multisig operation has been executed. \[approving, timepoint, multisig, call_hash\]
 		MultisigExecuted(AccountId, Timepoint<BlockNumber>, AccountId, CallHash, DispatchResult),
-		/// A multisig operation has been cancelled. [cancelling, timepoint, multisig, call_hash]
+		/// A multisig operation has been cancelled. \[cancelling, timepoint, multisig, call_hash\]
 		MultisigCancelled(AccountId, Timepoint<BlockNumber>, AccountId, CallHash),
 	}
 }
diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs
index 87a6e3b0d8b38..a1faedaf1cee6 100644
--- a/frame/nicks/src/lib.rs
+++ b/frame/nicks/src/lib.rs
@@ -86,15 +86,15 @@ decl_storage! {
 
 decl_event!(
 	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId, Balance = BalanceOf<T> {
-		/// A name was set. [who]
+		/// A name was set. \[who\]
 		NameSet(AccountId),
-		/// A name was forcibly set. [target]
+		/// A name was forcibly set. \[target\]
 		NameForced(AccountId),
-		/// A name was changed. [who]
+		/// A name was changed. \[who\]
 		NameChanged(AccountId),
-		/// A name was cleared, and the given balance returned. [who, deposit]
+		/// A name was cleared, and the given balance returned. \[who, deposit\]
 		NameCleared(AccountId, Balance),
-		/// A name was removed and the given balance slashed. [target, deposit]
+		/// A name was removed and the given balance slashed. \[target, deposit\]
 		NameKilled(AccountId, Balance),
 	}
 );
diff --git a/frame/offences/src/lib.rs b/frame/offences/src/lib.rs
index 9a067d903fe2d..bf072f4a405f3 100644
--- a/frame/offences/src/lib.rs
+++ b/frame/offences/src/lib.rs
@@ -112,7 +112,7 @@ decl_event!(
 		/// There is an offence reported of the given `kind` happened at the `session_index` and
 		/// (kind-specific) time slot. This event is not deposited for duplicate slashes. last
 		/// element indicates of the offence was applied (true) or queued (false) 
-		/// [kind, timeslot, applied].
+		/// \[kind, timeslot, applied\].
 		Offence(Kind, OpaqueTimeSlot, bool),
 	}
 );
diff --git a/frame/proxy/src/lib.rs b/frame/proxy/src/lib.rs
index 5a852ea9f5314..4746a4ab67c17 100644
--- a/frame/proxy/src/lib.rs
+++ b/frame/proxy/src/lib.rs
@@ -191,12 +191,12 @@ decl_event! {
 		ProxyType = <T as Trait>::ProxyType,
 		Hash = CallHashOf<T>,
 	{
-		/// A proxy was executed correctly, with the given [result].
+		/// A proxy was executed correctly, with the given \[result\].
 		ProxyExecuted(DispatchResult),
 		/// Anonymous account has been created by new proxy with given
-		/// disambiguation index and proxy type. [anonymous, who, proxy_type, disambiguation_index]
+		/// disambiguation index and proxy type. \[anonymous, who, proxy_type, disambiguation_index\]
 		AnonymousCreated(AccountId, AccountId, ProxyType, u16),
-		/// An announcement was placed to make a call in the future. [real, proxy, call_hash]
+		/// An announcement was placed to make a call in the future. \[real, proxy, call_hash\]
 		Announced(AccountId, AccountId, Hash),
 	}
 }
diff --git a/frame/recovery/src/lib.rs b/frame/recovery/src/lib.rs
index 1c0dd5041380f..b3aad8433eb3c 100644
--- a/frame/recovery/src/lib.rs
+++ b/frame/recovery/src/lib.rs
@@ -264,21 +264,21 @@ decl_event! {
 	pub enum Event<T> where
 		AccountId = <T as system::Trait>::AccountId,
 	{
-		/// A recovery process has been set up for an [account].
+		/// A recovery process has been set up for an \[account\].
 		RecoveryCreated(AccountId),
 		/// A recovery process has been initiated for lost account by rescuer account.
-		/// [lost, rescuer]
+		/// \[lost, rescuer\]
 		RecoveryInitiated(AccountId, AccountId),
 		/// A recovery process for lost account by rescuer account has been vouched for by sender.
-		/// [lost, rescuer, sender]
+		/// \[lost, rescuer, sender\]
 		RecoveryVouched(AccountId, AccountId, AccountId),
 		/// A recovery process for lost account by rescuer account has been closed.
-		/// [lost, rescuer]
+		/// \[lost, rescuer\]
 		RecoveryClosed(AccountId, AccountId),
 		/// Lost account has been successfully recovered by rescuer account.
-		/// [lost, rescuer]
+		/// \[lost, rescuer\]
 		AccountRecovered(AccountId, AccountId),
-		/// A recovery process has been removed for an [account].
+		/// A recovery process has been removed for an \[account\].
 		RecoveryRemoved(AccountId),
 	}
 }
diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs
index 831ed64d438d7..edd112bd89299 100644
--- a/frame/scheduler/src/lib.rs
+++ b/frame/scheduler/src/lib.rs
@@ -177,11 +177,11 @@ decl_storage! {
 
 decl_event!(
 	pub enum Event<T> where <T as system::Trait>::BlockNumber {
-		/// Scheduled some task. [when, index]
+		/// Scheduled some task. \[when, index\]
 		Scheduled(BlockNumber, u32),
-		/// Canceled some task. [when, index]
+		/// Canceled some task. \[when, index\]
 		Canceled(BlockNumber, u32),
-		/// Dispatched some task. [task, id, result]
+		/// Dispatched some task. \[task, id, result\]
 		Dispatched(TaskAddress<BlockNumber>, Option<Vec<u8>>, DispatchResult),
 	}
 );
diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs
index 2c1cba7137dcc..ede88b26f99bb 100644
--- a/frame/session/src/lib.rs
+++ b/frame/session/src/lib.rs
@@ -484,7 +484,7 @@ decl_storage! {
 
 decl_event!(
 	pub enum Event {
-		/// New session has happened. Note that the argument is the [session_index], not the block
+		/// New session has happened. Note that the argument is the \[session_index\], not the block
 		/// number as the type might suggest.
 		NewSession(SessionIndex),
 	}
diff --git a/frame/society/src/lib.rs b/frame/society/src/lib.rs
index 69ba46c832955..cbfe5a00de240 100644
--- a/frame/society/src/lib.rs
+++ b/frame/society/src/lib.rs
@@ -1111,40 +1111,40 @@ decl_event! {
 		AccountId = <T as system::Trait>::AccountId,
 		Balance = BalanceOf<T, I>
 	{
-		/// The society is founded by the given identity. [founder]
+		/// The society is founded by the given identity. \[founder\]
 		Founded(AccountId),
 		/// A membership bid just happened. The given account is the candidate's ID and their offer
-		/// is the second. [candidate_id, offer]
+		/// is the second. \[candidate_id, offer\]
 		Bid(AccountId, Balance),
 		/// A membership bid just happened by vouching. The given account is the candidate's ID and
-		/// their offer is the second. The vouching party is the third. [candidate_id, offer, vouching]
+		/// their offer is the second. The vouching party is the third. \[candidate_id, offer, vouching\]
 		Vouch(AccountId, Balance, AccountId),
-		/// A [candidate] was dropped (due to an excess of bids in the system).
+		/// A \[candidate\] was dropped (due to an excess of bids in the system).
 		AutoUnbid(AccountId),
-		/// A [candidate] was dropped (by their request).
+		/// A \[candidate\] was dropped (by their request).
 		Unbid(AccountId),
-		/// A [candidate] was dropped (by request of who vouched for them).
+		/// A \[candidate\] was dropped (by request of who vouched for them).
 		Unvouch(AccountId),
 		/// A group of candidates have been inducted. The batch's primary is the first value, the
-		/// batch in full is the second. [primary, candidates]
+		/// batch in full is the second. \[primary, candidates\]
 		Inducted(AccountId, Vec<AccountId>),
-		/// A suspended member has been judged. [who, judged]
+		/// A suspended member has been judged. \[who, judged\]
 		SuspendedMemberJudgement(AccountId, bool),
-		/// A [candidate] has been suspended
+		/// A \[candidate\] has been suspended
 		CandidateSuspended(AccountId),
-		/// A [member] has been suspended
+		/// A \[member\] has been suspended
 		MemberSuspended(AccountId),
-		/// A [member] has been challenged
+		/// A \[member\] has been challenged
 		Challenged(AccountId),
-		/// A vote has been placed [candidate, voter, vote]
+		/// A vote has been placed \[candidate, voter, vote\]
 		Vote(AccountId, AccountId, bool),
-		/// A vote has been placed for a defending member [voter, vote]
+		/// A vote has been placed for a defending member \[voter, vote\]
 		DefenderVote(AccountId, bool),
-		/// A new [max] member count has been set
+		/// A new \[max\] member count has been set
 		NewMaxMembers(u32),
-		/// Society is unfounded. [founder]
+		/// Society is unfounded. \[founder\]
 		Unfounded(AccountId),
-		/// Some funds were deposited into the society account. [value]
+		/// Some funds were deposited into the society account. \[value\]
 		Deposit(Balance),
 	}
 }
diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs
index b49ec12109d2a..279b6bb1dec7d 100644
--- a/frame/staking/src/lib.rs
+++ b/frame/staking/src/lib.rs
@@ -1241,29 +1241,29 @@ decl_event!(
 	pub enum Event<T> where Balance = BalanceOf<T>, <T as frame_system::Trait>::AccountId {
 		/// The era payout has been set; the first balance is the validator-payout; the second is
 		/// the remainder from the maximum amount of reward.
-		/// [era_index, validator_payout, remainder]
+		/// \[era_index, validator_payout, remainder\]
 		EraPayout(EraIndex, Balance, Balance),
-		/// The staker has been rewarded by this amount. [stash, amount]
+		/// The staker has been rewarded by this amount. \[stash, amount\]
 		Reward(AccountId, Balance),
 		/// One validator (and its nominators) has been slashed by the given amount.
-		/// [validator, amount]
+		/// \[validator, amount\]
 		Slash(AccountId, Balance),
 		/// An old slashing report from a prior era was discarded because it could
-		/// not be processed. [session_index]
+		/// not be processed. \[session_index\]
 		OldSlashingReportDiscarded(SessionIndex),
-		/// A new set of stakers was elected with the given [compute].
+		/// A new set of stakers was elected with the given \[compute\].
 		StakingElection(ElectionCompute),
-		/// A new solution for the upcoming election has been stored. [compute]
+		/// A new solution for the upcoming election has been stored. \[compute\]
 		SolutionStored(ElectionCompute),
-		/// An account has bonded this amount. [stash, amount]
+		/// An account has bonded this amount. \[stash, amount\]
 		///
 		/// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably,
 		/// it will not be emitted for staking rewards when they are added to stake.
 		Bonded(AccountId, Balance),
-		/// An account has unbonded this amount. [stash, amount]
+		/// An account has unbonded this amount. \[stash, amount\]
 		Unbonded(AccountId, Balance),
 		/// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance`
-		/// from the unlocking queue. [stash, amount]
+		/// from the unlocking queue. \[stash, amount\]
 		Withdrawn(AccountId, Balance),
 	}
 );
diff --git a/frame/sudo/src/lib.rs b/frame/sudo/src/lib.rs
index 113fa0dccc6c7..83e73d2ce4349 100644
--- a/frame/sudo/src/lib.rs
+++ b/frame/sudo/src/lib.rs
@@ -225,11 +225,11 @@ decl_module! {
 
 decl_event!(
 	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId {
-		/// A sudo just took place. [result]
+		/// A sudo just took place. \[result\]
 		Sudid(DispatchResult),
-		/// The [sudoer] just switched identity; the old key is supplied.
+		/// The \[sudoer\] just switched identity; the old key is supplied.
 		KeyChanged(AccountId),
-		/// A sudo just took place. [result]
+		/// A sudo just took place. \[result\]
 		SudoAsDone(bool),
 	}
 );
diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs
index 85599626ec2cc..181a1597a0451 100644
--- a/frame/support/src/dispatch.rs
+++ b/frame/support/src/dispatch.rs
@@ -106,7 +106,7 @@ impl<T> Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {}
 /// ### Shorthand Example
 ///
 /// The macro automatically expands a shorthand function declaration to return the
-/// [`DispatchResult`](dispatch::DispatchResult) type. These functions are the same:
+/// [`DispatchResult`] type. These functions are the same:
 ///
 /// ```
 /// # #[macro_use]
diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs
index d2c7e25676739..93dea5f473075 100644
--- a/frame/system/src/lib.rs
+++ b/frame/system/src/lib.rs
@@ -483,15 +483,15 @@ decl_storage! {
 decl_event!(
 	/// Event for the System module.
 	pub enum Event<T> where AccountId = <T as Trait>::AccountId {
-		/// An extrinsic completed successfully. [info]
+		/// An extrinsic completed successfully. \[info\]
 		ExtrinsicSuccess(DispatchInfo),
-		/// An extrinsic failed. [error, info]
+		/// An extrinsic failed. \[error, info\]
 		ExtrinsicFailed(DispatchError, DispatchInfo),
 		/// `:code` was updated.
 		CodeUpdated,
-		/// A new [account] was created.
+		/// A new \[account\] was created.
 		NewAccount(AccountId),
-		/// An [account] was reaped.
+		/// An \[account\] was reaped.
 		KilledAccount(AccountId),
 	}
 );
diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs
index af8d4a3cd0c2b..c99a845e29cb3 100644
--- a/frame/treasury/src/lib.rs
+++ b/frame/treasury/src/lib.rs
@@ -277,27 +277,28 @@ decl_event!(
 		<T as frame_system::Trait>::AccountId,
 		<T as frame_system::Trait>::Hash,
 	{
-		/// New proposal. [proposal_index]
+		/// New proposal. \[proposal_index\]
 		Proposed(ProposalIndex),
-		/// We have ended a spend period and will now allocate funds. [budget_remaining]
+		/// We have ended a spend period and will now allocate funds. \[budget_remaining\]
 		Spending(Balance),
-		/// Some funds have been allocated. [proposal_index, award, beneficiary]
+		/// Some funds have been allocated. \[proposal_index, award, beneficiary\]
 		Awarded(ProposalIndex, Balance, AccountId),
-		/// A proposal was rejected; funds were slashed. [proposal_index, slashed]
+		/// A proposal was rejected; funds were slashed. \[proposal_index, slashed\]
 		Rejected(ProposalIndex, Balance),
-		/// Some of our funds have been burnt. [burn]
+		/// Some of our funds have been burnt. \[burn\]
 		Burnt(Balance),
-		/// Spending has finished; this is the amount that rolls over until next spend. [budget_remaining]
+		/// Spending has finished; this is the amount that rolls over until next spend.
+		/// \[budget_remaining\]
 		Rollover(Balance),
-		/// Some funds have been deposited. [deposit]
+		/// Some funds have been deposited. \[deposit\]
 		Deposit(Balance),
-		/// A new tip suggestion has been opened. [tip_hash]
+		/// A new tip suggestion has been opened. \[tip_hash\]
 		NewTip(Hash),
-		/// A tip suggestion has reached threshold and is closing. [tip_hash]
+		/// A tip suggestion has reached threshold and is closing. \[tip_hash\]
 		TipClosing(Hash),
-		/// A tip suggestion has been closed. [tip_hash, who, payout]
+		/// A tip suggestion has been closed. \[tip_hash, who, payout\]
 		TipClosed(Hash, AccountId, Balance),
-		/// A tip suggestion has been retracted. [tip_hash]
+		/// A tip suggestion has been retracted. \[tip_hash\]
 		TipRetracted(Hash),
 	}
 );
diff --git a/frame/utility/src/lib.rs b/frame/utility/src/lib.rs
index d67fdc85db5a5..c39526ac0a7df 100644
--- a/frame/utility/src/lib.rs
+++ b/frame/utility/src/lib.rs
@@ -98,7 +98,7 @@ decl_event! {
 	/// Events type.
 	pub enum Event {
 		/// Batch of dispatches did not complete fully. Index of first failing dispatch given, as
-		/// well as the error. [index, error]
+		/// well as the error. \[index, error\]
 		BatchInterrupted(u32, DispatchError),
 		/// Batch of dispatches completed fully with no error.
 		BatchCompleted,
diff --git a/frame/vesting/src/lib.rs b/frame/vesting/src/lib.rs
index c521af1e03c59..2fe8e033bb25e 100644
--- a/frame/vesting/src/lib.rs
+++ b/frame/vesting/src/lib.rs
@@ -172,9 +172,9 @@ decl_event!(
 	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId, Balance = BalanceOf<T> {
 		/// The amount vested has been updated. This could indicate more funds are available. The
 		/// balance given is the amount which is left unvested (and thus locked). 
-		/// [account, unvested]
+		/// \[account, unvested\]
 		VestingUpdated(AccountId, Balance),
-		/// An [account] has become fully vested. No further vesting can happen.
+		/// An \[account\] has become fully vested. No further vesting can happen.
 		VestingCompleted(AccountId),
 	}
 );

From 4b471dd836c9b284754c83db0b96359459ce511d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Sat, 12 Sep 2020 12:17:26 +0200
Subject: [PATCH 075/122] Fix `storage::read` (#7084)

* Fix `storage::read`

It should return the length of the storage item after the given offset.
Before it returned always the length of the full storage item.

* Fix tests
---
 primitives/io/src/lib.rs      | 11 ++++++-----
 test-utils/runtime/src/lib.rs | 12 ++++--------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs
index 3248efaa17e50..9c4a0c59b51c2 100644
--- a/primitives/io/src/lib.rs
+++ b/primitives/io/src/lib.rs
@@ -94,7 +94,7 @@ pub trait Storage {
 			let data = &value[value_offset.min(value.len())..];
 			let written = std::cmp::min(data.len(), value_out.len());
 			value_out[..written].copy_from_slice(&data[..written]);
-			value.len() as u32
+			data.len() as u32
 		})
 	}
 
@@ -235,7 +235,7 @@ pub trait DefaultChildStorage {
 				let data = &value[value_offset.min(value.len())..];
 				let written = std::cmp::min(data.len(), value_out.len());
 				value_out[..written].copy_from_slice(&data[..written]);
-				value.len() as u32
+				data.len() as u32
 			})
 	}
 
@@ -1243,17 +1243,18 @@ mod tests {
 
 	#[test]
 	fn read_storage_works() {
+		let value = b"\x0b\0\0\0Hello world".to_vec();
 		let mut t = BasicExternalities::new(Storage {
-			top: map![b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec()],
+			top: map![b":test".to_vec() => value.clone()],
 			children_default: map![],
 		});
 
 		t.execute_with(|| {
 			let mut v = [0u8; 4];
-			assert!(storage::read(b":test", &mut v[..], 0).unwrap() >= 4);
+			assert_eq!(storage::read(b":test", &mut v[..], 0).unwrap(), value.len() as u32);
 			assert_eq!(v, [11u8, 0, 0, 0]);
 			let mut w = [0u8; 11];
-			assert!(storage::read(b":test", &mut w[..], 4).unwrap() >= 11);
+			assert_eq!(storage::read(b":test", &mut w[..], 4).unwrap(), value.len() as u32 - 4);
 			assert_eq!(&w, b"Hello world");
 		});
 	}
diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs
index a7c1c261b5541..a67d2455be18d 100644
--- a/test-utils/runtime/src/lib.rs
+++ b/test-utils/runtime/src/lib.rs
@@ -1067,17 +1067,13 @@ fn test_read_storage() {
 	sp_io::storage::set(KEY, b"test");
 
 	let mut v = [0u8; 4];
-	let r = sp_io::storage::read(
-		KEY,
-		&mut v,
-		0
-	);
+	let r = sp_io::storage::read(KEY, &mut v, 0);
 	assert_eq!(r, Some(4));
 	assert_eq!(&v, b"test");
 
 	let mut v = [0u8; 4];
-	let r = sp_io::storage::read(KEY, &mut v, 8);
-	assert_eq!(r, Some(4));
+	let r = sp_io::storage::read(KEY, &mut v, 4);
+	assert_eq!(r, Some(0));
 	assert_eq!(&v, &[0, 0, 0, 0]);
 }
 
@@ -1107,7 +1103,7 @@ fn test_read_child_storage() {
 		&mut v,
 		8,
 	);
-	assert_eq!(r, Some(4));
+	assert_eq!(r, Some(0));
 	assert_eq!(&v, &[0, 0, 0, 0]);
 }
 

From 6c89d0749d6146c9337cea3be3cc92ed9b529f34 Mon Sep 17 00:00:00 2001
From: Vincent Ulitzsch <vincent.ulitzsch@live.de>
Date: Sun, 13 Sep 2020 18:28:10 +0200
Subject: [PATCH 076/122] Add fuzzer for the compact custom codec
 implementation from PR #6720 (#7091)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add fuzzer for the compact custom codec implementation introduced in PR #6720.

This commit adds a fuzzing harness for the custom compact encoding/decoding
introduced in PR #6720.

* Update primitives/npos-elections/fuzzer/src/compact.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update Cargo.lock: Add changes in elections-fuzzer

* Change indentation from spaces to tabs

Co-authored-by: Vincent Ulitzsch <vincent@srlabs.de>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
---
 Cargo.lock                                    |  1 +
 primitives/npos-elections/fuzzer/Cargo.toml   |  5 +++
 .../npos-elections/fuzzer/src/compact.rs      | 34 +++++++++++++++++++
 3 files changed, 40 insertions(+)
 create mode 100644 primitives/npos-elections/fuzzer/src/compact.rs

diff --git a/Cargo.lock b/Cargo.lock
index 741f2ba7c48c4..81fcee59cda9d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8250,6 +8250,7 @@ name = "sp-npos-elections-fuzzer"
 version = "2.0.0-alpha.5"
 dependencies = [
  "honggfuzz",
+ "parity-scale-codec",
  "rand 0.7.3",
  "sp-npos-elections",
  "sp-runtime",
diff --git a/primitives/npos-elections/fuzzer/Cargo.toml b/primitives/npos-elections/fuzzer/Cargo.toml
index f0c9442aade5f..4d262bc50074e 100644
--- a/primitives/npos-elections/fuzzer/Cargo.toml
+++ b/primitives/npos-elections/fuzzer/Cargo.toml
@@ -19,6 +19,7 @@ sp-std = { version = "2.0.0-rc6", path = "../../std" }
 sp-runtime = { version = "2.0.0-rc6", path = "../../runtime" }
 honggfuzz = "0.5"
 rand = { version = "0.7.3", features = ["std", "small_rng"] }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
 
 [[bin]]
 name = "reduce"
@@ -27,3 +28,7 @@ path = "src/reduce.rs"
 [[bin]]
 name = "balance_solution"
 path = "src/balance_solution.rs"
+
+[[bin]]
+name = "compact"
+path = "src/compact.rs"
diff --git a/primitives/npos-elections/fuzzer/src/compact.rs b/primitives/npos-elections/fuzzer/src/compact.rs
new file mode 100644
index 0000000000000..91f734bb5b7cb
--- /dev/null
+++ b/primitives/npos-elections/fuzzer/src/compact.rs
@@ -0,0 +1,34 @@
+use honggfuzz::fuzz;
+use sp_npos_elections::generate_solution_type;
+use sp_npos_elections::sp_arithmetic::Percent;
+use sp_runtime::codec::{Encode, Error};
+
+fn main() {
+	generate_solution_type!(#[compact] pub struct InnerTestSolutionCompact::<u32, u32, Percent>(16));
+	loop {
+		fuzz!(|fuzzer_data: &[u8]| {
+			let result_decoded: Result<InnerTestSolutionCompact, Error> =
+				<InnerTestSolutionCompact as codec::Decode>::decode(&mut &fuzzer_data[..]);
+			// Ignore errors as not every random sequence of bytes can be decoded as InnerTestSolutionCompact
+			if let Ok(decoded) = result_decoded {
+				// Decoding works, let's re-encode it and compare results.
+				let reencoded: std::vec::Vec<u8> = decoded.encode();
+				// The reencoded value may or may not be equal to the original fuzzer output. However, the
+				// original decoder should be optimal (in the sense that there is no shorter encoding of
+				// the same object). So let's see if the fuzzer can find something shorter:
+				if fuzzer_data.len() < reencoded.len() {
+					panic!("fuzzer_data.len() < reencoded.len()");
+				}
+				// The reencoded value should definitely be decodable (if unwrap() fails that is a valid
+				// panic/finding for the fuzzer):
+				let decoded2: InnerTestSolutionCompact =
+					<InnerTestSolutionCompact as codec::Decode>::decode(
+						&mut reencoded.as_slice(),
+					).unwrap();
+				// And it should be equal to the original decoded object (resulting from directly
+				// decoding fuzzer_data):
+				assert_eq!(decoded, decoded2);
+			}
+		});
+	}
+}

From 8b82790ddd09b19b7b13951ef6d4ecfa64ea07a8 Mon Sep 17 00:00:00 2001
From: Kerwin Zhu <pfcoder97@gmail.com>
Date: Mon, 14 Sep 2020 16:59:01 +0800
Subject: [PATCH 077/122] add instantiable support for treasury pallet (#7058)

* add instantiable support for treasury pallet

* update treasury pallet benchmarking code to support multi-instance

* use benchmark_intance! macro; fix hard coded treasury identity string; fix over characters line width limitation error

* fix line return style
---
 frame/treasury/src/benchmarking.rs |  82 ++++++++---------
 frame/treasury/src/lib.rs          | 138 +++++++++++++++--------------
 frame/treasury/src/tests.rs        |  42 +++++----
 3 files changed, 141 insertions(+), 121 deletions(-)

diff --git a/frame/treasury/src/benchmarking.rs b/frame/treasury/src/benchmarking.rs
index 295326e1639ad..a972dc80bd428 100644
--- a/frame/treasury/src/benchmarking.rs
+++ b/frame/treasury/src/benchmarking.rs
@@ -22,7 +22,7 @@
 use super::*;
 
 use frame_system::RawOrigin;
-use frame_benchmarking::{benchmarks, account, whitelisted_caller};
+use frame_benchmarking::{benchmarks_instance, account, whitelisted_caller};
 use frame_support::traits::OnInitialize;
 
 use crate::Module as Treasury;
@@ -30,13 +30,13 @@ use crate::Module as Treasury;
 const SEED: u32 = 0;
 
 // Create the pre-requisite information needed to create a treasury `propose_spend`.
-fn setup_proposal<T: Trait>(u: u32) -> (
+fn setup_proposal<T: Trait<I>, I: Instance>(u: u32) -> (
 	T::AccountId,
-	BalanceOf<T>,
+	BalanceOf<T, I>,
 	<T::Lookup as StaticLookup>::Source,
 ) {
 	let caller = account("caller", u, SEED);
-	let value: BalanceOf<T> = T::ProposalBondMinimum::get().saturating_mul(100.into());
+	let value: BalanceOf<T, I> = T::ProposalBondMinimum::get().saturating_mul(100.into());
 	let _ = T::Currency::make_free_balance_be(&caller, value);
 	let beneficiary = account("beneficiary", u, SEED);
 	let beneficiary_lookup = T::Lookup::unlookup(beneficiary);
@@ -44,7 +44,7 @@ fn setup_proposal<T: Trait>(u: u32) -> (
 }
 
 // Create the pre-requisite information needed to create a `report_awesome`.
-fn setup_awesome<T: Trait>(length: u32) -> (T::AccountId, Vec<u8>, T::AccountId) {
+fn setup_awesome<T: Trait<I>, I: Instance>(length: u32) -> (T::AccountId, Vec<u8>, T::AccountId) {
 	let caller = whitelisted_caller();
 	let value = T::TipReportDepositBase::get()
 		+ T::TipReportDepositPerByte::get() * length.into()
@@ -56,8 +56,8 @@ fn setup_awesome<T: Trait>(length: u32) -> (T::AccountId, Vec<u8>, T::AccountId)
 }
 
 // Create the pre-requisite information needed to call `tip_new`.
-fn setup_tip<T: Trait>(r: u32, t: u32) ->
-	Result<(T::AccountId, Vec<u8>, T::AccountId, BalanceOf<T>), &'static str>
+fn setup_tip<T: Trait<I>, I: Instance>(r: u32, t: u32) ->
+	Result<(T::AccountId, Vec<u8>, T::AccountId, BalanceOf<T, I>), &'static str>
 {
 	let tippers_count = T::Tippers::count();
 
@@ -77,13 +77,15 @@ fn setup_tip<T: Trait>(r: u32, t: u32) ->
 
 // Create `t` new tips for the tip proposal with `hash`.
 // This function automatically makes the tip able to close.
-fn create_tips<T: Trait>(t: u32, hash: T::Hash, value: BalanceOf<T>) -> Result<(), &'static str> {
+fn create_tips<T: Trait<I>, I: Instance>(t: u32, hash: T::Hash, value: BalanceOf<T, I>) ->
+	Result<(), &'static str>
+{
 	for i in 0 .. t {
 		let caller = account("member", i, SEED);
 		ensure!(T::Tippers::contains(&caller), "caller is not a tipper");
-		Treasury::<T>::tip(RawOrigin::Signed(caller).into(), hash, value)?;
+		Treasury::<T, I>::tip(RawOrigin::Signed(caller).into(), hash, value)?;
 	}
-	Tips::<T>::mutate(hash, |maybe_tip| {
+	Tips::<T, I>::mutate(hash, |maybe_tip| {
 		if let Some(open_tip) = maybe_tip {
 			open_tip.closes = Some(T::BlockNumber::zero());
 		}
@@ -92,30 +94,30 @@ fn create_tips<T: Trait>(t: u32, hash: T::Hash, value: BalanceOf<T>) -> Result<(
 }
 
 // Create proposals that are approved for use in `on_initialize`.
-fn create_approved_proposals<T: Trait>(n: u32) -> Result<(), &'static str> {
+fn create_approved_proposals<T: Trait<I>, I: Instance>(n: u32) -> Result<(), &'static str> {
 	for i in 0 .. n {
-		let (caller, value, lookup) = setup_proposal::<T>(i);
-		Treasury::<T>::propose_spend(
+		let (caller, value, lookup) = setup_proposal::<T, I>(i);
+		Treasury::<T, I>::propose_spend(
 			RawOrigin::Signed(caller).into(),
 			value,
 			lookup
 		)?;
-		let proposal_id = ProposalCount::get() - 1;
-		Treasury::<T>::approve_proposal(RawOrigin::Root.into(), proposal_id)?;
+		let proposal_id = <ProposalCount<I>>::get() - 1;
+		Treasury::<T, I>::approve_proposal(RawOrigin::Root.into(), proposal_id)?;
 	}
-	ensure!(Approvals::get().len() == n as usize, "Not all approved");
+	ensure!(<Approvals<I>>::get().len() == n as usize, "Not all approved");
 	Ok(())
 }
 
 const MAX_BYTES: u32 = 16384;
 const MAX_TIPPERS: u32 = 100;
 
-benchmarks! {
+benchmarks_instance! {
 	_ { }
 
 	propose_spend {
 		let u in 0 .. 1000;
-		let (caller, value, beneficiary_lookup) = setup_proposal::<T>(u);
+		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(u);
 		// Whitelist caller account from further DB operations.
 		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
 		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
@@ -123,29 +125,29 @@ benchmarks! {
 
 	reject_proposal {
 		let u in 0 .. 1000;
-		let (caller, value, beneficiary_lookup) = setup_proposal::<T>(u);
-		Treasury::<T>::propose_spend(
+		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(u);
+		Treasury::<T, _>::propose_spend(
 			RawOrigin::Signed(caller).into(),
 			value,
 			beneficiary_lookup
 		)?;
-		let proposal_id = ProposalCount::get() - 1;
+		let proposal_id = Treasury::<T, _>::proposal_count() - 1;
 	}: _(RawOrigin::Root, proposal_id)
 
 	approve_proposal {
 		let u in 0 .. 1000;
-		let (caller, value, beneficiary_lookup) = setup_proposal::<T>(u);
-		Treasury::<T>::propose_spend(
+		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(u);
+		Treasury::<T, _>::propose_spend(
 			RawOrigin::Signed(caller).into(),
 			value,
 			beneficiary_lookup
 		)?;
-		let proposal_id = ProposalCount::get() - 1;
+		let proposal_id = Treasury::<T, _>::proposal_count() - 1;
 	}: _(RawOrigin::Root, proposal_id)
 
 	report_awesome {
 		let r in 0 .. MAX_BYTES;
-		let (caller, reason, awesome_person) = setup_awesome::<T>(r);
+		let (caller, reason, awesome_person) = setup_awesome::<T, _>(r);
 		// Whitelist caller account from further DB operations.
 		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
 		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
@@ -153,8 +155,8 @@ benchmarks! {
 
 	retract_tip {
 		let r in 0 .. MAX_BYTES;
-		let (caller, reason, awesome_person) = setup_awesome::<T>(r);
-		Treasury::<T>::report_awesome(
+		let (caller, reason, awesome_person) = setup_awesome::<T, _>(r);
+		Treasury::<T, _>::report_awesome(
 			RawOrigin::Signed(caller.clone()).into(),
 			reason.clone(),
 			awesome_person.clone()
@@ -170,7 +172,7 @@ benchmarks! {
 		let r in 0 .. MAX_BYTES;
 		let t in 1 .. MAX_TIPPERS;
 
-		let (caller, reason, beneficiary, value) = setup_tip::<T>(r, t)?;
+		let (caller, reason, beneficiary, value) = setup_tip::<T, _>(r, t)?;
 		// Whitelist caller account from further DB operations.
 		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
 		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
@@ -178,9 +180,9 @@ benchmarks! {
 
 	tip {
 		let t in 1 .. MAX_TIPPERS;
-		let (member, reason, beneficiary, value) = setup_tip::<T>(0, t)?;
+		let (member, reason, beneficiary, value) = setup_tip::<T, _>(0, t)?;
 		let value = T::Currency::minimum_balance().saturating_mul(100.into());
-		Treasury::<T>::tip_new(
+		Treasury::<T, _>::tip_new(
 			RawOrigin::Signed(member).into(),
 			reason.clone(),
 			beneficiary.clone(),
@@ -188,8 +190,8 @@ benchmarks! {
 		)?;
 		let reason_hash = T::Hashing::hash(&reason[..]);
 		let hash = T::Hashing::hash_of(&(&reason_hash, &beneficiary));
-		ensure!(Tips::<T>::contains_key(hash), "tip does not exist");
-		create_tips::<T>(t - 1, hash.clone(), value)?;
+		ensure!(Tips::<T, _>::contains_key(hash), "tip does not exist");
+		create_tips::<T, _>(t - 1, hash.clone(), value)?;
 		let caller = account("member", t - 1, SEED);
 		// Whitelist caller account from further DB operations.
 		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
@@ -200,14 +202,14 @@ benchmarks! {
 		let t in 1 .. MAX_TIPPERS;
 
 		// Make sure pot is funded
-		let pot_account = Treasury::<T>::account_id();
+		let pot_account = Treasury::<T, _>::account_id();
 		let value = T::Currency::minimum_balance().saturating_mul(1_000_000_000.into());
 		let _ = T::Currency::make_free_balance_be(&pot_account, value);
 
 		// Set up a new tip proposal
-		let (member, reason, beneficiary, value) = setup_tip::<T>(0, t)?;
+		let (member, reason, beneficiary, value) = setup_tip::<T, _>(0, t)?;
 		let value = T::Currency::minimum_balance().saturating_mul(100.into());
-		Treasury::<T>::tip_new(
+		Treasury::<T, _>::tip_new(
 			RawOrigin::Signed(member).into(),
 			reason.clone(),
 			beneficiary.clone(),
@@ -217,8 +219,8 @@ benchmarks! {
 		// Create a bunch of tips
 		let reason_hash = T::Hashing::hash(&reason[..]);
 		let hash = T::Hashing::hash_of(&(&reason_hash, &beneficiary));
-		ensure!(Tips::<T>::contains_key(hash), "tip does not exist");
-		create_tips::<T>(t, hash.clone(), value)?;
+		ensure!(Tips::<T, _>::contains_key(hash), "tip does not exist");
+		create_tips::<T, _>(t, hash.clone(), value)?;
 
 		let caller = account("caller", t, SEED);
 		// Whitelist caller account from further DB operations.
@@ -228,12 +230,12 @@ benchmarks! {
 
 	on_initialize {
 		let p in 0 .. 100;
-		let pot_account = Treasury::<T>::account_id();
+		let pot_account = Treasury::<T, _>::account_id();
 		let value = T::Currency::minimum_balance().saturating_mul(1_000_000_000.into());
 		let _ = T::Currency::make_free_balance_be(&pot_account, value);
-		create_approved_proposals::<T>(p)?;
+		create_approved_proposals::<T, _>(p)?;
 	}: {
-		Treasury::<T>::on_initialize(T::BlockNumber::zero());
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
 	}
 }
 
diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs
index c99a845e29cb3..cedc46eb8c0dd 100644
--- a/frame/treasury/src/lib.rs
+++ b/frame/treasury/src/lib.rs
@@ -107,9 +107,12 @@ use frame_system::{self as system, ensure_signed};
 mod tests;
 mod benchmarking;
 
-type BalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
-type PositiveImbalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::PositiveImbalance;
-type NegativeImbalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::NegativeImbalance;
+type BalanceOf<T, I> =
+	<<T as Trait<I>>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
+type PositiveImbalanceOf<T, I> =
+	<<T as Trait<I>>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::PositiveImbalance;
+type NegativeImbalanceOf<T, I> =
+	<<T as Trait<I>>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::NegativeImbalance;
 
 pub trait WeightInfo {
 	fn propose_spend(u: u32, ) -> Weight;
@@ -135,7 +138,7 @@ impl WeightInfo for () {
 	fn on_initialize(_p: u32, ) -> Weight { 1_000_000_000 }
 }
 
-pub trait Trait: frame_system::Trait {
+pub trait Trait<I=DefaultInstance>: frame_system::Trait {
 	/// The treasury's module id, used for deriving its sovereign account ID.
 	type ModuleId: Get<ModuleId>;
 
@@ -160,23 +163,23 @@ pub trait Trait: frame_system::Trait {
 	type TipFindersFee: Get<Percent>;
 
 	/// The amount held on deposit for placing a tip report.
-	type TipReportDepositBase: Get<BalanceOf<Self>>;
+	type TipReportDepositBase: Get<BalanceOf<Self, I>>;
 
 	/// The amount held on deposit per byte within the tip report reason.
-	type TipReportDepositPerByte: Get<BalanceOf<Self>>;
+	type TipReportDepositPerByte: Get<BalanceOf<Self, I>>;
 
 	/// The overarching event type.
-	type Event: From<Event<Self>> + Into<<Self as frame_system::Trait>::Event>;
+	type Event: From<Event<Self, I>> + Into<<Self as frame_system::Trait>::Event>;
 
 	/// Handler for the unbalanced decrease when slashing for a rejected proposal.
-	type ProposalRejection: OnUnbalanced<NegativeImbalanceOf<Self>>;
+	type ProposalRejection: OnUnbalanced<NegativeImbalanceOf<Self, I>>;
 
 	/// Fraction of a proposal's value that should be bonded in order to place the proposal.
 	/// An accepted proposal gets these back. A rejected proposal does not.
 	type ProposalBond: Get<Permill>;
 
 	/// Minimum amount of funds that should be placed in a deposit for making a proposal.
-	type ProposalBondMinimum: Get<BalanceOf<Self>>;
+	type ProposalBondMinimum: Get<BalanceOf<Self, I>>;
 
 	/// Period between successive spends.
 	type SpendPeriod: Get<Self::BlockNumber>;
@@ -185,7 +188,7 @@ pub trait Trait: frame_system::Trait {
 	type Burn: Get<Permill>;
 
 	/// Handler for the unbalanced decrease when treasury funds are burned.
-	type BurnDestination: OnUnbalanced<NegativeImbalanceOf<Self>>;
+	type BurnDestination: OnUnbalanced<NegativeImbalanceOf<Self, I>>;
 
 	/// Weight information for extrinsics in this pallet.
 	type WeightInfo: WeightInfo;
@@ -236,14 +239,14 @@ pub struct OpenTip<
 }
 
 decl_storage! {
-	trait Store for Module<T: Trait> as Treasury {
+	trait Store for Module<T: Trait<I>, I: Instance=DefaultInstance> as Treasury {
 		/// Number of proposals that have been made.
 		ProposalCount get(fn proposal_count): ProposalIndex;
 
 		/// Proposals that have been made.
 		Proposals get(fn proposals):
 			map hasher(twox_64_concat) ProposalIndex
-			=> Option<Proposal<T::AccountId, BalanceOf<T>>>;
+			=> Option<Proposal<T::AccountId, BalanceOf<T, I>>>;
 
 		/// Proposal indices that have been approved but not yet awarded.
 		Approvals get(fn approvals): Vec<ProposalIndex>;
@@ -253,7 +256,7 @@ decl_storage! {
 		/// guaranteed to be a secure hash.
 		pub Tips get(fn tips):
 			map hasher(twox_64_concat) T::Hash
-			=> Option<OpenTip<T::AccountId, BalanceOf<T>, T::BlockNumber, T::Hash>>;
+			=> Option<OpenTip<T::AccountId, BalanceOf<T, I>, T::BlockNumber, T::Hash>>;
 
 		/// Simple preimage lookup from the reason's hash to the original data. Again, has an
 		/// insecure enumerable hash since the key is guaranteed to be the result of a secure hash.
@@ -263,7 +266,7 @@ decl_storage! {
 		build(|_config| {
 			// Create Treasury account
 			let _ = T::Currency::make_free_balance_be(
-				&<Module<T>>::account_id(),
+				&<Module<T, I>>::account_id(),
 				T::Currency::minimum_balance(),
 			);
 		});
@@ -271,9 +274,9 @@ decl_storage! {
 }
 
 decl_event!(
-	pub enum Event<T>
+	pub enum Event<T, I=DefaultInstance>
 	where
-		Balance = BalanceOf<T>,
+		Balance = BalanceOf<T, I>,
 		<T as frame_system::Trait>::AccountId,
 		<T as frame_system::Trait>::Hash,
 	{
@@ -305,7 +308,7 @@ decl_event!(
 
 decl_error! {
 	/// Error for the treasury module.
-	pub enum Error for Module<T: Trait> {
+	pub enum Error for Module<T: Trait<I>, I: Instance> {
 		/// Proposer's balance is too low.
 		InsufficientProposersBalance,
 		/// No proposal at that index.
@@ -326,13 +329,16 @@ decl_error! {
 }
 
 decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+	pub struct Module<T: Trait<I>, I: Instance=DefaultInstance>
+		for enum Call
+		where origin: T::Origin
+	{
 		/// Fraction of a proposal's value that should be bonded in order to place the proposal.
 		/// An accepted proposal gets these back. A rejected proposal does not.
 		const ProposalBond: Permill = T::ProposalBond::get();
 
 		/// Minimum amount of funds that should be placed in a deposit for making a proposal.
-		const ProposalBondMinimum: BalanceOf<T> = T::ProposalBondMinimum::get();
+		const ProposalBondMinimum: BalanceOf<T, I> = T::ProposalBondMinimum::get();
 
 		/// Period between successive spends.
 		const SpendPeriod: T::BlockNumber = T::SpendPeriod::get();
@@ -347,15 +353,15 @@ decl_module! {
 		const TipFindersFee: Percent = T::TipFindersFee::get();
 
 		/// The amount held on deposit for placing a tip report.
-		const TipReportDepositBase: BalanceOf<T> = T::TipReportDepositBase::get();
+		const TipReportDepositBase: BalanceOf<T, I> = T::TipReportDepositBase::get();
 
 		/// The amount held on deposit per byte within the tip report reason.
-		const TipReportDepositPerByte: BalanceOf<T> = T::TipReportDepositPerByte::get();
+		const TipReportDepositPerByte: BalanceOf<T, I> = T::TipReportDepositPerByte::get();
 
 		/// The treasury's module id, used for deriving its sovereign account ID.
 		const ModuleId: ModuleId = T::ModuleId::get();
 
-		type Error = Error<T>;
+		type Error = Error<T, I>;
 
 		fn deposit_event() = default;
 
@@ -371,7 +377,7 @@ decl_module! {
 		#[weight = 120_000_000 + T::DbWeight::get().reads_writes(1, 2)]
 		fn propose_spend(
 			origin,
-			#[compact] value: BalanceOf<T>,
+			#[compact] value: BalanceOf<T, I>,
 			beneficiary: <T::Lookup as StaticLookup>::Source
 		) {
 			let proposer = ensure_signed(origin)?;
@@ -379,11 +385,11 @@ decl_module! {
 
 			let bond = Self::calculate_bond(value);
 			T::Currency::reserve(&proposer, bond)
-				.map_err(|_| Error::<T>::InsufficientProposersBalance)?;
+				.map_err(|_| Error::<T, I>::InsufficientProposersBalance)?;
 
 			let c = Self::proposal_count();
-			ProposalCount::put(c + 1);
-			<Proposals<T>>::insert(c, Proposal { proposer, value, beneficiary, bond });
+			<ProposalCount<I>>::put(c + 1);
+			<Proposals<T, I>>::insert(c, Proposal { proposer, value, beneficiary, bond });
 
 			Self::deposit_event(RawEvent::Proposed(c));
 		}
@@ -401,12 +407,12 @@ decl_module! {
 		fn reject_proposal(origin, #[compact] proposal_id: ProposalIndex) {
 			T::RejectOrigin::ensure_origin(origin)?;
 
-			let proposal = <Proposals<T>>::take(&proposal_id).ok_or(Error::<T>::InvalidProposalIndex)?;
+			let proposal = <Proposals<T, I>>::take(&proposal_id).ok_or(Error::<T, I>::InvalidProposalIndex)?;
 			let value = proposal.bond;
 			let imbalance = T::Currency::slash_reserved(&proposal.proposer, value).0;
 			T::ProposalRejection::on_unbalanced(imbalance);
 
-			Self::deposit_event(Event::<T>::Rejected(proposal_id, value));
+			Self::deposit_event(Event::<T, I>::Rejected(proposal_id, value));
 		}
 
 		/// Approve a proposal. At a later time, the proposal will be allocated to the beneficiary
@@ -423,8 +429,8 @@ decl_module! {
 		fn approve_proposal(origin, #[compact] proposal_id: ProposalIndex) {
 			T::ApproveOrigin::ensure_origin(origin)?;
 
-			ensure!(<Proposals<T>>::contains_key(proposal_id), Error::<T>::InvalidProposalIndex);
-			Approvals::mutate(|v| v.push(proposal_id));
+			ensure!(<Proposals<T, I>>::contains_key(proposal_id), Error::<T, I>::InvalidProposalIndex);
+			<Approvals<I>>::mutate(|v| v.push(proposal_id));
 		}
 
 		/// Report something `reason` that deserves a tip and claim any eventual the finder's fee.
@@ -451,18 +457,18 @@ decl_module! {
 			let finder = ensure_signed(origin)?;
 
 			const MAX_SENSIBLE_REASON_LENGTH: usize = 16384;
-			ensure!(reason.len() <= MAX_SENSIBLE_REASON_LENGTH, Error::<T>::ReasonTooBig);
+			ensure!(reason.len() <= MAX_SENSIBLE_REASON_LENGTH, Error::<T, I>::ReasonTooBig);
 
 			let reason_hash = T::Hashing::hash(&reason[..]);
-			ensure!(!Reasons::<T>::contains_key(&reason_hash), Error::<T>::AlreadyKnown);
+			ensure!(!Reasons::<T, I>::contains_key(&reason_hash), Error::<T, I>::AlreadyKnown);
 			let hash = T::Hashing::hash_of(&(&reason_hash, &who));
-			ensure!(!Tips::<T>::contains_key(&hash), Error::<T>::AlreadyKnown);
+			ensure!(!Tips::<T, I>::contains_key(&hash), Error::<T, I>::AlreadyKnown);
 
 			let deposit = T::TipReportDepositBase::get()
 				+ T::TipReportDepositPerByte::get() * (reason.len() as u32).into();
 			T::Currency::reserve(&finder, deposit)?;
 
-			Reasons::<T>::insert(&reason_hash, &reason);
+			Reasons::<T, I>::insert(&reason_hash, &reason);
 			let tip = OpenTip {
 				reason: reason_hash,
 				who,
@@ -472,7 +478,7 @@ decl_module! {
 				tips: vec![],
 				finders_fee: true
 			};
-			Tips::<T>::insert(&hash, tip);
+			Tips::<T, I>::insert(&hash, tip);
 			Self::deposit_event(RawEvent::NewTip(hash));
 		}
 
@@ -498,11 +504,11 @@ decl_module! {
 		#[weight = 120_000_000 + T::DbWeight::get().reads_writes(1, 2)]
 		fn retract_tip(origin, hash: T::Hash) {
 			let who = ensure_signed(origin)?;
-			let tip = Tips::<T>::get(&hash).ok_or(Error::<T>::UnknownTip)?;
-			ensure!(tip.finder == who, Error::<T>::NotFinder);
+			let tip = Tips::<T, I>::get(&hash).ok_or(Error::<T, I>::UnknownTip)?;
+			ensure!(tip.finder == who, Error::<T, I>::NotFinder);
 
-			Reasons::<T>::remove(&tip.reason);
-			Tips::<T>::remove(&hash);
+			Reasons::<T, I>::remove(&tip.reason);
+			Tips::<T, I>::remove(&hash);
 			if !tip.deposit.is_zero() {
 				let _ = T::Currency::unreserve(&who, tip.deposit);
 			}
@@ -535,14 +541,14 @@ decl_module! {
 			+ 4_000 * reason.len() as Weight
 			+ 480_000 * T::Tippers::max_len() as Weight
 			+ T::DbWeight::get().reads_writes(2, 2)]
-		fn tip_new(origin, reason: Vec<u8>, who: T::AccountId, tip_value: BalanceOf<T>) {
+		fn tip_new(origin, reason: Vec<u8>, who: T::AccountId, tip_value: BalanceOf<T, I>) {
 			let tipper = ensure_signed(origin)?;
 			ensure!(T::Tippers::contains(&tipper), BadOrigin);
 			let reason_hash = T::Hashing::hash(&reason[..]);
-			ensure!(!Reasons::<T>::contains_key(&reason_hash), Error::<T>::AlreadyKnown);
+			ensure!(!Reasons::<T, I>::contains_key(&reason_hash), Error::<T, I>::AlreadyKnown);
 			let hash = T::Hashing::hash_of(&(&reason_hash, &who));
 
-			Reasons::<T>::insert(&reason_hash, &reason);
+			Reasons::<T, I>::insert(&reason_hash, &reason);
 			Self::deposit_event(RawEvent::NewTip(hash.clone()));
 			let tips = vec![(tipper.clone(), tip_value)];
 			let tip = OpenTip {
@@ -554,7 +560,7 @@ decl_module! {
 				tips,
 				finders_fee: false,
 			};
-			Tips::<T>::insert(&hash, tip);
+			Tips::<T, I>::insert(&hash, tip);
 		}
 
 		/// Declare a tip value for an already-open tip.
@@ -584,15 +590,15 @@ decl_module! {
 		/// # </weight>
 		#[weight = 68_000_000 + 2_000_000 * T::Tippers::max_len() as Weight
 			+ T::DbWeight::get().reads_writes(2, 1)]
-		fn tip(origin, hash: T::Hash, tip_value: BalanceOf<T>) {
+		fn tip(origin, hash: T::Hash, tip_value: BalanceOf<T, I>) {
 			let tipper = ensure_signed(origin)?;
 			ensure!(T::Tippers::contains(&tipper), BadOrigin);
 
-			let mut tip = Tips::<T>::get(hash).ok_or(Error::<T>::UnknownTip)?;
+			let mut tip = Tips::<T, I>::get(hash).ok_or(Error::<T, I>::UnknownTip)?;
 			if Self::insert_tip_and_check_closing(&mut tip, tipper, tip_value) {
 				Self::deposit_event(RawEvent::TipClosing(hash.clone()));
 			}
-			Tips::<T>::insert(&hash, tip);
+			Tips::<T, I>::insert(&hash, tip);
 		}
 
 		/// Close and payout a tip.
@@ -617,12 +623,12 @@ decl_module! {
 		fn close_tip(origin, hash: T::Hash) {
 			ensure_signed(origin)?;
 
-			let tip = Tips::<T>::get(hash).ok_or(Error::<T>::UnknownTip)?;
-			let n = tip.closes.as_ref().ok_or(Error::<T>::StillOpen)?;
-			ensure!(system::Module::<T>::block_number() >= *n, Error::<T>::Premature);
+			let tip = Tips::<T, I>::get(hash).ok_or(Error::<T, I>::UnknownTip)?;
+			let n = tip.closes.as_ref().ok_or(Error::<T, I>::StillOpen)?;
+			ensure!(system::Module::<T>::block_number() >= *n, Error::<T, I>::Premature);
 			// closed.
-			Reasons::<T>::remove(&tip.reason);
-			Tips::<T>::remove(hash);
+			Reasons::<T, I>::remove(&tip.reason);
+			Tips::<T, I>::remove(hash);
 			Self::payout_tip(hash, tip);
 		}
 
@@ -647,7 +653,7 @@ decl_module! {
 	}
 }
 
-impl<T: Trait> Module<T> {
+impl<T: Trait<I>, I: Instance> Module<T, I> {
 	// Add public immutables and private mutables.
 
 	/// The account ID of the treasury pot.
@@ -659,7 +665,7 @@ impl<T: Trait> Module<T> {
 	}
 
 	/// The needed bond for a proposal whose spend is `value`.
-	fn calculate_bond(value: BalanceOf<T>) -> BalanceOf<T> {
+	fn calculate_bond(value: BalanceOf<T, I>) -> BalanceOf<T, I> {
 		T::ProposalBondMinimum::get().max(T::ProposalBond::get() * value)
 	}
 
@@ -668,9 +674,9 @@ impl<T: Trait> Module<T> {
 	///
 	/// `O(T)` and one storage access.
 	fn insert_tip_and_check_closing(
-		tip: &mut OpenTip<T::AccountId, BalanceOf<T>, T::BlockNumber, T::Hash>,
+		tip: &mut OpenTip<T::AccountId, BalanceOf<T, I>, T::BlockNumber, T::Hash>,
 		tipper: T::AccountId,
-		tip_value: BalanceOf<T>,
+		tip_value: BalanceOf<T, I>,
 	) -> bool {
 		match tip.tips.binary_search_by_key(&&tipper, |x| &x.0) {
 			Ok(pos) => tip.tips[pos] = (tipper, tip_value),
@@ -687,7 +693,7 @@ impl<T: Trait> Module<T> {
 	}
 
 	/// Remove any non-members of `Tippers` from a `tips` vector. `O(T)`.
-	fn retain_active_tips(tips: &mut Vec<(T::AccountId, BalanceOf<T>)>) {
+	fn retain_active_tips(tips: &mut Vec<(T::AccountId, BalanceOf<T, I>)>) {
 		let members = T::Tippers::sorted_members();
 		let mut members_iter = members.iter();
 		let mut member = members_iter.next();
@@ -711,7 +717,7 @@ impl<T: Trait> Module<T> {
 	///
 	/// Up to three balance operations.
 	/// Plus `O(T)` (`T` is Tippers length).
-	fn payout_tip(hash: T::Hash, tip: OpenTip<T::AccountId, BalanceOf<T>, T::BlockNumber, T::Hash>) {
+	fn payout_tip(hash: T::Hash, tip: OpenTip<T::AccountId, BalanceOf<T, I>, T::BlockNumber, T::Hash>) {
 		let mut tips = tip.tips;
 		Self::retain_active_tips(&mut tips);
 		tips.sort_by_key(|i| i.1);
@@ -742,15 +748,15 @@ impl<T: Trait> Module<T> {
 		Self::deposit_event(RawEvent::Spending(budget_remaining));
 
 		let mut missed_any = false;
-		let mut imbalance = <PositiveImbalanceOf<T>>::zero();
-		let prior_approvals_len = Approvals::mutate(|v| {
+		let mut imbalance = <PositiveImbalanceOf<T, I>>::zero();
+		let prior_approvals_len = <Approvals<I>>::mutate(|v| {
 			let prior_approvals_len = v.len() as u64;
 			v.retain(|&index| {
 				// Should always be true, but shouldn't panic if false or we're screwed.
 				if let Some(p) = Self::proposals(index) {
 					if p.value <= budget_remaining {
 						budget_remaining -= p.value;
-						<Proposals<T>>::remove(index);
+						<Proposals<T, I>>::remove(index);
 
 						// return their deposit.
 						let _ = T::Currency::unreserve(&p.proposer, p.bond);
@@ -804,7 +810,7 @@ impl<T: Trait> Module<T> {
 
 	/// Return the amount of money in the pot.
 	// The existential deposit is not part of the pot so treasury account never gets deleted.
-	fn pot() -> BalanceOf<T> {
+	fn pot() -> BalanceOf<T, I> {
 		T::Currency::free_balance(&Self::account_id())
 			// Must never be less than 0 but better be safe.
 			.saturating_sub(T::Currency::minimum_balance())
@@ -838,9 +844,9 @@ impl<T: Trait> Module<T> {
 
 		for (hash, old_tip) in StorageKeyIterator::<
 			T::Hash,
-			OldOpenTip<T::AccountId, BalanceOf<T>, T::BlockNumber, T::Hash>,
+			OldOpenTip<T::AccountId, BalanceOf<T, I>, T::BlockNumber, T::Hash>,
 			Twox64Concat,
-		>::new(b"Treasury", b"Tips").drain()
+		>::new(I::PREFIX.as_bytes(), b"Tips").drain()
 		{
 			let (finder, deposit, finders_fee) = match old_tip.finder {
 				Some((finder, deposit)) => (finder, deposit, true),
@@ -855,13 +861,13 @@ impl<T: Trait> Module<T> {
 				tips: old_tip.tips,
 				finders_fee
 			};
-			Tips::<T>::insert(hash, new_tip)
+			Tips::<T, I>::insert(hash, new_tip)
 		}
 	}
 }
 
-impl<T: Trait> OnUnbalanced<NegativeImbalanceOf<T>> for Module<T> {
-	fn on_nonzero_unbalanced(amount: NegativeImbalanceOf<T>) {
+impl<T: Trait<I>, I: Instance> OnUnbalanced<NegativeImbalanceOf<T, I>> for Module<T, I> {
+	fn on_nonzero_unbalanced(amount: NegativeImbalanceOf<T, I>) {
 		let numeric_amount = amount.peek();
 
 		// Must resolve into existing but better to be safe.
diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs
index 59a41a263cc9f..f9928c37b36a3 100644
--- a/frame/treasury/src/tests.rs
+++ b/frame/treasury/src/tests.rs
@@ -162,7 +162,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities {
 		// Total issuance will be 200 with treasury account initialized at ED.
 		balances: vec![(0, 100), (1, 98), (2, 1)],
 	}.assimilate_storage(&mut t).unwrap();
-	GenesisConfig::default().assimilate_storage::<Test>(&mut t).unwrap();
+	GenesisConfig::default().assimilate_storage::<Test, _>(&mut t).unwrap();
 	t.into()
 }
 
@@ -185,7 +185,7 @@ fn tip_new_cannot_be_used_twice() {
 		assert_ok!(Treasury::tip_new(Origin::signed(10), b"awesome.dot".to_vec(), 3, 10));
 		assert_noop!(
 			Treasury::tip_new(Origin::signed(11), b"awesome.dot".to_vec(), 3, 10),
-			Error::<Test>::AlreadyKnown
+			Error::<Test, _>::AlreadyKnown
 		);
 	});
 }
@@ -201,7 +201,7 @@ fn report_awesome_and_tip_works() {
 		// other reports don't count.
 		assert_noop!(
 			Treasury::report_awesome(Origin::signed(1), b"awesome.dot".to_vec(), 3),
-			Error::<Test>::AlreadyKnown
+			Error::<Test, _>::AlreadyKnown
 		);
 
 		let h = tip_hash();
@@ -259,7 +259,7 @@ fn close_tip_works() {
 
 		assert_ok!(Treasury::tip(Origin::signed(11), h.clone(), 10));
 
-		assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::<Test>::StillOpen);
+		assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::<Test, _>::StillOpen);
 
 		assert_ok!(Treasury::tip(Origin::signed(12), h.clone(), 10));
 
@@ -273,7 +273,7 @@ fn close_tip_works() {
 			RawEvent::TipClosing(h),
 		);
 
-		assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::<Test>::Premature);
+		assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::<Test, _>::Premature);
 
 		System::set_block_number(2);
 		assert_noop!(Treasury::close_tip(Origin::none(), h.into()), BadOrigin);
@@ -290,7 +290,7 @@ fn close_tip_works() {
 			RawEvent::TipClosed(h, 3, 10),
 		);
 
-		assert_noop!(Treasury::close_tip(Origin::signed(100), h.into()), Error::<Test>::UnknownTip);
+		assert_noop!(Treasury::close_tip(Origin::signed(100), h.into()), Error::<Test, _>::UnknownTip);
 	});
 }
 
@@ -304,10 +304,10 @@ fn retract_tip_works() {
 		assert_ok!(Treasury::tip(Origin::signed(10), h.clone(), 10));
 		assert_ok!(Treasury::tip(Origin::signed(11), h.clone(), 10));
 		assert_ok!(Treasury::tip(Origin::signed(12), h.clone(), 10));
-		assert_noop!(Treasury::retract_tip(Origin::signed(10), h.clone()), Error::<Test>::NotFinder);
+		assert_noop!(Treasury::retract_tip(Origin::signed(10), h.clone()), Error::<Test, _>::NotFinder);
 		assert_ok!(Treasury::retract_tip(Origin::signed(0), h.clone()));
 		System::set_block_number(2);
-		assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::<Test>::UnknownTip);
+		assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::<Test, _>::UnknownTip);
 
 		// with tip new
 		Balances::make_free_balance_be(&Treasury::account_id(), 101);
@@ -315,10 +315,10 @@ fn retract_tip_works() {
 		let h = tip_hash();
 		assert_ok!(Treasury::tip(Origin::signed(11), h.clone(), 10));
 		assert_ok!(Treasury::tip(Origin::signed(12), h.clone(), 10));
-		assert_noop!(Treasury::retract_tip(Origin::signed(0), h.clone()), Error::<Test>::NotFinder);
+		assert_noop!(Treasury::retract_tip(Origin::signed(0), h.clone()), Error::<Test, _>::NotFinder);
 		assert_ok!(Treasury::retract_tip(Origin::signed(10), h.clone()));
 		System::set_block_number(2);
-		assert_noop!(Treasury::close_tip(Origin::signed(10), h.into()), Error::<Test>::UnknownTip);
+		assert_noop!(Treasury::close_tip(Origin::signed(10), h.into()), Error::<Test, _>::UnknownTip);
 	});
 }
 
@@ -387,7 +387,7 @@ fn spend_proposal_fails_when_proposer_poor() {
 	new_test_ext().execute_with(|| {
 		assert_noop!(
 			Treasury::propose_spend(Origin::signed(2), 100, 3),
-			Error::<Test>::InsufficientProposersBalance,
+			Error::<Test, _>::InsufficientProposersBalance,
 		);
 	});
 }
@@ -440,21 +440,30 @@ fn reject_already_rejected_spend_proposal_fails() {
 
 		assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3));
 		assert_ok!(Treasury::reject_proposal(Origin::root(), 0));
-		assert_noop!(Treasury::reject_proposal(Origin::root(), 0), Error::<Test>::InvalidProposalIndex);
+		assert_noop!(
+			Treasury::reject_proposal(Origin::root(), 0),
+			Error::<Test, _>::InvalidProposalIndex,
+		);
 	});
 }
 
 #[test]
 fn reject_non_existent_spend_proposal_fails() {
 	new_test_ext().execute_with(|| {
-		assert_noop!(Treasury::reject_proposal(Origin::root(), 0), Error::<Test>::InvalidProposalIndex);
+		assert_noop!(
+			Treasury::reject_proposal(Origin::root(), 0),
+			Error::<Test, _>::InvalidProposalIndex,
+		);
 	});
 }
 
 #[test]
 fn accept_non_existent_spend_proposal_fails() {
 	new_test_ext().execute_with(|| {
-		assert_noop!(Treasury::approve_proposal(Origin::root(), 0), Error::<Test>::InvalidProposalIndex);
+		assert_noop!(
+			Treasury::approve_proposal(Origin::root(), 0),
+			Error::<Test, _>::InvalidProposalIndex,
+		);
 	});
 }
 
@@ -465,7 +474,10 @@ fn accept_already_rejected_spend_proposal_fails() {
 
 		assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3));
 		assert_ok!(Treasury::reject_proposal(Origin::root(), 0));
-		assert_noop!(Treasury::approve_proposal(Origin::root(), 0), Error::<Test>::InvalidProposalIndex);
+		assert_noop!(
+			Treasury::approve_proposal(Origin::root(), 0),
+			Error::<Test, _>::InvalidProposalIndex,
+		);
 	});
 }
 

From a5977f5b9da56b58b887e289069a045292ed3dbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= <jon.haggblad@gmail.com>
Date: Mon, 14 Sep 2020 13:02:37 +0200
Subject: [PATCH 078/122] grandpa-rpc don't share subscription manager, only
 executor (#7039)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* service builder: fix todo about jsonrpc Option workaround

* grandpa-rpc: only share executor instead of sub manager

* grandpa-rpc: fix compilation

* grandpa-rpc: rename to subscription_executor

* node/cli: remove another unused jsonrpc dependency

* grandpa: apply style fixes from code review

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
---
 Cargo.lock                             |  3 --
 bin/node/cli/Cargo.toml                |  2 --
 bin/node/cli/src/service.rs            |  6 ++--
 bin/node/rpc/Cargo.toml                |  1 -
 bin/node/rpc/src/lib.rs                | 10 +++----
 client/finality-grandpa/rpc/src/lib.rs | 15 ++++++----
 client/service/src/builder.rs          | 40 +++++++++++++++-----------
 7 files changed, 42 insertions(+), 35 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 81fcee59cda9d..9264534efa02b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3731,8 +3731,6 @@ dependencies = [
  "frame-system",
  "futures 0.3.5",
  "hex-literal",
- "jsonrpc-core",
- "jsonrpc-pubsub",
  "log",
  "nix",
  "node-executor",
@@ -3866,7 +3864,6 @@ name = "node-rpc"
 version = "2.0.0-rc6"
 dependencies = [
  "jsonrpc-core",
- "jsonrpc-pubsub",
  "node-primitives",
  "node-runtime",
  "pallet-contracts-rpc",
diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml
index 92f223427a710..fdc63f09555bb 100644
--- a/bin/node/cli/Cargo.toml
+++ b/bin/node/cli/Cargo.toml
@@ -38,8 +38,6 @@ codec = { package = "parity-scale-codec", version = "1.3.4" }
 serde = { version = "1.0.102", features = ["derive"] }
 futures = { version = "0.3.1", features = ["compat"] }
 hex-literal = "0.3.1"
-jsonrpc-core = "14.2.0"
-jsonrpc-pubsub = "14.2.0"
 log = "0.4.8"
 rand = "0.7.2"
 structopt = { version = "0.3.8", optional = true }
diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs
index 51232df9b82b1..03347e455e6a3 100644
--- a/bin/node/cli/src/service.rs
+++ b/bin/node/cli/src/service.rs
@@ -51,7 +51,7 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 	(
 		impl Fn(
 			node_rpc::DenyUnsafe,
-			jsonrpc_pubsub::manager::SubscriptionManager
+			sc_rpc::SubscriptionTaskExecutor,
 		) -> node_rpc::IoHandler,
 		(
 			sc_consensus_babe::BabeBlockImport<Block, FullClient, FullGrandpaBlockImport>,
@@ -119,7 +119,7 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 		let select_chain = select_chain.clone();
 		let keystore = keystore.clone();
 
-		let rpc_extensions_builder = move |deny_unsafe, subscriptions| {
+		let rpc_extensions_builder = move |deny_unsafe, subscription_executor| {
 			let deps = node_rpc::FullDeps {
 				client: client.clone(),
 				pool: pool.clone(),
@@ -134,7 +134,7 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 					shared_voter_state: shared_voter_state.clone(),
 					shared_authority_set: shared_authority_set.clone(),
 					justification_stream: justification_stream.clone(),
-					subscriptions,
+					subscription_executor,
 				},
 			};
 
diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml
index 9ed8c22fbe320..e9abaef35f655 100644
--- a/bin/node/rpc/Cargo.toml
+++ b/bin/node/rpc/Cargo.toml
@@ -12,7 +12,6 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 jsonrpc-core = "14.2.0"
-jsonrpc-pubsub = "14.2.0"
 node-primitives = { version = "2.0.0-rc6", path = "../primitives" }
 node-runtime = { version = "2.0.0-rc6", path = "../runtime" }
 pallet-contracts-rpc = { version = "0.8.0-rc6", path = "../../../frame/contracts/rpc/" }
diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs
index a20fb03ebe1e6..420fd0eb1475b 100644
--- a/bin/node/rpc/src/lib.rs
+++ b/bin/node/rpc/src/lib.rs
@@ -32,7 +32,6 @@
 
 use std::sync::Arc;
 
-use jsonrpc_pubsub::manager::SubscriptionManager;
 use node_primitives::{Block, BlockNumber, AccountId, Index, Balance, Hash};
 use sc_consensus_babe::{Config, Epoch};
 use sc_consensus_babe_rpc::BabeRpcHandler;
@@ -46,6 +45,7 @@ use sp_block_builder::BlockBuilder;
 use sp_blockchain::{Error as BlockChainError, HeaderMetadata, HeaderBackend};
 use sp_consensus::SelectChain;
 use sp_consensus_babe::BabeApi;
+use sc_rpc::SubscriptionTaskExecutor;
 use sp_transaction_pool::TransactionPool;
 
 /// Light client extra dependencies.
@@ -78,8 +78,8 @@ pub struct GrandpaDeps {
 	pub shared_authority_set: SharedAuthoritySet<Hash, BlockNumber>,
 	/// Receives notifications about justification events from Grandpa.
 	pub justification_stream: GrandpaJustificationStream<Block>,
-	/// Subscription manager to keep track of pubsub subscribers.
-	pub subscriptions: SubscriptionManager,
+	/// Executor to drive the subscription manager in the Grandpa RPC handler.
+	pub subscription_executor: SubscriptionTaskExecutor,
 }
 
 /// Full client dependencies.
@@ -139,7 +139,7 @@ pub fn create_full<C, P, SC>(
 		shared_voter_state,
 		shared_authority_set,
 		justification_stream,
-		subscriptions,
+		subscription_executor,
 	} = grandpa;
 
 	io.extend_with(
@@ -172,7 +172,7 @@ pub fn create_full<C, P, SC>(
 				shared_authority_set,
 				shared_voter_state,
 				justification_stream,
-				subscriptions,
+				subscription_executor,
 			)
 		)
 	);
diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs
index 5606da42d5947..fedd7220d3115 100644
--- a/client/finality-grandpa/rpc/src/lib.rs
+++ b/client/finality-grandpa/rpc/src/lib.rs
@@ -19,6 +19,7 @@
 //! RPC API for GRANDPA.
 #![warn(missing_docs)]
 
+use std::sync::Arc;
 use futures::{FutureExt, TryFutureExt, TryStreamExt, StreamExt};
 use log::warn;
 use jsonrpc_derive::rpc;
@@ -27,6 +28,7 @@ use jsonrpc_core::futures::{
 	sink::Sink as Sink01,
 	stream::Stream as Stream01,
 	future::Future as Future01,
+	future::Executor as Executor01,
 };
 
 mod error;
@@ -92,12 +94,16 @@ pub struct GrandpaRpcHandler<AuthoritySet, VoterState, Block: BlockT> {
 
 impl<AuthoritySet, VoterState, Block: BlockT> GrandpaRpcHandler<AuthoritySet, VoterState, Block> {
 	/// Creates a new GrandpaRpcHandler instance.
-	pub fn new(
+	pub fn new<E>(
 		authority_set: AuthoritySet,
 		voter_state: VoterState,
 		justification_stream: GrandpaJustificationStream<Block>,
-		manager: SubscriptionManager,
-	) -> Self {
+		executor: E,
+	) -> Self
+	where
+		E: Executor01<Box<dyn Future01<Item = (), Error = ()> + Send>> + Send + Sync + 'static,
+	{
+		let manager = SubscriptionManager::new(Arc::new(executor));
 		Self {
 			authority_set,
 			voter_state,
@@ -232,13 +238,12 @@ mod tests {
 		VoterState: ReportVoterState + Send + Sync + 'static,
 	{
 		let (justification_sender, justification_stream) = GrandpaJustificationStream::channel();
-		let manager = SubscriptionManager::new(Arc::new(sc_rpc::testing::TaskExecutor));
 
 		let handler = GrandpaRpcHandler::new(
 			TestAuthoritySet,
 			voter_state,
 			justification_stream,
-			manager,
+			sc_rpc::testing::TaskExecutor,
 		);
 
 		let mut io = jsonrpc_core::MetaIoHandler::default();
diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs
index 93e6c3fc91ba7..49f54365ddf37 100644
--- a/client/service/src/builder.rs
+++ b/client/service/src/builder.rs
@@ -46,7 +46,7 @@ use sp_runtime::traits::{
 };
 use sp_api::{ProvideRuntimeApi, CallApiAt};
 use sc_executor::{NativeExecutor, NativeExecutionDispatch, RuntimeInfo};
-use std::{collections::HashMap, sync::Arc};
+use std::sync::Arc;
 use wasm_timer::SystemTime;
 use sc_telemetry::{telemetry, SUBSTRATE_INFO};
 use sp_transaction_pool::MaintainedTransactionPool;
@@ -73,17 +73,25 @@ pub trait RpcExtensionBuilder {
 
 	/// Returns an instance of the RPC extension for a particular `DenyUnsafe`
 	/// value, e.g. the RPC extension might not expose some unsafe methods.
-	fn build(&self, deny: sc_rpc::DenyUnsafe, subscriptions: SubscriptionManager) -> Self::Output;
+	fn build(
+		&self,
+		deny: sc_rpc::DenyUnsafe,
+		subscription_executor: sc_rpc::SubscriptionTaskExecutor,
+	) -> Self::Output;
 }
 
 impl<F, R> RpcExtensionBuilder for F where
-	F: Fn(sc_rpc::DenyUnsafe, SubscriptionManager) -> R,
+	F: Fn(sc_rpc::DenyUnsafe, sc_rpc::SubscriptionTaskExecutor) -> R,
 	R: sc_rpc::RpcExtension<sc_rpc::Metadata>,
 {
 	type Output = R;
 
-	fn build(&self, deny: sc_rpc::DenyUnsafe, subscriptions: SubscriptionManager) -> Self::Output {
-		(*self)(deny, subscriptions)
+	fn build(
+		&self,
+		deny: sc_rpc::DenyUnsafe,
+		subscription_executor: sc_rpc::SubscriptionTaskExecutor,
+	) -> Self::Output {
+		(*self)(deny, subscription_executor)
 	}
 }
 
@@ -97,7 +105,11 @@ impl<R> RpcExtensionBuilder for NoopRpcExtensionBuilder<R> where
 {
 	type Output = R;
 
-	fn build(&self, _deny: sc_rpc::DenyUnsafe, _subscriptions: SubscriptionManager) -> Self::Output {
+	fn build(
+		&self,
+		_deny: sc_rpc::DenyUnsafe,
+		_subscription_executor: sc_rpc::SubscriptionTaskExecutor,
+	) -> Self::Output {
 		self.0.clone()
 	}
 }
@@ -694,7 +706,7 @@ fn gen_handler<TBl, TBackend, TExPool, TRpc, TCl>(
 	};
 
 	let task_executor = sc_rpc::SubscriptionTaskExecutor::new(spawn_handle);
-	let subscriptions = SubscriptionManager::new(Arc::new(task_executor));
+	let subscriptions = SubscriptionManager::new(Arc::new(task_executor.clone()));
 
 	let (chain, state, child_state) = if let (Some(remote_blockchain), Some(on_demand)) =
 		(remote_blockchain, on_demand) {
@@ -723,20 +735,16 @@ fn gen_handler<TBl, TBackend, TExPool, TRpc, TCl>(
 	let author = sc_rpc::author::Author::new(
 		client,
 		transaction_pool,
-		subscriptions.clone(),
+		subscriptions,
 		keystore,
 		deny_unsafe,
 	);
 	let system = system::System::new(system_info, system_rpc_tx, deny_unsafe);
 
-	let maybe_offchain_rpc = offchain_storage
-	.map(|storage| {
+	let maybe_offchain_rpc = offchain_storage.map(|storage| {
 		let offchain = sc_rpc::offchain::Offchain::new(storage, deny_unsafe);
-		// FIXME: Use plain Option (don't collect into HashMap) when we upgrade to jsonrpc 14.1
-		// https://github.com/paritytech/jsonrpc/commit/20485387ed06a48f1a70bf4d609a7cde6cf0accf
-		let delegate = offchain::OffchainApi::to_delegate(offchain);
-			delegate.into_iter().collect::<HashMap<_, _>>()
-	}).unwrap_or_default();
+		offchain::OffchainApi::to_delegate(offchain)
+	});
 
 	sc_rpc_server::rpc_handler((
 		state::StateApi::to_delegate(state),
@@ -745,7 +753,7 @@ fn gen_handler<TBl, TBackend, TExPool, TRpc, TCl>(
 		maybe_offchain_rpc,
 		author::AuthorApi::to_delegate(author),
 		system::SystemApi::to_delegate(system),
-		rpc_extensions_builder.build(deny_unsafe, subscriptions),
+		rpc_extensions_builder.build(deny_unsafe, task_executor),
 	))
 }
 

From caab5385b2dd3aceb5fc62944c11812b942f3219 Mon Sep 17 00:00:00 2001
From: Wei Tang <wei@that.world>
Date: Mon, 14 Sep 2020 16:16:11 +0200
Subject: [PATCH 079/122] pallet-collective: allow customized default vote
 (#6984)

* collective: add DefaultVote trait

* Fix test and node compile

* Expose the whole prime_vote

* Add test for MoreThanMajorityThenPrimeDefaultVote

* Docs fix
---
 bin/node/runtime/src/lib.rs |   4 +-
 frame/collective/src/lib.rs | 118 +++++++++++++++++++++++++++++++++---
 2 files changed, 111 insertions(+), 11 deletions(-)

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index 54dea704bd7f6..eeac6d83b8777 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -109,7 +109,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	// and set impl_version to 0. If only runtime
 	// implementation changes and behavior does not, then leave spec_version as
 	// is and increment impl_version.
-	spec_version: 258,
+	spec_version: 259,
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
 	transaction_version: 1,
@@ -520,6 +520,7 @@ impl pallet_collective::Trait<CouncilCollective> for Runtime {
 	type MotionDuration = CouncilMotionDuration;
 	type MaxProposals = CouncilMaxProposals;
 	type MaxMembers = CouncilMaxMembers;
+	type DefaultVote = pallet_collective::PrimeDefaultVote;
 	type WeightInfo = weights::pallet_collective::WeightInfo;
 }
 
@@ -569,6 +570,7 @@ impl pallet_collective::Trait<TechnicalCollective> for Runtime {
 	type MotionDuration = TechnicalMotionDuration;
 	type MaxProposals = TechnicalMaxProposals;
 	type MaxMembers = TechnicalMaxMembers;
+	type DefaultVote = pallet_collective::PrimeDefaultVote;
 	type WeightInfo = weights::pallet_collective::WeightInfo;
 }
 
diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs
index 20c701e3f0491..e19d220533d45 100644
--- a/frame/collective/src/lib.rs
+++ b/frame/collective/src/lib.rs
@@ -23,8 +23,11 @@
 //! The pallet assumes that the amount of members stays at or below `MaxMembers` for its weight
 //! calculations, but enforces this neither in `set_members` nor in `change_members_sorted`.
 //!
-//! A "prime" member may be set allowing their vote to act as the default vote in case of any
-//! abstentions after the voting period.
+//! A "prime" member may be set to help determine the default vote behavior based on chain
+//! config. If `PreimDefaultVote` is used, the prime vote acts as the default vote in case of any
+//! abstentions after the voting period. If `MoreThanMajorityThenPrimeDefaultVote` is used, then
+//! abstentations will first follow the majority of the collective voting, and then the prime
+//! member.
 //!
 //! Voting happens through motions comprising a proposal (i.e. a curried dispatchable) plus a
 //! number of approvals required for it to pass and be called. Motions are open for members to
@@ -71,6 +74,52 @@ pub type ProposalIndex = u32;
 /// vote exactly once, therefore also the number of votes for any given motion.
 pub type MemberCount = u32;
 
+/// Default voting strategy when a member is inactive.
+pub trait DefaultVote {
+	/// Get the default voting strategy, given:
+	///
+	/// - Whether the prime member voted Aye.
+	/// - Raw number of yes votes.
+	/// - Raw number of no votes.
+	/// - Total number of member count.
+	fn default_vote(
+		prime_vote: Option<bool>,
+		yes_votes: MemberCount,
+		no_votes: MemberCount,
+		len: MemberCount,
+	) -> bool;
+}
+
+/// Set the prime member's vote as the default vote.
+pub struct PrimeDefaultVote;
+
+impl DefaultVote for PrimeDefaultVote {
+	fn default_vote(
+		prime_vote: Option<bool>,
+		_yes_votes: MemberCount,
+		_no_votes: MemberCount,
+		_len: MemberCount,
+	) -> bool {
+		prime_vote.unwrap_or(false)
+	}
+}
+
+/// First see if yes vote are over majority of the whole collective. If so, set the default vote
+/// as yes. Otherwise, use the prime meber's vote as the default vote.
+pub struct MoreThanMajorityThenPrimeDefaultVote;
+
+impl DefaultVote for MoreThanMajorityThenPrimeDefaultVote {
+	fn default_vote(
+		prime_vote: Option<bool>,
+		yes_votes: MemberCount,
+		_no_votes: MemberCount,
+		len: MemberCount,
+	) -> bool {
+		let more_than_majority = yes_votes * 2 > len;
+		more_than_majority || prime_vote.unwrap_or(false)
+	}
+}
+
 pub trait WeightInfo {
 	fn set_members(m: u32, n: u32, p: u32, ) -> Weight;
 	fn execute(b: u32, m: u32, ) -> Weight;
@@ -110,6 +159,9 @@ pub trait Trait<I: Instance=DefaultInstance>: frame_system::Trait {
 	/// + This pallet assumes that dependents keep to the limit without enforcing it.
 	type MaxMembers: Get<MemberCount>;
 
+	/// Default vote strategy of this collective.
+	type DefaultVote: DefaultVote;
+
 	/// Weight information for extrinsics in this pallet.
 	type WeightInfo: WeightInfo;
 }
@@ -157,8 +209,7 @@ decl_storage! {
 		pub ProposalCount get(fn proposal_count): u32;
 		/// The current members of the collective. This is stored sorted (just by value).
 		pub Members get(fn members): Vec<T::AccountId>;
-		/// The member who provides the default vote for any other members that do not vote before
-		/// the timeout. If None, then no member has that privilege.
+		/// The prime member that helps determine the default vote behavior in case of absentations.
 		pub Prime get(fn prime): Option<T::AccountId>;
 	}
 	add_extra_genesis {
@@ -587,8 +638,10 @@ decl_module! {
 			// Only allow actual closing of the proposal after the voting period has ended.
 			ensure!(system::Module::<T>::block_number() >= voting.end, Error::<T, I>::TooEarly);
 
-			// default to true only if there's a prime and they voted in favour.
-			let default = Self::prime().map_or(false, |who| voting.ayes.iter().any(|a| a == &who));
+			let prime_vote = Self::prime().map(|who| voting.ayes.iter().any(|a| a == &who));
+
+			// default voting strategy.
+			let default = T::DefaultVote::default_vote(prime_vote, yes_votes, no_votes, seats);
 
 			let abstentions = seats - (yes_votes + no_votes);
 			match default {
@@ -945,6 +998,17 @@ mod tests {
 		type MotionDuration = MotionDuration;
 		type MaxProposals = MaxProposals;
 		type MaxMembers = MaxMembers;
+		type DefaultVote = PrimeDefaultVote;
+		type WeightInfo = ();
+	}
+	impl Trait<Instance2> for Test {
+		type Origin = Origin;
+		type Proposal = Call;
+		type Event = Event;
+		type MotionDuration = MotionDuration;
+		type MaxProposals = MaxProposals;
+		type MaxMembers = MaxMembers;
+		type DefaultVote = MoreThanMajorityThenPrimeDefaultVote;
 		type WeightInfo = ();
 	}
 	impl Trait for Test {
@@ -954,6 +1018,7 @@ mod tests {
 		type MotionDuration = MotionDuration;
 		type MaxProposals = MaxProposals;
 		type MaxMembers = MaxMembers;
+		type DefaultVote = PrimeDefaultVote;
 		type WeightInfo = ();
 	}
 
@@ -968,6 +1033,7 @@ mod tests {
 		{
 			System: system::{Module, Call, Event<T>},
 			Collective: collective::<Instance1>::{Module, Call, Event<T>, Origin<T>, Config<T>},
+			CollectiveMajority: collective::<Instance2>::{Module, Call, Event<T>, Origin<T>, Config<T>},
 			DefaultCollective: collective::{Module, Call, Event<T>, Origin<T>, Config<T>},
 		}
 	);
@@ -978,12 +1044,20 @@ mod tests {
 				members: vec![1, 2, 3],
 				phantom: Default::default(),
 			}),
+			collective_Instance2: Some(collective::GenesisConfig {
+				members: vec![1, 2, 3, 4, 5],
+				phantom: Default::default(),
+			}),
 			collective: None,
 		}.build_storage().unwrap().into();
 		ext.execute_with(|| System::set_block_number(1));
 		ext
 	}
 
+	fn make_proposal(value: u64) -> Call {
+		Call::System(frame_system::Call::remark(value.encode()))
+	}
+
 	#[test]
 	fn motions_basic_environment_works() {
 		new_test_ext().execute_with(|| {
@@ -992,10 +1066,6 @@ mod tests {
 		});
 	}
 
-	fn make_proposal(value: u64) -> Call {
-		Call::System(frame_system::Call::remark(value.encode()))
-	}
-
 	#[test]
 	fn close_works() {
 		new_test_ext().execute_with(|| {
@@ -1114,6 +1184,34 @@ mod tests {
 		});
 	}
 
+	#[test]
+	fn close_with_no_prime_but_majority_works() {
+		new_test_ext().execute_with(|| {
+			let proposal = make_proposal(42);
+			let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32);
+			let proposal_weight = proposal.get_dispatch_info().weight;
+			let hash = BlakeTwo256::hash_of(&proposal);
+			assert_ok!(CollectiveMajority::set_members(Origin::root(), vec![1, 2, 3, 4, 5], Some(5), MaxMembers::get()));
+
+			assert_ok!(CollectiveMajority::propose(Origin::signed(1), 5, Box::new(proposal.clone()), proposal_len));
+			assert_ok!(CollectiveMajority::vote(Origin::signed(2), hash.clone(), 0, true));
+			assert_ok!(CollectiveMajority::vote(Origin::signed(3), hash.clone(), 0, true));
+
+			System::set_block_number(4);
+			assert_ok!(CollectiveMajority::close(Origin::signed(4), hash.clone(), 0, proposal_weight, proposal_len));
+
+			let record = |event| EventRecord { phase: Phase::Initialization, event, topics: vec![] };
+			assert_eq!(System::events(), vec![
+				record(Event::collective_Instance2(RawEvent::Proposed(1, 0, hash.clone(), 5))),
+				record(Event::collective_Instance2(RawEvent::Voted(2, hash.clone(), true, 2, 0))),
+				record(Event::collective_Instance2(RawEvent::Voted(3, hash.clone(), true, 3, 0))),
+				record(Event::collective_Instance2(RawEvent::Closed(hash.clone(), 5, 0))),
+				record(Event::collective_Instance2(RawEvent::Approved(hash.clone()))),
+				record(Event::collective_Instance2(RawEvent::Executed(hash.clone(), Err(DispatchError::BadOrigin))))
+			]);
+		});
+	}
+
 	#[test]
 	fn removal_of_old_voters_votes_works() {
 		new_test_ext().execute_with(|| {

From 5f6987aceef728829c3b83377a94ead74c4f0f1c Mon Sep 17 00:00:00 2001
From: Roman Borschel <romanb@users.noreply.github.com>
Date: Mon, 14 Sep 2020 16:27:58 +0200
Subject: [PATCH 080/122] Upgrade to libp2p-0.28. (#7077)

* Upgrade to libp2p-0.28

* Clean up test imports.

* CI

* CI

* CI?

* CI once more.

* One more.

* CI

* CI

* CI
---
 Cargo.lock                                    | 229 +++++++-----------
 Cargo.toml                                    |   2 -
 bin/node/browser-testing/Cargo.toml           |   2 +-
 client/authority-discovery/Cargo.toml         |   2 +-
 client/cli/Cargo.toml                         |   2 +-
 client/network-gossip/Cargo.toml              |   2 +-
 client/network/Cargo.toml                     |   4 +-
 client/network/src/block_requests.rs          |   2 +-
 client/network/src/discovery.rs               |  27 +--
 client/network/src/finality_requests.rs       |   2 +-
 client/network/src/light_client_handler.rs    |   2 +-
 .../protocol/generic_proto/handler/group.rs   |  26 +-
 .../protocol/generic_proto/handler/legacy.rs  |  14 +-
 .../generic_proto/handler/notif_in.rs         |   8 +-
 .../generic_proto/handler/notif_out.rs        |  17 +-
 .../src/protocol/generic_proto/tests.rs       |  40 ++-
 client/network/src/request_responses.rs       |  54 ++---
 client/network/src/service.rs                 |   2 +
 client/network/src/transport.rs               |   2 +-
 client/network/test/Cargo.toml                |   2 +-
 client/peerset/Cargo.toml                     |   2 +-
 client/telemetry/Cargo.toml                   |   2 +-
 primitives/consensus/common/Cargo.toml        |   2 +-
 utils/browser/Cargo.toml                      |   2 +-
 24 files changed, 181 insertions(+), 268 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9264534efa02b..c33ff0020acae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -40,23 +40,11 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7001367fde4c768a19d1029f0a8be5abd9308e1119846d5bd9ad26297b8faf5"
 dependencies = [
- "aes-soft 0.4.0",
- "aesni 0.7.0",
+ "aes-soft",
+ "aesni",
  "block-cipher",
 ]
 
-[[package]]
-name = "aes-ctr"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee"
-dependencies = [
- "aes-soft 0.3.3",
- "aesni 0.6.0",
- "ctr",
- "stream-cipher 0.3.2",
-]
-
 [[package]]
 name = "aes-gcm"
 version = "0.6.0"
@@ -70,17 +58,6 @@ dependencies = [
  "subtle 2.2.3",
 ]
 
-[[package]]
-name = "aes-soft"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d"
-dependencies = [
- "block-cipher-trait",
- "byteorder 1.3.4",
- "opaque-debug 0.2.3",
-]
-
 [[package]]
 name = "aes-soft"
 version = "0.4.0"
@@ -92,17 +69,6 @@ dependencies = [
  "opaque-debug 0.2.3",
 ]
 
-[[package]]
-name = "aesni"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100"
-dependencies = [
- "block-cipher-trait",
- "opaque-debug 0.2.3",
- "stream-cipher 0.3.2",
-]
-
 [[package]]
 name = "aesni"
 version = "0.7.0"
@@ -529,15 +495,6 @@ dependencies = [
  "generic-array 0.14.3",
 ]
 
-[[package]]
-name = "block-cipher-trait"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c924d49bd09e7c06003acda26cd9742e796e34282ec6c1189404dee0c1f4774"
-dependencies = [
- "generic-array 0.12.3",
-]
-
 [[package]]
 name = "block-padding"
 version = "0.1.5"
@@ -1114,16 +1071,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "ctr"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "022cd691704491df67d25d006fe8eca083098253c4d43516c2206479c58c6736"
-dependencies = [
- "block-cipher-trait",
- "stream-cipher 0.3.2",
-]
-
 [[package]]
 name = "cuckoofilter"
 version = "0.3.2"
@@ -1283,9 +1230,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.5.3"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
+checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f"
 
 [[package]]
 name = "enumflags2"
@@ -2810,9 +2757,9 @@ checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a"
 
 [[package]]
 name = "libp2p"
-version = "0.24.0"
+version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76c101edbb9c06955fd4085b77d2abc31cf3650134d77068b35c44967756ada8"
+checksum = "571f5a4604c1a40d75651da141dfde29ad15329f537a779528803297d2220274"
 dependencies = [
  "atomic",
  "bytes 0.5.6",
@@ -2833,7 +2780,6 @@ dependencies = [
  "libp2p-plaintext",
  "libp2p-pnet",
  "libp2p-request-response",
- "libp2p-secio",
  "libp2p-swarm",
  "libp2p-tcp",
  "libp2p-uds",
@@ -2850,9 +2796,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-core"
-version = "0.21.0"
+version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17cea54ea4a846a7c47e4347db0fc7a4129dcb0fb57f07f57e473820edbfcbde"
+checksum = "52f13ba8c7df0768af2eb391696d562c7de88cc3a35122531aaa6a7d77754d25"
 dependencies = [
  "asn1_der",
  "bs58",
@@ -2894,9 +2840,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-deflate"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc6174d6addc9cc5fd84af7099480774035dd1a7cdf48dd31b23dea45cf57638"
+checksum = "74029ae187f35f4b8ddf26b9779a68b340045d708528a103917cdca49a296db5"
 dependencies = [
  "flate2",
  "futures 0.3.5",
@@ -2905,9 +2851,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-dns"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fce8769cfe677a567d2677dc02a9e5be27a24acf1ff78a59cef425caae009a6a"
+checksum = "7cf319822e08dd65c8e060d2354e9f952895bbc433f5706c75ed010c152aee5e"
 dependencies = [
  "futures 0.3.5",
  "libp2p-core",
@@ -2916,9 +2862,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-floodsub"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f2342965ac7ea4b85f4df5288089796421f9297ba4020dc9692f4ef728590dc"
+checksum = "d8a9acb43a3e4a4e413e0c4abe0fa49308df7c6335c88534757b647199cb8a51"
 dependencies = [
  "cuckoofilter",
  "fnv",
@@ -2933,9 +2879,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-gossipsub"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0828b4f0c76c2edc68da574e391ce981bac5316d65785cddfe8c273d4c9bd4bb"
+checksum = "ab20fcb60edebe3173bbb708c6ac3444afdf1e3152dc2866b10c4f5497f17467"
 dependencies = [
  "base64 0.11.0",
  "byteorder 1.3.4",
@@ -2959,9 +2905,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-identify"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41efcb5b521b65d2c45432a244ce6427cdd3649228cd192f397d1fa67682aef2"
+checksum = "56396ee63aa9164eacf40c2c5d2bda8c4133c2f57e1b0425d51d3a4e362583b1"
 dependencies = [
  "futures 0.3.5",
  "libp2p-core",
@@ -2975,9 +2921,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-kad"
-version = "0.22.1"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca9b4ccc868863317af3f65eb241811ceadd971d133183040140f5496037e0ae"
+checksum = "cc7fa9047f8b8f544278a35c2d9d45d3b2c1785f2d86d4e1629d6edf97be3955"
 dependencies = [
  "arrayvec 0.5.1",
  "bytes 0.5.6",
@@ -3002,9 +2948,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-mdns"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fe5614c2c5af74ef5870aad0fce73c9e4707716c4ee7cdf06cf9a0376d3815"
+checksum = "3173b5a6b2f690c29ae07798d85b9441a131ac76ddae9015ef22905b623d0c69"
 dependencies = [
  "async-std",
  "data-encoding",
@@ -3024,9 +2970,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-mplex"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df9e79541e71590846f773efce1b6d0538804992ee54ff2f407e05d63a9ddc23"
+checksum = "8a73a799cc8410b36e40b8f4c4b6babbcb9efd3727111bf517876e4acfa612d3"
 dependencies = [
  "bytes 0.5.6",
  "fnv",
@@ -3040,9 +2986,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-noise"
-version = "0.23.0"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0beba6459d06153f5f8e23da3df1d2183798b1f457c7c9468ff99760bcbcc60b"
+checksum = "6ef6c490042f549fb1025f2892dfe6083d97a77558f450c1feebe748ca9eb15a"
 dependencies = [
  "bytes 0.5.6",
  "curve25519-dalek",
@@ -3062,9 +3008,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-ping"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "670261ef938567b614746b078e049b03b55617538a8d415071c518f97532d043"
+checksum = "ad063c21dfcea4518ac9e8bd4119d33a5b26c41e674f602f41f05617a368a5c8"
 dependencies = [
  "futures 0.3.5",
  "libp2p-core",
@@ -3077,9 +3023,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-plaintext"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3a61dfd53d1264ddff1206e4827193efaa72bab27782dfcd63c0dec120a1875"
+checksum = "903a12e99c72dbebefea258de887982adeacc7025baa1ceb10b7fa9928f54791"
 dependencies = [
  "bytes 0.5.6",
  "futures 0.3.5",
@@ -3109,57 +3055,31 @@ dependencies = [
 
 [[package]]
 name = "libp2p-request-response"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4af0de0e56a11d46c5191a61019733b5618dc955c0a36f82866bb6d5d81a7f8f"
+checksum = "9c0c9e8a4cd69d97e9646c54313d007512f411aba8c5226cfcda16df6a6e84a3"
 dependencies = [
  "async-trait",
+ "bytes 0.5.6",
  "futures 0.3.5",
  "libp2p-core",
  "libp2p-swarm",
  "log",
  "lru 0.6.0",
+ "minicbor",
  "rand 0.7.3",
  "smallvec 1.4.1",
+ "unsigned-varint 0.5.1",
  "wasm-timer",
 ]
 
-[[package]]
-name = "libp2p-secio"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a04b320cc0394554e8d0adca21f4efd9f8c2da4930211d92e411a19a4dfd769e"
-dependencies = [
- "aes-ctr",
- "ctr",
- "futures 0.3.5",
- "hmac",
- "js-sys",
- "lazy_static",
- "libp2p-core",
- "log",
- "parity-send-wrapper",
- "pin-project",
- "prost",
- "prost-build",
- "quicksink",
- "rand 0.7.3",
- "ring",
- "rw-stream-sink",
- "sha2 0.8.2",
- "static_assertions",
- "twofish",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "web-sys",
-]
-
 [[package]]
 name = "libp2p-swarm"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57e4a7e64156e9d1a2daae36b5d791f057b9c53c9364a8e75f7f9848b54f9d68"
+checksum = "7193e444210132237b81b755ec7fe53f1c4bd2f53cf719729b94c0c72eb6eaa1"
 dependencies = [
+ "either",
  "futures 0.3.5",
  "libp2p-core",
  "log",
@@ -3171,9 +3091,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-tcp"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0f65400ccfbbf9a356733bceca6c519c9db0deb5fbcc0b81f89837c4cd53997"
+checksum = "44f42ec130d7a37a7e47bf4398026b7ad9185c08ed26972e2720f8b94112796f"
 dependencies = [
  "async-std",
  "futures 0.3.5",
@@ -3187,9 +3107,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-uds"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95bc8b0ca1dda4cccb1bb156d47a32e45cfa447ef18f737209f014a63f94a4a2"
+checksum = "dea7acb0a034f70d7db94c300eba3f65c0f6298820105624088a9609c9974d77"
 dependencies = [
  "async-std",
  "futures 0.3.5",
@@ -3199,9 +3119,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-wasm-ext"
-version = "0.21.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2f7b06d80d036ac5763a811185b7fe6951ad71c00544b17cc378a9069bb7c2"
+checksum = "34c1faac6f92c21fbe155417957863ea822fba9e9fd5eb24c0912336a100e63f"
 dependencies = [
  "futures 0.3.5",
  "js-sys",
@@ -3213,9 +3133,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-websocket"
-version = "0.22.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5b350db65cf0a7c83a539a596ea261caae1552c0df2245df0f916ed2fd04572"
+checksum = "d650534ebd99f48f6fa292ed5db10d30df2444943afde4407ceeddab8e513fca"
 dependencies = [
  "async-tls",
  "either",
@@ -3233,13 +3153,13 @@ dependencies = [
 
 [[package]]
 name = "libp2p-yamux"
-version = "0.21.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3969ead4ce530efb6f304623924245caf410f3b0b0139bd7007f205933788aa"
+checksum = "781d9b9f043dcdabc40640807125368596b849fd4d96cdca2dcf052fdf6f33fd"
 dependencies = [
  "futures 0.3.5",
  "libp2p-core",
- "parking_lot 0.10.2",
+ "parking_lot 0.11.0",
  "thiserror",
  "yamux",
 ]
@@ -3490,6 +3410,26 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "minicbor"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fc03ad6f8f548db7194a5ff5a6f96342ecae4e3ef67d2bf18bacc0e245cd041"
+dependencies = [
+ "minicbor-derive",
+]
+
+[[package]]
+name = "minicbor-derive"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c214bf3d90099b52f3e4b328ae0fe34837fd0fab683ad1e10fceb4629106df48"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.4.0"
@@ -5149,9 +5089,9 @@ dependencies = [
 
 [[package]]
 name = "parity-multiaddr"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc20af3143a62c16e7c9e92ea5c6ae49f7d271d97d4d8fe73afc28f0514a3d0f"
+checksum = "2165a93382a93de55868dcbfa11e4a8f99676a9164eee6a2b4a9479ad319c257"
 dependencies = [
  "arrayref",
  "bs58",
@@ -9585,17 +9525,6 @@ dependencies = [
  "toml",
 ]
 
-[[package]]
-name = "twofish"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712d261e83e727c8e2dbb75dacac67c36e35db36a958ee504f2164fc052434e1"
-dependencies = [
- "block-cipher-trait",
- "byteorder 1.3.4",
- "opaque-debug 0.2.3",
-]
-
 [[package]]
 name = "twox-hash"
 version = "1.5.0"
@@ -9696,6 +9625,16 @@ dependencies = [
  "futures_codec",
 ]
 
+[[package]]
+name = "unsigned-varint"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7fdeedbf205afadfe39ae559b75c3240f24e257d0ca27e85f85cb82aa19ac35"
+dependencies = [
+ "futures-io",
+ "futures-util",
+]
+
 [[package]]
 name = "untrusted"
 version = "0.7.1"
@@ -10269,14 +10208,14 @@ dependencies = [
 
 [[package]]
 name = "yamux"
-version = "0.4.7"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd37e58a1256a0b328ce9c67d8b62ecdd02f4803ba443df478835cb1a41a637c"
+checksum = "9aeb8c4043cac71c3c299dff107171c220d179492350ea198e109a414981b83c"
 dependencies = [
  "futures 0.3.5",
  "log",
  "nohash-hasher",
- "parking_lot 0.10.2",
+ "parking_lot 0.11.0",
  "rand 0.7.3",
  "static_assertions",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 534b71357cc76..8f483234dfa87 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -203,7 +203,6 @@ members = [
 #
 # This list is ordered alphabetically.
 [profile.dev.package]
-aes-ctr = { opt-level = 3 }
 aes-soft = { opt-level = 3 }
 aesni = { opt-level = 3 }
 blake2 = { opt-level = 3 }
@@ -217,7 +216,6 @@ crc32fast = { opt-level = 3 }
 crossbeam-deque = { opt-level = 3 }
 crossbeam-queue = { opt-level = 3 }
 crypto-mac = { opt-level = 3 }
-ctr = { opt-level = 3 }
 curve25519-dalek = { opt-level = 3 }
 ed25519-dalek = { opt-level = 3 }
 evm-core = { opt-level = 3 }
diff --git a/bin/node/browser-testing/Cargo.toml b/bin/node/browser-testing/Cargo.toml
index 1cfc0623dd980..a6945d3163555 100644
--- a/bin/node/browser-testing/Cargo.toml
+++ b/bin/node/browser-testing/Cargo.toml
@@ -8,7 +8,7 @@ license = "Apache-2.0"
 
 [dependencies]
 futures-timer = "3.0.2"
-libp2p = { version = "0.24.0", default-features = false }
+libp2p = { version = "0.28.1", default-features = false }
 jsonrpc-core = "14.2.0"
 serde = "1.0.106"
 serde_json = "1.0.48"
diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml
index 8c898ab496422..e2be0f68e2318 100644
--- a/client/authority-discovery/Cargo.toml
+++ b/client/authority-discovery/Cargo.toml
@@ -22,7 +22,7 @@ derive_more = "0.99.2"
 either = "1.5.3"
 futures = "0.3.4"
 futures-timer = "3.0.1"
-libp2p = { version = "0.24.0", default-features = false, features = ["kad"] }
+libp2p = { version = "0.28.1", default-features = false, features = ["kad"] }
 log = "0.4.8"
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc6"}
 prost = "0.6.1"
diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml
index 8b634d687c717..6bee1afc5a909 100644
--- a/client/cli/Cargo.toml
+++ b/client/cli/Cargo.toml
@@ -23,7 +23,7 @@ lazy_static = "1.4.0"
 tokio = { version = "0.2.21", features = [ "signal", "rt-core", "rt-threaded", "blocking" ] }
 futures = "0.3.4"
 fdlimit = "0.2.0"
-libp2p = "0.24.0"
+libp2p = "0.28.1"
 parity-scale-codec = "1.3.0"
 hex = "0.4.2"
 rand = "0.7.3"
diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml
index f826bb88bade2..0ff86e8d43709 100644
--- a/client/network-gossip/Cargo.toml
+++ b/client/network-gossip/Cargo.toml
@@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 [dependencies]
 futures = "0.3.4"
 futures-timer = "3.0.1"
-libp2p = { version = "0.24.0", default-features = false }
+libp2p = { version = "0.28.1", default-features = false }
 log = "0.4.8"
 lru = "0.4.3"
 sc-network = { version = "0.8.0-rc6", path = "../network" }
diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml
index d5729ae06b2ca..7c06de7ccd1cb 100644
--- a/client/network/Cargo.toml
+++ b/client/network/Cargo.toml
@@ -63,14 +63,14 @@ wasm-timer = "0.2"
 zeroize = "1.0.0"
 
 [dependencies.libp2p]
-version = "0.24.0"
+version = "0.28.1"
 default-features = false
 features = ["identify", "kad", "mdns-async-std", "mplex", "noise", "ping", "request-response", "tcp-async-std", "websocket", "yamux"]
 
 [dev-dependencies]
 assert_matches = "1.3"
 env_logger = "0.7.0"
-libp2p = { version = "0.24.0", default-features = false, features = ["secio"] }
+libp2p = { version = "0.28.1", default-features = false }
 quickcheck = "0.9.0"
 rand = "0.7.2"
 sp-keyring = { version = "2.0.0-rc6", path = "../../primitives/keyring" }
diff --git a/client/network/src/block_requests.rs b/client/network/src/block_requests.rs
index f1fbe8fb2523e..7ee8f18f3a26f 100644
--- a/client/network/src/block_requests.rs
+++ b/client/network/src/block_requests.rs
@@ -478,7 +478,7 @@ where
 		let mut cfg = OneShotHandlerConfig::default();
 		cfg.keep_alive_timeout = self.config.inactivity_timeout;
 		cfg.outbound_substream_timeout = self.config.request_timeout;
-		OneShotHandler::new(SubstreamProtocol::new(p), cfg)
+		OneShotHandler::new(SubstreamProtocol::new(p, ()), cfg)
 	}
 
 	fn addresses_of_peer(&mut self, _: &PeerId) -> Vec<Multiaddr> {
diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs
index 51ee224a9378b..bb68468475fa9 100644
--- a/client/network/src/discovery.rs
+++ b/client/network/src/discovery.rs
@@ -765,8 +765,9 @@ mod tests {
 	use libp2p::{Multiaddr, PeerId};
 	use libp2p::core::upgrade;
 	use libp2p::core::transport::{Transport, MemoryTransport};
-	use libp2p::core::upgrade::{InboundUpgradeExt, OutboundUpgradeExt};
+	use libp2p::noise;
 	use libp2p::swarm::Swarm;
+	use libp2p::yamux;
 	use std::{collections::HashSet, task::Poll};
 	use super::{DiscoveryConfig, DiscoveryOut, protocol_name_from_protocol_id};
 
@@ -779,25 +780,15 @@ mod tests {
 		// the first swarm via `with_user_defined`.
 		let mut swarms = (0..25).map(|i| {
 			let keypair = Keypair::generate_ed25519();
-			let keypair2 = keypair.clone();
+
+			let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
+				.into_authentic(&keypair)
+				.unwrap();
 
 			let transport = MemoryTransport
-				.and_then(move |out, endpoint| {
-					let secio = libp2p::secio::SecioConfig::new(keypair2);
-					libp2p::core::upgrade::apply(
-						out,
-						secio,
-						endpoint,
-						upgrade::Version::V1
-					)
-				})
-				.and_then(move |(peer_id, stream), endpoint| {
-					let peer_id2 = peer_id.clone();
-					let upgrade = libp2p::yamux::Config::default()
-						.map_inbound(move |muxer| (peer_id, muxer))
-						.map_outbound(move |muxer| (peer_id2, muxer));
-					upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1)
-				});
+				.upgrade(upgrade::Version::V1)
+				.authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated())
+				.multiplex(yamux::Config::default());
 
 			let behaviour = {
 				let mut config = DiscoveryConfig::new(keypair.public());
diff --git a/client/network/src/finality_requests.rs b/client/network/src/finality_requests.rs
index 9b99521ba681b..55f56b9a0cc25 100644
--- a/client/network/src/finality_requests.rs
+++ b/client/network/src/finality_requests.rs
@@ -235,7 +235,7 @@ where
 		};
 		let mut cfg = OneShotHandlerConfig::default();
 		cfg.keep_alive_timeout = self.config.inactivity_timeout;
-		OneShotHandler::new(SubstreamProtocol::new(p), cfg)
+		OneShotHandler::new(SubstreamProtocol::new(p, ()), cfg)
 	}
 
 	fn addresses_of_peer(&mut self, _: &PeerId) -> Vec<Multiaddr> {
diff --git a/client/network/src/light_client_handler.rs b/client/network/src/light_client_handler.rs
index 98af34092ab21..7f5ec54470e5d 100644
--- a/client/network/src/light_client_handler.rs
+++ b/client/network/src/light_client_handler.rs
@@ -758,7 +758,7 @@ where
 		};
 		let mut cfg = OneShotHandlerConfig::default();
 		cfg.keep_alive_timeout = self.config.inactivity_timeout;
-		OneShotHandler::new(SubstreamProtocol::new(p), cfg)
+		OneShotHandler::new(SubstreamProtocol::new(p, ()), cfg)
 	}
 
 	fn addresses_of_peer(&mut self, peer: &PeerId) -> Vec<Multiaddr> {
diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs
index acb241af2ad2d..bcdba87e1037c 100644
--- a/client/network/src/protocol/generic_proto/handler/group.rs
+++ b/client/network/src/protocol/generic_proto/handler/group.rs
@@ -394,25 +394,27 @@ impl ProtocolsHandler for NotifsHandler {
 	type OutboundProtocol = EitherUpgrade<NotificationsOut, RegisteredProtocol>;
 	// Index within the `out_handlers`; None for legacy
 	type OutboundOpenInfo = Option<usize>;
+	type InboundOpenInfo = ();
 
-	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
 		let in_handlers = self.in_handlers.iter()
 			.map(|(h, _)| h.listen_protocol().into_upgrade().1)
 			.collect::<UpgradeCollec<_>>();
 
 		let proto = SelectUpgrade::new(in_handlers, self.legacy.listen_protocol().into_upgrade().1);
-		SubstreamProtocol::new(proto)
+		SubstreamProtocol::new(proto, ())
 	}
 
 	fn inject_fully_negotiated_inbound(
 		&mut self,
-		out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+		out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+		(): ()
 	) {
 		match out {
 			EitherOutput::First((out, num)) =>
-				self.in_handlers[num].0.inject_fully_negotiated_inbound(out),
+				self.in_handlers[num].0.inject_fully_negotiated_inbound(out, ()),
 			EitherOutput::Second(out) =>
-				self.legacy.inject_fully_negotiated_inbound(out),
+				self.legacy.inject_fully_negotiated_inbound(out, ()),
 		}
 	}
 
@@ -619,10 +621,11 @@ impl ProtocolsHandler for NotifsHandler {
 		if self.pending_legacy_handshake.is_none() {
 			while let Poll::Ready(ev) = self.legacy.poll(cx) {
 				match ev {
-					ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } =>
+					ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol } =>
 						return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-							protocol: protocol.map_upgrade(EitherUpgrade::B),
-							info: None,
+							protocol: protocol
+								.map_upgrade(EitherUpgrade::B)
+								.map_info(|()| None)
 						}),
 					ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen {
 						received_handshake,
@@ -705,10 +708,11 @@ impl ProtocolsHandler for NotifsHandler {
 		for (handler_num, (handler, _)) in self.out_handlers.iter_mut().enumerate() {
 			while let Poll::Ready(ev) = handler.poll(cx) {
 				match ev {
-					ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } =>
+					ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol } =>
 						return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-							protocol: protocol.map_upgrade(EitherUpgrade::A),
-							info: Some(handler_num),
+							protocol: protocol
+								.map_upgrade(EitherUpgrade::A)
+								.map_info(|()| Some(handler_num))
 						}),
 					ProtocolsHandlerEvent::Close(err) => void::unreachable(err),
 
diff --git a/client/network/src/protocol/generic_proto/handler/legacy.rs b/client/network/src/protocol/generic_proto/handler/legacy.rs
index d98d864dfc6fa..d17b5e612daf2 100644
--- a/client/network/src/protocol/generic_proto/handler/legacy.rs
+++ b/client/network/src/protocol/generic_proto/handler/legacy.rs
@@ -253,8 +253,7 @@ impl LegacyProtoHandler {
 				if incoming.is_empty() {
 					if let ConnectedPoint::Dialer { .. } = self.endpoint {
 						self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-							protocol: SubstreamProtocol::new(self.protocol.clone()),
-							info: (),
+							protocol: SubstreamProtocol::new(self.protocol.clone(), ()),
 						});
 					}
 					ProtocolState::Opening {
@@ -428,8 +427,7 @@ impl LegacyProtoHandler {
 						deadline: Delay::new(Duration::from_secs(60))
 					};
 					Some(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-						protocol: SubstreamProtocol::new(self.protocol.clone()),
-						info: (),
+						protocol: SubstreamProtocol::new(self.protocol.clone(), ()),
 					})
 				} else {
 					self.state = ProtocolState::Disabled { shutdown, reenable };
@@ -498,14 +496,16 @@ impl ProtocolsHandler for LegacyProtoHandler {
 	type InboundProtocol = RegisteredProtocol;
 	type OutboundProtocol = RegisteredProtocol;
 	type OutboundOpenInfo = ();
+	type InboundOpenInfo = ();
 
-	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
-		SubstreamProtocol::new(self.protocol.clone())
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
+		SubstreamProtocol::new(self.protocol.clone(), ())
 	}
 
 	fn inject_fully_negotiated_inbound(
 		&mut self,
-		(substream, handshake): <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+		(substream, handshake): <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+		(): ()
 	) {
 		self.inject_fully_negotiated(substream, handshake);
 	}
diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs
index 5a50cce268117..d3b505e0de3e2 100644
--- a/client/network/src/protocol/generic_proto/handler/notif_in.rs
+++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs
@@ -175,14 +175,16 @@ impl ProtocolsHandler for NotifsInHandler {
 	type InboundProtocol = NotificationsIn;
 	type OutboundProtocol = DeniedUpgrade;
 	type OutboundOpenInfo = ();
+	type InboundOpenInfo = ();
 
-	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
-		SubstreamProtocol::new(self.in_protocol.clone())
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
+		SubstreamProtocol::new(self.in_protocol.clone(), ())
 	}
 
 	fn inject_fully_negotiated_inbound(
 		&mut self,
-		(msg, proto): <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+		(msg, proto): <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+		(): ()
 	) {
 		// If a substream already exists, we drop it and replace it with the new incoming one.
 		if self.substream.is_some() {
diff --git a/client/network/src/protocol/generic_proto/handler/notif_out.rs b/client/network/src/protocol/generic_proto/handler/notif_out.rs
index 4079d2fa2a6b2..414e62c0d135f 100644
--- a/client/network/src/protocol/generic_proto/handler/notif_out.rs
+++ b/client/network/src/protocol/generic_proto/handler/notif_out.rs
@@ -267,14 +267,16 @@ impl ProtocolsHandler for NotifsOutHandler {
 	type InboundProtocol = DeniedUpgrade;
 	type OutboundProtocol = NotificationsOut;
 	type OutboundOpenInfo = ();
+	type InboundOpenInfo = ();
 
-	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
-		SubstreamProtocol::new(DeniedUpgrade)
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
+		SubstreamProtocol::new(DeniedUpgrade, ())
 	}
 
 	fn inject_fully_negotiated_inbound(
 		&mut self,
-		proto: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output
+		proto: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+		(): ()
 	) {
 		// We should never reach here. `proto` is a `Void`.
 		void::unreachable(proto)
@@ -309,8 +311,7 @@ impl ProtocolsHandler for NotifsOutHandler {
 					State::Disabled => {
 						let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone());
 						self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-							protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT),
-							info: (),
+							protocol: SubstreamProtocol::new(proto, ()).with_timeout(OPEN_TIMEOUT),
 						});
 						self.state = State::Opening { initial_message };
 					},
@@ -329,8 +330,7 @@ impl ProtocolsHandler for NotifsOutHandler {
 
 						let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone());
 						self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-							protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT),
-							info: (),
+							protocol: SubstreamProtocol::new(proto, ()).with_timeout(OPEN_TIMEOUT),
 						});
 						self.state = State::Opening { initial_message };
 					},
@@ -414,8 +414,7 @@ impl ProtocolsHandler for NotifsOutHandler {
 						self.state = State::Opening { initial_message: initial_message.clone() };
 						let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message);
 						self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-							protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT),
-							info: (),
+							protocol: SubstreamProtocol::new(proto, ()).with_timeout(OPEN_TIMEOUT),
 						});
 						return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed));
 					}
diff --git a/client/network/src/protocol/generic_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs
index dbe02c350100f..daa02efd02a05 100644
--- a/client/network/src/protocol/generic_proto/tests.rs
+++ b/client/network/src/protocol/generic_proto/tests.rs
@@ -20,7 +20,14 @@ use crate::protocol::generic_proto::{GenericProto, GenericProtoOut};
 
 use futures::prelude::*;
 use libp2p::{PeerId, Multiaddr, Transport};
-use libp2p::core::{connection::{ConnectionId, ListenerId}, ConnectedPoint};
+use libp2p::core::{
+	connection::{ConnectionId, ListenerId},
+	ConnectedPoint,
+	muxing,
+	transport::MemoryTransport,
+	upgrade
+};
+use libp2p::{identity, noise, yamux};
 use libp2p::swarm::{
 	Swarm, ProtocolsHandler, IntoProtocolsHandler, PollParameters,
 	NetworkBehaviour, NetworkBehaviourAction
@@ -32,7 +39,7 @@ use std::{error, io, task::Context, task::Poll, time::Duration};
 fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
 	let mut out = Vec::with_capacity(2);
 
-	let keypairs: Vec<_> = (0..2).map(|_| libp2p::identity::Keypair::generate_ed25519()).collect();
+	let keypairs: Vec<_> = (0..2).map(|_| identity::Keypair::generate_ed25519()).collect();
 	let addrs: Vec<Multiaddr> = (0..2)
 		.map(|_| format!("/memory/{}", rand::random::<u64>()).parse().unwrap())
 		.collect();
@@ -40,25 +47,16 @@ fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
 	for index in 0 .. 2 {
 		let keypair = keypairs[index].clone();
 		let local_peer_id = keypair.public().into_peer_id();
-		let transport = libp2p::core::transport::MemoryTransport
-			.and_then(move |out, endpoint| {
-				let secio = libp2p::secio::SecioConfig::new(keypair);
-				libp2p::core::upgrade::apply(
-					out,
-					secio,
-					endpoint,
-					libp2p::core::upgrade::Version::V1
-				)
-			})
-			.and_then(move |(peer_id, stream), endpoint| {
-				libp2p::core::upgrade::apply(
-					stream,
-					libp2p::yamux::Config::default(),
-					endpoint,
-					libp2p::core::upgrade::Version::V1
-				)
-					.map_ok(|muxer| (peer_id, libp2p::core::muxing::StreamMuxerBox::new(muxer)))
-			})
+
+		let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
+			.into_authentic(&keypair)
+			.unwrap();
+
+		let transport = MemoryTransport
+			.upgrade(upgrade::Version::V1)
+			.authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated())
+			.multiplex(yamux::Config::default())
+			.map(|(peer, muxer), _| (peer, muxing::StreamMuxerBox::new(muxer)))
 			.timeout(Duration::from_secs(20))
 			.map_err(|err| io::Error::new(io::ErrorKind::Other, err))
 			.boxed();
diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs
index 3065d83286137..5141e6db70141 100644
--- a/client/network/src/request_responses.rs
+++ b/client/network/src/request_responses.rs
@@ -409,7 +409,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
 						// Received a request from a remote.
 						RequestResponseEvent::Message {
 							peer,
-							message: RequestResponseMessage::Request { request, channel },
+							message: RequestResponseMessage::Request { request, channel, .. },
 						} => {
 							let (tx, rx) = oneshot::channel();
 
@@ -473,7 +473,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
 						}
 
 						// Remote has tried to send a request but failed.
-						RequestResponseEvent::InboundFailure { peer, error } => {
+						RequestResponseEvent::InboundFailure { peer, error, .. } => {
 							let out = Event::InboundRequest {
 								peer,
 								protocol: protocol.clone(),
@@ -660,7 +660,7 @@ mod tests {
 	use libp2p::Multiaddr;
 	use libp2p::core::upgrade;
 	use libp2p::core::transport::{Transport, MemoryTransport};
-	use libp2p::core::upgrade::{InboundUpgradeExt, OutboundUpgradeExt};
+	use libp2p::noise;
 	use libp2p::swarm::{Swarm, SwarmEvent};
 	use std::{iter, time::Duration};
 
@@ -672,25 +672,15 @@ mod tests {
 		let mut swarms = (0..2)
 			.map(|_| {
 				let keypair = Keypair::generate_ed25519();
-				let keypair2 = keypair.clone();
+
+				let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
+					.into_authentic(&keypair)
+					.unwrap();
 
 				let transport = MemoryTransport
-					.and_then(move |out, endpoint| {
-						let secio = libp2p::secio::SecioConfig::new(keypair2);
-						libp2p::core::upgrade::apply(
-							out,
-							secio,
-							endpoint,
-							upgrade::Version::V1
-						)
-					})
-					.and_then(move |(peer_id, stream), endpoint| {
-						let peer_id2 = peer_id.clone();
-						let upgrade = libp2p::yamux::Config::default()
-							.map_inbound(move |muxer| (peer_id, muxer))
-							.map_outbound(move |muxer| (peer_id2, muxer));
-						upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1)
-					});
+					.upgrade(upgrade::Version::V1)
+					.authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated())
+					.multiplex(libp2p::yamux::Config::default());
 
 				let behaviour = {
 					let (tx, mut rx) = mpsc::channel(64);
@@ -784,25 +774,15 @@ mod tests {
 		let mut swarms = (0..2)
 			.map(|_| {
 				let keypair = Keypair::generate_ed25519();
-				let keypair2 = keypair.clone();
+
+				let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
+					.into_authentic(&keypair)
+					.unwrap();
 
 				let transport = MemoryTransport
-					.and_then(move |out, endpoint| {
-						let secio = libp2p::secio::SecioConfig::new(keypair2);
-						libp2p::core::upgrade::apply(
-							out,
-							secio,
-							endpoint,
-							upgrade::Version::V1
-						)
-					})
-					.and_then(move |(peer_id, stream), endpoint| {
-						let peer_id2 = peer_id.clone();
-						let upgrade = libp2p::yamux::Config::default()
-							.map_inbound(move |muxer| (peer_id, muxer))
-							.map_outbound(move |muxer| (peer_id2, muxer));
-						upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1)
-					});
+					.upgrade(upgrade::Version::V1)
+					.authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated())
+					.multiplex(libp2p::yamux::Config::default());
 
 				let behaviour = {
 					let (tx, mut rx) = mpsc::channel(64);
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index f9f877030fe10..7db6ea4972c53 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -1353,6 +1353,8 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 									ResponseFailure::Network(InboundFailure::Timeout) => "timeout",
 									ResponseFailure::Network(InboundFailure::UnsupportedProtocols) =>
 										"unsupported",
+									ResponseFailure::Network(InboundFailure::ConnectionClosed) =>
+										"connection-closed",
 								};
 
 								metrics.requests_in_failure_total
diff --git a/client/network/src/transport.rs b/client/network/src/transport.rs
index e8836c4c269a5..c9226a10a3045 100644
--- a/client/network/src/transport.rs
+++ b/client/network/src/transport.rs
@@ -47,6 +47,7 @@ pub fn build_transport(
 	// Legacy noise configurations for backward compatibility.
 	let mut noise_legacy = noise::LegacyConfig::default();
 	noise_legacy.send_legacy_handshake = true;
+	noise_legacy.recv_legacy_handshake = true;
 
 	// Build configuration objects for encryption mechanisms.
 	let noise_config = {
@@ -76,7 +77,6 @@ pub fn build_transport(
 	mplex_config.max_buffer_len(usize::MAX);
 
 	let mut yamux_config = libp2p::yamux::Config::default();
-	yamux_config.set_lazy_open(true); // Only set SYN flag on first data frame sent to the remote.
 
 	if use_yamux_flow_control {
 		// Enable proper flow-control: window updates are only sent when
diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml
index 7f3f535ebbd8a..fc6c47699fb8f 100644
--- a/client/network/test/Cargo.toml
+++ b/client/network/test/Cargo.toml
@@ -19,7 +19,7 @@ parking_lot = "0.10.0"
 futures = "0.3.4"
 futures-timer = "3.0.1"
 rand = "0.7.2"
-libp2p = { version = "0.24.0", default-features = false }
+libp2p = { version = "0.28.1", default-features = false }
 sp-consensus = { version = "0.8.0-rc6", path = "../../../primitives/consensus/common" }
 sc-consensus = { version = "0.8.0-rc6", path = "../../../client/consensus/common" }
 sc-client-api = { version = "2.0.0-rc6", path = "../../api" }
diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml
index 5856abf4e7edb..13aaae5dba1ec 100644
--- a/client/peerset/Cargo.toml
+++ b/client/peerset/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 futures = "0.3.4"
-libp2p = { version = "0.24.0", default-features = false }
+libp2p = { version = "0.28.1", default-features = false }
 sp-utils = { version = "2.0.0-rc6", path = "../../primitives/utils"}
 log = "0.4.8"
 serde_json = "1.0.41"
diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml
index 3ad82f5612502..d0d05cccf5178 100644
--- a/client/telemetry/Cargo.toml
+++ b/client/telemetry/Cargo.toml
@@ -18,7 +18,7 @@ parking_lot = "0.10.0"
 futures = "0.3.4"
 futures-timer = "3.0.1"
 wasm-timer = "0.2.0"
-libp2p = { version = "0.24.0", default-features = false, features = ["dns", "tcp-async-std", "wasm-ext", "websocket"] }
+libp2p = { version = "0.28.1", default-features = false, features = ["dns", "tcp-async-std", "wasm-ext", "websocket"] }
 log = "0.4.8"
 pin-project = "0.4.6"
 rand = "0.7.2"
diff --git a/primitives/consensus/common/Cargo.toml b/primitives/consensus/common/Cargo.toml
index 7af0cbd949a73..3cb0d5127b23f 100644
--- a/primitives/consensus/common/Cargo.toml
+++ b/primitives/consensus/common/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 derive_more = "0.99.2"
-libp2p = { version = "0.24.0", default-features = false }
+libp2p = { version = "0.28.1", default-features = false }
 log = "0.4.8"
 sp-core = { path= "../../core", version = "2.0.0-rc6"}
 sp-inherents = { version = "2.0.0-rc6", path = "../../inherents" }
diff --git a/utils/browser/Cargo.toml b/utils/browser/Cargo.toml
index 085939ffdcf26..38496b93ab672 100644
--- a/utils/browser/Cargo.toml
+++ b/utils/browser/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 futures = { version = "0.3", features = ["compat"] }
 futures01 = { package = "futures", version = "0.1.29" }
 log = "0.4.8"
-libp2p-wasm-ext = { version = "0.21", features = ["websocket"] }
+libp2p-wasm-ext = { version = "0.22", features = ["websocket"] }
 console_error_panic_hook = "0.1.6"
 console_log = "0.1.2"
 js-sys = "0.3.34"

From c071d06dd2d0ea5b8a3561c635ccd6f82061a2b4 Mon Sep 17 00:00:00 2001
From: Wei Tang <wei@that.world>
Date: Mon, 14 Sep 2020 17:03:26 +0200
Subject: [PATCH 081/122] pow: support uniform tie breaking in fork choice
 (#7073)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* pow: support uniform tie breaking in fork choice

* Update client/consensus/pow/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Refactor fetch seal

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
---
 client/consensus/pow/src/lib.rs | 68 ++++++++++++++++++++++++---------
 1 file changed, 49 insertions(+), 19 deletions(-)

diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs
index ca1a8584e2a0b..70a7bb47873f6 100644
--- a/client/consensus/pow/src/lib.rs
+++ b/client/consensus/pow/src/lib.rs
@@ -37,6 +37,7 @@ use std::borrow::Cow;
 use std::thread;
 use std::collections::HashMap;
 use std::marker::PhantomData;
+use std::cmp::Ordering;
 use sc_client_api::{BlockOf, backend::AuxStore};
 use sp_blockchain::{HeaderBackend, ProvideCache, well_known_cache_keys::Id as CacheKeyId};
 use sp_block_builder::BlockBuilder as BlockBuilderApi;
@@ -170,6 +171,19 @@ pub trait PowAlgorithm<B: BlockT> {
 	) -> Result<Option<bool>, Error<B>> {
 		Ok(None)
 	}
+	/// Break a fork choice tie.
+	///
+	/// By default this chooses the earliest block seen. Using uniform tie
+	/// breaking algorithms will help to protect against selfish mining.
+	///
+	/// Returns if the new seal should be considered best block.
+	fn break_tie(
+		&self,
+		_own_seal: &Seal,
+		_new_seal: &Seal,
+	) -> bool {
+		false
+	}
 	/// Verify that the difficulty is valid against given seal.
 	fn verify(
 		&self,
@@ -194,7 +208,7 @@ pub trait PowAlgorithm<B: BlockT> {
 pub struct PowBlockImport<B: BlockT, I, C, S, Algorithm, CAW> {
 	algorithm: Algorithm,
 	inner: I,
-	select_chain: Option<S>,
+	select_chain: S,
 	client: Arc<C>,
 	inherent_data_providers: sp_inherents::InherentDataProviders,
 	check_inherents_after: <<B as BlockT>::Header as HeaderT>::Number,
@@ -232,7 +246,7 @@ impl<B, I, C, S, Algorithm, CAW> PowBlockImport<B, I, C, S, Algorithm, CAW> wher
 		client: Arc<C>,
 		algorithm: Algorithm,
 		check_inherents_after: <<B as BlockT>::Header as HeaderT>::Number,
-		select_chain: Option<S>,
+		select_chain: S,
 		inherent_data_providers: sp_inherents::InherentDataProviders,
 		can_author_with: CAW,
 	) -> Self {
@@ -324,12 +338,9 @@ impl<B, I, C, S, Algorithm, CAW> BlockImport<B> for PowBlockImport<B, I, C, S, A
 		mut block: BlockImportParams<B, Self::Transaction>,
 		new_cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
-		let best_hash = match self.select_chain.as_ref() {
-			Some(select_chain) => select_chain.best_chain()
-				.map_err(|e| format!("Fetch best chain failed via select chain: {:?}", e))?
-				.hash(),
-			None => self.client.info().best_hash,
-		};
+		let best_header = self.select_chain.best_chain()
+			.map_err(|e| format!("Fetch best chain failed via select chain: {:?}", e))?;
+		let best_hash = best_header.hash();
 
 		let parent_hash = *block.header.parent_hash();
 		let best_aux = PowAux::read::<_, B>(self.client.as_ref(), &best_hash)?;
@@ -352,16 +363,7 @@ impl<B, I, C, S, Algorithm, CAW> BlockImport<B> for PowBlockImport<B, I, C, S, A
 			block.body = Some(check_block.deconstruct().1);
 		}
 
-		let inner_seal = match block.post_digests.last() {
-			Some(DigestItem::Seal(id, seal)) => {
-				if id == &POW_ENGINE_ID {
-					seal.clone()
-				} else {
-					return Err(Error::<B>::WrongEngine(*id).into())
-				}
-			},
-			_ => return Err(Error::<B>::HeaderUnsealed(block.header.hash()).into()),
-		};
+		let inner_seal = fetch_seal::<B>(block.post_digests.last(), block.header.hash())?;
 
 		let intermediate = block.take_intermediate::<PowIntermediate::<Algorithm::Difficulty>>(
 			INTERMEDIATE_KEY
@@ -391,7 +393,18 @@ impl<B, I, C, S, Algorithm, CAW> BlockImport<B> for PowBlockImport<B, I, C, S, A
 		block.auxiliary.push((key, Some(aux.encode())));
 		if block.fork_choice.is_none() {
 			block.fork_choice = Some(ForkChoiceStrategy::Custom(
-				aux.total_difficulty > best_aux.total_difficulty
+				match aux.total_difficulty.cmp(&best_aux.total_difficulty) {
+					Ordering::Less => false,
+					Ordering::Greater => true,
+					Ordering::Equal => {
+						let best_inner_seal = fetch_seal::<B>(
+							best_header.digest().logs.last(),
+							best_hash,
+						)?;
+
+						self.algorithm.break_tie(&best_inner_seal, &inner_seal)
+					},
+				}
 			));
 		}
 
@@ -729,3 +742,20 @@ fn find_pre_digest<B: BlockT>(header: &B::Header) -> Result<Option<Vec<u8>>, Err
 
 	Ok(pre_digest)
 }
+
+/// Fetch PoW seal.
+fn fetch_seal<B: BlockT>(
+	digest: Option<&DigestItem<B::Hash>>,
+	hash: B::Hash,
+) -> Result<Vec<u8>, Error<B>> {
+	match digest {
+		Some(DigestItem::Seal(id, seal)) => {
+			if id == &POW_ENGINE_ID {
+				Ok(seal.clone())
+			} else {
+				return Err(Error::<B>::WrongEngine(*id).into())
+			}
+		},
+		_ => return Err(Error::<B>::HeaderUnsealed(hash).into()),
+	}
+}

From 72ea91f56d768028e508629dc700ae9ff1bfe7eb Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Mon, 14 Sep 2020 17:08:23 +0200
Subject: [PATCH 082/122] Allow remotes to not open a legacy substream (#7075)

* Allow remotes to not open a legacy substream

* Misc fixes

* Special case first protocol as the one bearing the handshake
---
 client/network/src/protocol.rs                | 235 +++++++++---------
 .../src/protocol/generic_proto/behaviour.rs   |   9 +-
 .../protocol/generic_proto/handler/group.rs   |  57 +++--
 .../src/protocol/generic_proto/tests.rs       |   7 +-
 client/network/src/protocol/message.rs        |   6 -
 client/rpc-api/src/system/helpers.rs          |   5 +-
 client/rpc/src/system/tests.rs                |   2 -
 client/service/src/lib.rs                     |   1 -
 8 files changed, 168 insertions(+), 154 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 08de0691b3c94..ae96106473231 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -36,7 +36,7 @@ use sp_consensus::{
 	block_validation::BlockAnnounceValidator,
 	import_queue::{BlockImportResult, BlockImportError, IncomingBlock, Origin}
 };
-use codec::{Decode, Encode};
+use codec::{Decode, DecodeAll, Encode};
 use sp_runtime::{generic::BlockId, ConsensusEngineId, Justification};
 use sp_runtime::traits::{
 	Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub
@@ -53,7 +53,7 @@ use std::borrow::Cow;
 use std::collections::{HashMap, HashSet, VecDeque, hash_map::Entry};
 use std::sync::Arc;
 use std::fmt::Write;
-use std::{io, num::NonZeroUsize, pin::Pin, task::Poll, time};
+use std::{io, iter, num::NonZeroUsize, pin::Pin, task::Poll, time};
 use log::{log, Level, trace, debug, warn, error};
 use wasm_timer::Instant;
 
@@ -275,8 +275,6 @@ struct Peer<B: BlockT, H: ExHashT> {
 pub struct PeerInfo<B: BlockT> {
 	/// Roles
 	pub roles: Roles,
-	/// Protocol version
-	pub protocol_version: u32,
 	/// Peer best block hash
 	pub best_hash: B::Hash,
 	/// Peer best block number
@@ -395,14 +393,6 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		};
 
 		let (peerset, peerset_handle) = sc_peerset::Peerset::from_config(peerset_config);
-		let versions = &((MIN_VERSION as u8)..=(CURRENT_VERSION as u8)).collect::<Vec<u8>>();
-		let mut behaviour = GenericProto::new(
-			local_peer_id,
-			protocol_id.clone(),
-			versions,
-			build_status_message(&config, &chain),
-			peerset,
-		);
 
 		let mut legacy_equiv_by_name = HashMap::new();
 
@@ -413,7 +403,6 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			proto.push_str("/transactions/1");
 			proto
 		});
-		behaviour.register_notif_protocol(transactions_protocol.clone(), Vec::new());
 		legacy_equiv_by_name.insert(transactions_protocol.clone(), Fallback::Transactions);
 
 		let block_announces_protocol: Cow<'static, str> = Cow::from({
@@ -423,12 +412,24 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			proto.push_str("/block-announces/1");
 			proto
 		});
-		behaviour.register_notif_protocol(
-			block_announces_protocol.clone(),
-			BlockAnnouncesHandshake::build(&config, &chain).encode()
-		);
 		legacy_equiv_by_name.insert(block_announces_protocol.clone(), Fallback::BlockAnnounce);
 
+		let behaviour = {
+			let versions = &((MIN_VERSION as u8)..=(CURRENT_VERSION as u8)).collect::<Vec<u8>>();
+			let block_announces_handshake = BlockAnnouncesHandshake::build(&config, &chain).encode();
+			GenericProto::new(
+				local_peer_id,
+				protocol_id.clone(),
+				versions,
+				build_status_message(&config, &chain),
+				peerset,
+				// As documented in `GenericProto`, the first protocol in the list is always the
+				// one carrying the handshake reported in the `CustomProtocolOpen` event.
+				iter::once((block_announces_protocol.clone(), block_announces_handshake))
+					.chain(iter::once((transactions_protocol.clone(), vec![]))),
+			)
+		};
+
 		let protocol = Protocol {
 			tick_timeout: Box::pin(interval(TICK_TIMEOUT)),
 			propagate_timeout: Box::pin(interval(PROPAGATE_TIMEOUT)),
@@ -839,99 +840,86 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		}
 	}
 
-	/// Called on receipt of a status message via the legacy protocol on the first connection between two peers.
-	pub fn on_peer_connected(
+	/// Called on the first connection between two peers, after their exchange of handshake.
+	fn on_peer_connected(
 		&mut self,
 		who: PeerId,
-		status: message::Status<B>,
+		status: BlockAnnouncesHandshake<B>,
 		notifications_sink: NotificationsSink,
 	) -> CustomMessageOutcome<B> {
 		trace!(target: "sync", "New peer {} {:?}", who, status);
-		let _protocol_version = {
-			if self.context_data.peers.contains_key(&who) {
-				debug!(target: "sync", "Ignoring duplicate status packet from {}", who);
-				return CustomMessageOutcome::None;
-			}
-			if status.genesis_hash != self.genesis_hash {
-				log!(
-					target: "sync",
-					if self.important_peers.contains(&who) { Level::Warn } else { Level::Trace },
-					"Peer is on different chain (our genesis: {} theirs: {})",
-					self.genesis_hash, status.genesis_hash
-				);
-				self.peerset_handle.report_peer(who.clone(), rep::GENESIS_MISMATCH);
-				self.behaviour.disconnect_peer(&who);
 
-				if self.boot_node_ids.contains(&who) {
-					error!(
-						target: "sync",
-						"Bootnode with peer id `{}` is on a different chain (our genesis: {} theirs: {})",
-						who,
-						self.genesis_hash,
-						status.genesis_hash,
-					);
-				}
+		if self.context_data.peers.contains_key(&who) {
+			debug!(target: "sync", "Ignoring duplicate status packet from {}", who);
+			return CustomMessageOutcome::None;
+		}
+		if status.genesis_hash != self.genesis_hash {
+			log!(
+				target: "sync",
+				if self.important_peers.contains(&who) { Level::Warn } else { Level::Trace },
+				"Peer is on different chain (our genesis: {} theirs: {})",
+				self.genesis_hash, status.genesis_hash
+			);
+			self.peerset_handle.report_peer(who.clone(), rep::GENESIS_MISMATCH);
+			self.behaviour.disconnect_peer(&who);
 
-				return CustomMessageOutcome::None;
-			}
-			if status.version < MIN_VERSION && CURRENT_VERSION < status.min_supported_version {
-				log!(
+			if self.boot_node_ids.contains(&who) {
+				error!(
 					target: "sync",
-					if self.important_peers.contains(&who) { Level::Warn } else { Level::Trace },
-					"Peer {:?} using unsupported protocol version {}", who, status.version
+					"Bootnode with peer id `{}` is on a different chain (our genesis: {} theirs: {})",
+					who,
+					self.genesis_hash,
+					status.genesis_hash,
 				);
-				self.peerset_handle.report_peer(who.clone(), rep::BAD_PROTOCOL);
-				self.behaviour.disconnect_peer(&who);
-				return CustomMessageOutcome::None;
 			}
 
-			if self.config.roles.is_light() {
-				// we're not interested in light peers
-				if status.roles.is_light() {
-					debug!(target: "sync", "Peer {} is unable to serve light requests", who);
-					self.peerset_handle.report_peer(who.clone(), rep::BAD_ROLE);
-					self.behaviour.disconnect_peer(&who);
-					return CustomMessageOutcome::None;
-				}
+			return CustomMessageOutcome::None;
+		}
 
-				// we don't interested in peers that are far behind us
-				let self_best_block = self
-					.context_data
-					.chain
-					.info()
-					.best_number;
-				let blocks_difference = self_best_block
-					.checked_sub(&status.best_number)
-					.unwrap_or_else(Zero::zero)
-					.saturated_into::<u64>();
-				if blocks_difference > LIGHT_MAXIMAL_BLOCKS_DIFFERENCE {
-					debug!(target: "sync", "Peer {} is far behind us and will unable to serve light requests", who);
-					self.peerset_handle.report_peer(who.clone(), rep::PEER_BEHIND_US_LIGHT);
-					self.behaviour.disconnect_peer(&who);
-					return CustomMessageOutcome::None;
-				}
+		if self.config.roles.is_light() {
+			// we're not interested in light peers
+			if status.roles.is_light() {
+				debug!(target: "sync", "Peer {} is unable to serve light requests", who);
+				self.peerset_handle.report_peer(who.clone(), rep::BAD_ROLE);
+				self.behaviour.disconnect_peer(&who);
+				return CustomMessageOutcome::None;
 			}
 
-			let peer = Peer {
-				info: PeerInfo {
-					protocol_version: status.version,
-					roles: status.roles,
-					best_hash: status.best_hash,
-					best_number: status.best_number
-				},
-				block_request: None,
-				known_transactions: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS)
-					.expect("Constant is nonzero")),
-				known_blocks: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_BLOCKS)
-					.expect("Constant is nonzero")),
-				next_request_id: 0,
-				obsolete_requests: HashMap::new(),
-			};
-			self.context_data.peers.insert(who.clone(), peer);
+			// we don't interested in peers that are far behind us
+			let self_best_block = self
+				.context_data
+				.chain
+				.info()
+				.best_number;
+			let blocks_difference = self_best_block
+				.checked_sub(&status.best_number)
+				.unwrap_or_else(Zero::zero)
+				.saturated_into::<u64>();
+			if blocks_difference > LIGHT_MAXIMAL_BLOCKS_DIFFERENCE {
+				debug!(target: "sync", "Peer {} is far behind us and will unable to serve light requests", who);
+				self.peerset_handle.report_peer(who.clone(), rep::PEER_BEHIND_US_LIGHT);
+				self.behaviour.disconnect_peer(&who);
+				return CustomMessageOutcome::None;
+			}
+		}
 
-			debug!(target: "sync", "Connected {}", who);
-			status.version
+		let peer = Peer {
+			info: PeerInfo {
+				roles: status.roles,
+				best_hash: status.best_hash,
+				best_number: status.best_number
+			},
+			block_request: None,
+			known_transactions: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS)
+				.expect("Constant is nonzero")),
+			known_blocks: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_BLOCKS)
+				.expect("Constant is nonzero")),
+			next_request_id: 0,
+			obsolete_requests: HashMap::new(),
 		};
+		self.context_data.peers.insert(who.clone(), peer);
+
+		debug!(target: "sync", "Connected {}", who);
 
 		let info = self.context_data.peers.get(&who).expect("We just inserted above; QED").info.clone();
 		self.pending_messages.push_back(CustomMessageOutcome::PeerNewBest(who.clone(), status.best_number));
@@ -1161,20 +1149,12 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			if inserted || force {
 				let message = message::BlockAnnounce {
 					header: header.clone(),
-					state: if peer.info.protocol_version >= 4  {
-						if is_best {
-							Some(message::BlockState::Best)
-						} else {
-							Some(message::BlockState::Normal)
-						}
-					} else  {
-						None
-					},
-					data: if peer.info.protocol_version >= 4 {
-						Some(data.clone())
+					state: if is_best {
+						Some(message::BlockState::Best)
 					} else {
-						None
+						Some(message::BlockState::Normal)
 					},
+					data: Some(data.clone()),
 				};
 
 				self.behaviour.write_notification(
@@ -1620,9 +1600,20 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 
 		let outcome = match event {
 			GenericProtoOut::CustomProtocolOpen { peer_id, received_handshake, notifications_sink, .. } => {
-				match <Message<B> as Decode>::decode(&mut &received_handshake[..]) {
-					Ok(GenericMessage::Status(handshake)) =>
-						self.on_peer_connected(peer_id, handshake, notifications_sink),
+				// `received_handshake` can be either a `Status` message if received from the
+				// legacy substream ,or a `BlockAnnouncesHandshake` if received from the block
+				// announces substream.
+				match <Message<B> as DecodeAll>::decode_all(&mut &received_handshake[..]) {
+					Ok(GenericMessage::Status(handshake)) => {
+						let handshake = BlockAnnouncesHandshake {
+							roles: handshake.roles,
+							best_number: handshake.best_number,
+							best_hash: handshake.best_hash,
+							genesis_hash: handshake.genesis_hash,
+						};
+
+						self.on_peer_connected(peer_id, handshake, notifications_sink)
+					},
 					Ok(msg) => {
 						debug!(
 							target: "sync",
@@ -1634,15 +1625,23 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 						CustomMessageOutcome::None
 					}
 					Err(err) => {
-						debug!(
-							target: "sync",
-							"Couldn't decode handshake sent by {}: {:?}: {}",
-							peer_id,
-							received_handshake,
-							err.what()
-						);
-						self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE);
-						CustomMessageOutcome::None
+						match <BlockAnnouncesHandshake<B> as DecodeAll>::decode_all(&mut &received_handshake[..]) {
+							Ok(handshake) => {
+								self.on_peer_connected(peer_id, handshake, notifications_sink)
+							}
+							Err(err2) => {
+								debug!(
+									target: "sync",
+									"Couldn't decode handshake sent by {}: {:?}: {} & {}",
+									peer_id,
+									received_handshake,
+									err.what(),
+									err2,
+								);
+								self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE);
+								CustomMessageOutcome::None
+							}
+						}
 					}
 				}
 			}
diff --git a/client/network/src/protocol/generic_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs
index 996a810605d13..e7e2cb035d65c 100644
--- a/client/network/src/protocol/generic_proto/behaviour.rs
+++ b/client/network/src/protocol/generic_proto/behaviour.rs
@@ -336,14 +336,21 @@ impl GenericProto {
 		versions: &[u8],
 		handshake_message: Vec<u8>,
 		peerset: sc_peerset::Peerset,
+		notif_protocols: impl Iterator<Item = (Cow<'static, str>, Vec<u8>)>,
 	) -> Self {
+		let notif_protocols = notif_protocols
+			.map(|(n, hs)| (n, Arc::new(RwLock::new(hs))))
+			.collect::<Vec<_>>();
+
+		assert!(!notif_protocols.is_empty());
+
 		let legacy_handshake_message = Arc::new(RwLock::new(handshake_message));
 		let legacy_protocol = RegisteredProtocol::new(protocol, versions, legacy_handshake_message);
 
 		GenericProto {
 			local_peer_id,
 			legacy_protocol,
-			notif_protocols: Vec::new(),
+			notif_protocols,
 			peerset,
 			peers: FnvHashMap::default(),
 			delays: Default::default(),
diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs
index bcdba87e1037c..f355fba60fb04 100644
--- a/client/network/src/protocol/generic_proto/handler/group.rs
+++ b/client/network/src/protocol/generic_proto/handler/group.rs
@@ -113,10 +113,11 @@ pub struct NotifsHandler {
 	/// Handler for backwards-compatibility.
 	legacy: LegacyProtoHandler,
 
-	/// In the situation where `legacy.is_open()` is true, but we haven't sent out any
-	/// [`NotifsHandlerOut::Open`] event yet, this contains the handshake received on the legacy
-	/// substream.
-	pending_legacy_handshake: Option<Vec<u8>>,
+	/// In the situation where either the legacy substream has been opened or the handshake-bearing
+	/// notifications protocol is open, but we haven't sent out any [`NotifsHandlerOut::Open`]
+	/// event yet, this contains the received handshake waiting to be reported through the
+	/// external API.
+	pending_handshake: Option<Vec<u8>>,
 
 	/// State of this handler.
 	enabled: EnabledState,
@@ -172,7 +173,7 @@ impl IntoProtocolsHandler for NotifsHandlerProto {
 				.collect(),
 			endpoint: connected_point.clone(),
 			legacy: self.legacy.into_handler(remote_peer_id, connected_point),
-			pending_legacy_handshake: None,
+			pending_handshake: None,
 			enabled: EnabledState::Initial,
 			pending_in: Vec::new(),
 			notifications_sink_rx: None,
@@ -360,11 +361,20 @@ impl NotifsHandlerProto {
 	/// `list` is a list of notification protocols names, and the message to send as part of the
 	/// handshake. At the moment, the message is always the same whether we open a substream
 	/// ourselves or respond to handshake from the remote.
+	///
+	/// The first protocol in `list` is special-cased as the protocol that contains the handshake
+	/// to report through the [`NotifsHandlerOut::Open`] event.
+	///
+	/// # Panic
+	///
+	/// - Panics if `list` is empty.
+	///
 	pub fn new(
 		legacy: RegisteredProtocol,
 		list: impl Into<Vec<(Cow<'static, str>, Arc<RwLock<Vec<u8>>>)>>,
 	) -> Self {
 		let list = list.into();
+		assert!(!list.is_empty());
 
 		let out_handlers = list
 			.clone()
@@ -614,11 +624,12 @@ impl ProtocolsHandler for NotifsHandler {
 			}
 		}
 
-		// If `self.pending_legacy_handshake` is `Some`, we are in a state where the legacy
-		// substream is open but the user isn't aware yet of the substreams being open.
+		// If `self.pending_handshake` is `Some`, we are in a state where the handshake-bearing
+		// substream (either the legacy substream or the one special-cased as providing the
+		// handshake) is open but the user isn't aware yet of the substreams being open.
 		// When that is the case, neither the legacy substream nor the incoming notifications
 		// substreams should be polled, otherwise there is a risk of receiving messages from them.
-		if self.pending_legacy_handshake.is_none() {
+		if self.pending_handshake.is_none() {
 			while let Poll::Ready(ev) = self.legacy.poll(cx) {
 				match ev {
 					ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol } =>
@@ -631,14 +642,16 @@ impl ProtocolsHandler for NotifsHandler {
 						received_handshake,
 						..
 					}) => {
-						self.pending_legacy_handshake = Some(received_handshake);
+						if self.notifications_sink_rx.is_none() {
+							debug_assert!(self.pending_handshake.is_none());
+							self.pending_handshake = Some(received_handshake);
+						}
 						cx.waker().wake_by_ref();
 						return Poll::Pending;
 					},
 					ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { reason, .. }) => {
 						// We consciously drop the receivers despite notifications being potentially
 						// still buffered up.
-						debug_assert!(self.notifications_sink_rx.is_some());
 						self.notifications_sink_rx = None;
 
 						return Poll::Ready(ProtocolsHandlerEvent::Custom(
@@ -646,7 +659,6 @@ impl ProtocolsHandler for NotifsHandler {
 						))
 					},
 					ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomMessage { message }) => {
-						debug_assert!(self.notifications_sink_rx.is_some());
 						return Poll::Ready(ProtocolsHandlerEvent::Custom(
 							NotifsHandlerOut::CustomMessage { message }
 						))
@@ -663,7 +675,7 @@ impl ProtocolsHandler for NotifsHandler {
 
 		for (handler_num, (handler, handshake_message)) in self.in_handlers.iter_mut().enumerate() {
 			loop {
-				let poll = if self.pending_legacy_handshake.is_none() {
+				let poll = if self.notifications_sink_rx.is_some() {
 					handler.poll(cx)
 				} else {
 					handler.poll_process(cx)
@@ -692,7 +704,7 @@ impl ProtocolsHandler for NotifsHandler {
 						},
 					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed) => {},
 					ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(message)) => {
-						debug_assert!(self.pending_legacy_handshake.is_none());
+						debug_assert!(self.pending_handshake.is_none());
 						if self.notifications_sink_rx.is_some() {
 							let msg = NotifsHandlerOut::Notification {
 								message,
@@ -716,12 +728,17 @@ impl ProtocolsHandler for NotifsHandler {
 						}),
 					ProtocolsHandlerEvent::Close(err) => void::unreachable(err),
 
-					// At the moment we don't actually care whether any notifications protocol
-					// opens or closes.
-					// Whether our communications with the remote are open or closed entirely
-					// depends on the legacy substream, because as long as we are open the user of
-					// this struct might try to send legacy protocol messages which we need to
-					// deliver for things to work properly.
+					// Opened substream on the handshake-bearing notification protocol.
+					ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Open { handshake })
+						if handler_num == 0 =>
+					{
+						if self.notifications_sink_rx.is_none() && self.pending_handshake.is_none() {
+							self.pending_handshake = Some(handshake);
+						}
+					},
+
+					// Nothing to do in response to other notification substreams being opened
+					// or closed.
 					ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Open { .. }) => {},
 					ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed) => {},
 					ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Refused) => {},
@@ -730,7 +747,7 @@ impl ProtocolsHandler for NotifsHandler {
 		}
 
 		if self.out_handlers.iter().all(|(h, _)| h.is_open() || h.is_refused()) {
-			if let Some(handshake) = self.pending_legacy_handshake.take() {
+			if let Some(handshake) = self.pending_handshake.take() {
 				let (async_tx, async_rx) = mpsc::channel(ASYNC_NOTIFICATIONS_BUFFER_SIZE);
 				let (sync_tx, sync_rx) = mpsc::channel(SYNC_NOTIFICATIONS_BUFFER_SIZE);
 				let notifications_sink = NotificationsSink {
diff --git a/client/network/src/protocol/generic_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs
index daa02efd02a05..d604645d4ac87 100644
--- a/client/network/src/protocol/generic_proto/tests.rs
+++ b/client/network/src/protocol/generic_proto/tests.rs
@@ -32,7 +32,7 @@ use libp2p::swarm::{
 	Swarm, ProtocolsHandler, IntoProtocolsHandler, PollParameters,
 	NetworkBehaviour, NetworkBehaviourAction
 };
-use std::{error, io, task::Context, task::Poll, time::Duration};
+use std::{error, io, iter, task::{Context, Poll}, time::Duration};
 
 /// Builds two nodes that have each other as bootstrap nodes.
 /// This is to be used only for testing, and a panic will happen if something goes wrong.
@@ -78,7 +78,10 @@ fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
 		});
 
 		let behaviour = CustomProtoWithAddr {
-			inner: GenericProto::new(local_peer_id, "test", &[1], vec![], peerset),
+			inner: GenericProto::new(
+				local_peer_id, "test", &[1], vec![], peerset,
+				iter::once(("/foo".into(), Vec::new()))
+			),
 			addrs: addrs
 				.iter()
 				.enumerate()
diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs
index a7fbb92387cf6..1cd78c0ed1dda 100644
--- a/client/network/src/protocol/message.rs
+++ b/client/network/src/protocol/message.rs
@@ -41,12 +41,6 @@ pub type Message<B> = generic::Message<
 	<B as BlockT>::Extrinsic,
 >;
 
-/// Type alias for using the status type using block type parameters.
-pub type Status<B> = generic::Status<
-	<B as BlockT>::Hash,
-	<<B as BlockT>::Header as HeaderT>::Number,
->;
-
 /// Type alias for using the block request type using block type parameters.
 pub type BlockRequest<B> = generic::BlockRequest<
 	<B as BlockT>::Hash,
diff --git a/client/rpc-api/src/system/helpers.rs b/client/rpc-api/src/system/helpers.rs
index 5dbe93543d8e5..dd3294c243116 100644
--- a/client/rpc-api/src/system/helpers.rs
+++ b/client/rpc-api/src/system/helpers.rs
@@ -67,8 +67,6 @@ pub struct PeerInfo<Hash, Number> {
 	pub peer_id: String,
 	/// Roles
 	pub roles: String,
-	/// Protocol version
-	pub protocol_version: u32,
 	/// Peer best block hash
 	pub best_hash: Hash,
 	/// Peer best block number
@@ -110,11 +108,10 @@ mod tests {
 			::serde_json::to_string(&PeerInfo {
 				peer_id: "2".into(),
 				roles: "a".into(),
-				protocol_version: 2,
 				best_hash: 5u32,
 				best_number: 6u32,
 			}).unwrap(),
-			r#"{"peerId":"2","roles":"a","protocolVersion":2,"bestHash":5,"bestNumber":6}"#,
+			r#"{"peerId":"2","roles":"a","bestHash":5,"bestNumber":6}"#,
 		);
 	}
 }
diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs
index 099504bb009e6..f16d7da5b1a8b 100644
--- a/client/rpc/src/system/tests.rs
+++ b/client/rpc/src/system/tests.rs
@@ -73,7 +73,6 @@ fn api<T: Into<Option<Status>>>(sync: T) -> System<Block> {
 						peers.push(PeerInfo {
 							peer_id: status.peer_id.to_base58(),
 							roles: format!("{}", Role::Full),
-							protocol_version: 1,
 							best_hash: Default::default(),
 							best_number: 1,
 						});
@@ -259,7 +258,6 @@ fn system_peers() {
 		vec![PeerInfo {
 			peer_id: peer_id.to_base58(),
 			roles: "FULL".into(),
-			protocol_version: 1,
 			best_hash: Default::default(),
 			best_number: 1u64,
 		}]
diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs
index fac09beb8bd60..d5d503d22d171 100644
--- a/client/service/src/lib.rs
+++ b/client/service/src/lib.rs
@@ -285,7 +285,6 @@ async fn build_network_future<
 							sc_rpc::system::PeerInfo {
 								peer_id: peer_id.to_base58(),
 								roles: format!("{:?}", p.roles),
-								protocol_version: p.protocol_version,
 								best_hash: p.best_hash,
 								best_number: p.best_number,
 							}

From f5e0c6378cc501364dd4938ec4bef566b183dde1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Tue, 15 Sep 2020 00:10:02 +0200
Subject: [PATCH 083/122] Use diener for Polkadot companion prs (#7102)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Use diener for Polkadot companion prs

* Fix script

* Use gitlab env variable

* Update .maintain/gitlab/check_polkadot_companion_build.sh

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
---
 .../gitlab/check_polkadot_companion_build.sh  | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh
index b78c26dea8458..2ee1e824aed5d 100755
--- a/.maintain/gitlab/check_polkadot_companion_build.sh
+++ b/.maintain/gitlab/check_polkadot_companion_build.sh
@@ -1,9 +1,9 @@
-#!/bin/sh
+#!/usr/bin/env sh
 #
-# check if a pr is compatible with polkadot companion pr or master if not 
+# check if a pr is compatible with polkadot companion pr or master if not
 # available
 #
-# to override one that was just mentioned mark companion pr in the body of the 
+# to override one that was just mentioned mark companion pr in the body of the
 # polkadot pr like
 #
 # polkadot companion: paritytech/polkadot#567
@@ -12,7 +12,7 @@
 
 github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls"
 # use github api v3 in order to access the data without authentication
-github_header="Authorization: token ${GITHUB_PR_TOKEN}" 
+github_header="Authorization: token ${GITHUB_PR_TOKEN}"
 
 boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
 boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
@@ -40,7 +40,7 @@ EOT
 git config --global user.name 'CI system'
 git config --global user.email '<>'
 
-SUBSTRATE_PATH=$(pwd)
+cargo install -f --version 0.2.0 diener
 
 # Merge master into our branch before building Polkadot to make sure we don't miss
 # any commits that are required by Polkadot.
@@ -85,14 +85,9 @@ else
   boldprint "this is not a pull request - building polkadot:master"
 fi
 
-# Make sure we override the crates in native and wasm build
-# patching the git path as described in the link below did not test correctly
-# https://doc.rust-lang.org/cargo/reference/overriding-dependencies.html
-mkdir .cargo
-echo "paths = [ \"$SUBSTRATE_PATH\" ]" > .cargo/config
-
-mkdir -p target/debug/wbuild/.cargo
-cp .cargo/config target/debug/wbuild/.cargo/config
+cd ..
+$CARGO_HOME/bin/diener --substrate --branch $CI_COMMIT_REF_NAME --git https://gitlab.parity.io/parity/substrate.git --path polkadot
+cd polkadot
 
 # Test Polkadot pr or master branch with this Substrate commit.
 time cargo test --all --release --verbose

From b2b0db5fd2a683863348924e7014d802638f85a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Tue, 15 Sep 2020 07:54:35 +0200
Subject: [PATCH 084/122] Update ui tests for rust 1.46.0 (#7106)

---
 .../api/test/tests/ui/mock_only_self_reference.stderr     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/primitives/api/test/tests/ui/mock_only_self_reference.stderr b/primitives/api/test/tests/ui/mock_only_self_reference.stderr
index 6d1ac0e9a2563..ed5b64144a6f6 100644
--- a/primitives/api/test/tests/ui/mock_only_self_reference.stderr
+++ b/primitives/api/test/tests/ui/mock_only_self_reference.stderr
@@ -24,8 +24,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr
 12 |   sp_api::mock_impl_runtime_apis! {
    |   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()`
    |
-   = note: expected fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<_>) -> std::result::Result<_, _>`
-              found fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<()>, std::vec::Vec<_>) -> std::result::Result<_, _>`
+   = note: expected fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime_client::substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<_>) -> std::result::Result<_, _>`
+              found fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime_client::substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<()>, std::vec::Vec<_>) -> std::result::Result<_, _>`
    = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for trait
@@ -42,6 +42,6 @@ error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for t
 12 |   sp_api::mock_impl_runtime_apis! {
    |   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()`
    |
-   = note: expected fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<_>) -> std::result::Result<_, _>`
-              found fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<()>, std::vec::Vec<_>) -> std::result::Result<_, _>`
+   = note: expected fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime_client::substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<_>) -> std::result::Result<_, _>`
+              found fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime_client::substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<()>, std::vec::Vec<_>) -> std::result::Result<_, _>`
    = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)

From 9d9cf8a7d563c41390eb51304dfa3009840a0e9f Mon Sep 17 00:00:00 2001
From: Max Inden <mail@max-inden.de>
Date: Tue, 15 Sep 2020 11:34:22 +0200
Subject: [PATCH 085/122] client/network: Expose number of entries per Kademlia
 bucket (#7104)

Extend `sub_libp2p_kbuckets_num_nodes` Prometheus metric to expose the
number of nodes per bucket per Kademlia instance instead of only per
Kademlia instance.
---
 client/network/src/behaviour.rs       |  9 ++++++---
 client/network/src/discovery.rs       | 14 +++++++++++---
 client/network/src/service.rs         |  8 ++++++--
 client/network/src/service/metrics.rs |  5 ++---
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs
index 2c399cfdf7707..6b3cfac38ae99 100644
--- a/client/network/src/behaviour.rs
+++ b/client/network/src/behaviour.rs
@@ -211,9 +211,12 @@ impl<B: BlockT, H: ExHashT> Behaviour<B, H> {
 		self.discovery.add_known_address(peer_id, addr)
 	}
 
-	/// Returns the number of nodes that are in the Kademlia k-buckets.
-	pub fn num_kbuckets_entries(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, usize)> {
-		self.discovery.num_kbuckets_entries()
+	/// Returns the number of nodes in each Kademlia kbucket for each Kademlia instance.
+	///
+	/// Identifies Kademlia instances by their [`ProtocolId`] and kbuckets by the base 2 logarithm
+	/// of their lower bound.
+	pub fn num_entries_per_kbucket(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, Vec<(u32, usize)>)> {
+		self.discovery.num_entries_per_kbucket()
 	}
 
 	/// Returns the number of records in the Kademlia record stores.
diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs
index bb68468475fa9..6ef97708c1336 100644
--- a/client/network/src/discovery.rs
+++ b/client/network/src/discovery.rs
@@ -329,10 +329,18 @@ impl DiscoveryBehaviour {
 		}
 	}
 
-	/// Returns the number of nodes that are in the Kademlia k-buckets.
-	pub fn num_kbuckets_entries(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, usize)> {
+	/// Returns the number of nodes in each Kademlia kbucket for each Kademlia instance.
+	///
+	/// Identifies Kademlia instances by their [`ProtocolId`] and kbuckets by the base 2 logarithm
+	/// of their lower bound.
+	pub fn num_entries_per_kbucket(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, Vec<(u32, usize)>)> {
 		self.kademlias.iter_mut()
-			.map(|(id, kad)| (id, kad.kbuckets().map(|bucket| bucket.iter().count()).sum()))
+			.map(|(id, kad)| {
+				let buckets = kad.kbuckets()
+					.map(|bucket| (bucket.range().0.ilog2().unwrap_or(0), bucket.iter().count()))
+					.collect();
+				(id, buckets)
+			})
 	}
 
 	/// Returns the number of records in the Kademlia record stores.
diff --git a/client/network/src/service.rs b/client/network/src/service.rs
index 7db6ea4972c53..59f55f01a45d1 100644
--- a/client/network/src/service.rs
+++ b/client/network/src/service.rs
@@ -1670,8 +1670,12 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 		this.is_major_syncing.store(is_major_syncing, Ordering::Relaxed);
 
 		if let Some(metrics) = this.metrics.as_ref() {
-			for (proto, num_entries) in this.network_service.num_kbuckets_entries() {
-				metrics.kbuckets_num_nodes.with_label_values(&[&proto.as_ref()]).set(num_entries as u64);
+			for (proto, buckets) in this.network_service.num_entries_per_kbucket() {
+				for (lower_ilog2_bucket_bound, num_entries) in buckets {
+					metrics.kbuckets_num_nodes
+						.with_label_values(&[&proto.as_ref(), &lower_ilog2_bucket_bound.to_string()])
+						.set(num_entries as u64);
+				}
 			}
 			for (proto, num_entries) in this.network_service.num_kademlia_records() {
 				metrics.kademlia_records_count.with_label_values(&[&proto.as_ref()]).set(num_entries as u64);
diff --git a/client/network/src/service/metrics.rs b/client/network/src/service/metrics.rs
index bbb0ba8056615..a63ce7a18a519 100644
--- a/client/network/src/service/metrics.rs
+++ b/client/network/src/service/metrics.rs
@@ -171,9 +171,9 @@ impl Metrics {
 			kbuckets_num_nodes: prometheus::register(GaugeVec::new(
 				Opts::new(
 					"sub_libp2p_kbuckets_num_nodes",
-					"Number of nodes in the Kademlia k-buckets"
+					"Number of nodes per kbucket per Kademlia instance"
 				),
-				&["protocol"]
+				&["protocol", "lower_ilog2_bucket_bound"]
 			)?, registry)?,
 			listeners_local_addresses: prometheus::register(Gauge::new(
 				"sub_libp2p_listeners_local_addresses", "Number of local addresses we're listening on"
@@ -355,4 +355,3 @@ impl MetricSource for NumConnectedGauge {
 		set(&[], self.0.load(Ordering::Relaxed) as u64);
 	}
 }
-

From c2ef92d442eb44be1a97dc44ff018bded78c0365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Tue, 15 Sep 2020 11:42:23 +0200
Subject: [PATCH 086/122] Improve error output of wasm-builder when wasm ins't
 installed (#7105)

This improves the error message of wasm-builder when the wasm toolchain
isn't installed. Currently we print that the wasm toolchain is not
installed, but the actual problem is that there is a bug in the
packaging in rust. This will now be much easier to debug, by printing
the full error message of the compiler.
---
 Cargo.lock                              |  1 +
 utils/wasm-builder/Cargo.toml           |  1 +
 utils/wasm-builder/src/lib.rs           |  7 +++-
 utils/wasm-builder/src/prerequisites.rs | 47 +++++++++++++++++++------
 utils/wasm-builder/src/wasm_project.rs  |  2 +-
 5 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c33ff0020acae..d8602d2c6f3f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8858,6 +8858,7 @@ dependencies = [
 name = "substrate-wasm-builder"
 version = "2.0.0"
 dependencies = [
+ "ansi_term 0.12.1",
  "atty",
  "build-helper",
  "cargo_metadata",
diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml
index 5e90625620508..de0f11e84670d 100644
--- a/utils/wasm-builder/Cargo.toml
+++ b/utils/wasm-builder/Cargo.toml
@@ -22,3 +22,4 @@ fs2 = "0.4.3"
 wasm-gc-api = "0.1.11"
 atty = "0.2.13"
 itertools = "0.8.2"
+ansi_term = "0.12.1"
diff --git a/utils/wasm-builder/src/lib.rs b/utils/wasm-builder/src/lib.rs
index 500025c29678e..bb50729f71ce5 100644
--- a/utils/wasm-builder/src/lib.rs
+++ b/utils/wasm-builder/src/lib.rs
@@ -179,7 +179,7 @@ pub fn build_project_with_default_rustflags(
 			bloaty.wasm_binary_bloaty_path_escaped(),
 		)
 	};
-	
+
 	write_file_if_changed(
 			file_name.into(),
 			format!(
@@ -309,3 +309,8 @@ impl CargoCommand {
 				.contains("-nightly")
 	}
 }
+
+/// Returns `true` when color output is enabled.
+fn color_output_enabled() -> bool {
+	env::var(crate::WASM_BUILD_NO_COLOR).is_err()
+}
diff --git a/utils/wasm-builder/src/prerequisites.rs b/utils/wasm-builder/src/prerequisites.rs
index d7c15095762e8..2a9801744c45b 100644
--- a/utils/wasm-builder/src/prerequisites.rs
+++ b/utils/wasm-builder/src/prerequisites.rs
@@ -18,14 +18,24 @@
 use std::fs;
 
 use tempfile::tempdir;
+use ansi_term::Color;
+
+/// Print an error message.
+fn print_error_message(message: &str) -> String {
+	if super::color_output_enabled() {
+		Color::Red.bold().paint(message).to_string()
+	} else {
+		message.into()
+	}
+}
 
 /// Checks that all prerequisites are installed.
 ///
 /// # Returns
 /// Returns `None` if everything was found and `Some(ERR_MSG)` if something could not be found.
-pub fn check() -> Option<&'static str> {
+pub fn check() -> Option<String> {
 	if !check_nightly_installed(){
-		return Some("Rust nightly not installed, please install it!")
+		return Some(print_error_message("Rust nightly not installed, please install it!"))
 	}
 
 	check_wasm_toolchain_installed()
@@ -35,7 +45,7 @@ fn check_nightly_installed() -> bool {
 	crate::get_nightly_cargo().is_nightly()
 }
 
-fn check_wasm_toolchain_installed() -> Option<&'static str> {
+fn check_wasm_toolchain_installed() -> Option<String> {
 	let temp = tempdir().expect("Creating temp dir does not fail; qed");
 	fs::create_dir_all(temp.path().join("src")).expect("Creating src dir does not fail; qed");
 
@@ -59,22 +69,39 @@ fn check_wasm_toolchain_installed() -> Option<&'static str> {
 	fs::write(&test_file, "pub fn test() {}")
 		.expect("Writing to the test file does not fail; qed");
 
-	let err_msg = "Rust WASM toolchain not installed, please install it!";
+	let err_msg = print_error_message("Rust WASM toolchain not installed, please install it!");
 	let manifest_path = manifest_path.display().to_string();
-	crate::get_nightly_cargo()
-		.command()
-		.args(&["build", "--target=wasm32-unknown-unknown", "--manifest-path", &manifest_path])
+
+	let mut build_cmd = crate::get_nightly_cargo().command();
+
+	build_cmd.args(&["build", "--target=wasm32-unknown-unknown", "--manifest-path", &manifest_path]);
+
+	if super::color_output_enabled() {
+		build_cmd.arg("--color=always");
+	}
+
+	build_cmd
 		.output()
-		.map_err(|_| err_msg)
+		.map_err(|_| err_msg.clone())
 		.and_then(|s|
 			if s.status.success() {
 				Ok(())
 			} else {
 				match String::from_utf8(s.stderr) {
 					Ok(ref err) if err.contains("linker `rust-lld` not found") => {
-						Err("`rust-lld` not found, please install it!")
+						Err(print_error_message("`rust-lld` not found, please install it!"))
 					},
-					_ => Err(err_msg)
+					Ok(ref err) => Err(
+						format!(
+							"{}\n\n{}\n{}\n{}{}\n",
+							err_msg,
+							Color::Yellow.bold().paint("Further error information:"),
+							Color::Yellow.bold().paint("-".repeat(60)),
+							err,
+							Color::Yellow.bold().paint("-".repeat(60)),
+						)
+					),
+					Err(_) => Err(err_msg),
 				}
 			}
 		)
diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs
index 4c927f7bdeabd..1d4a4484cf45e 100644
--- a/utils/wasm-builder/src/wasm_project.rs
+++ b/utils/wasm-builder/src/wasm_project.rs
@@ -449,7 +449,7 @@ fn build_project(project: &Path, default_rustflags: &str) {
 		// We don't want to call ourselves recursively
 		.env(crate::SKIP_BUILD_ENV, "");
 
-	if env::var(crate::WASM_BUILD_NO_COLOR).is_err() {
+	if super::color_output_enabled() {
 		build_cmd.arg("--color=always");
 	}
 

From b805fafb34c59e5bc12bbdbca5ad7078c8f8e414 Mon Sep 17 00:00:00 2001
From: DarkPay <42247799+DarkPayCoin@users.noreply.github.com>
Date: Tue, 15 Sep 2020 12:52:04 +0200
Subject: [PATCH 087/122] Add ss58 address for Dark network (#6982)

Hello,
This PR adds a new ss58 address 17 for Dark network.
Thanks!
---
 primitives/core/src/crypto.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs
index 527808fab9ccc..e710f346efb31 100644
--- a/primitives/core/src/crypto.rs
+++ b/primitives/core/src/crypto.rs
@@ -472,6 +472,8 @@ ss58_address_format!(
 		(13, "substratee", "Any SubstraTEE off-chain network private account (*25519).")
 	KulupuAccount =>
 		(16, "kulupu", "Kulupu mainnet, standard account (*25519).")
+	DarkAccount =>
+		(17, "dark", "Dark mainnet, standard account (*25519).")
 	DarwiniaAccount =>
 		(18, "darwinia", "Darwinia Chain mainnet, standard account (*25519).")
 	StafiAccount =>

From c867bc2ec7ab1f4868a69d1bcc65430cff53680d Mon Sep 17 00:00:00 2001
From: Guillaume Thiolliere <gui.thiolliere@gmail.com>
Date: Tue, 15 Sep 2020 13:39:52 +0200
Subject: [PATCH 088/122] Frame-support storage: make iterations and translate
 consistent (#5470)

* implementation and factorisation

* factorize test

* doc

* fix bug and improve test

* address suggestions
---
 frame/support/src/dispatch.rs                 |   2 +
 .../src/storage/generator/double_map.rs       | 163 +++++++++---------
 frame/support/src/storage/generator/map.rs    | 153 ++++++++++++----
 frame/support/src/storage/mod.rs              | 138 ++++++++-------
 4 files changed, 288 insertions(+), 168 deletions(-)

diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs
index 181a1597a0451..5446b4a59bd66 100644
--- a/frame/support/src/dispatch.rs
+++ b/frame/support/src/dispatch.rs
@@ -2067,6 +2067,7 @@ macro_rules! __dispatch_impl_metadata {
 			where $( $other_where_bounds )*
 		{
 			#[doc(hidden)]
+			#[allow(dead_code)]
 			pub fn call_functions() -> &'static [$crate::dispatch::FunctionMetadata] {
 				$crate::__call_to_functions!($($rest)*)
 			}
@@ -2140,6 +2141,7 @@ macro_rules! __impl_module_constants_metadata {
 			$mod_type<$trait_instance $(, $instance)?> where $( $other_where_bounds )*
 		{
 			#[doc(hidden)]
+			#[allow(dead_code)]
 			pub fn module_constants_metadata() -> &'static [$crate::dispatch::ModuleConstantMetadata] {
 				// Create the `ByteGetter`s
 				$(
diff --git a/frame/support/src/storage/generator/double_map.rs b/frame/support/src/storage/generator/double_map.rs
index 3c82f4156a271..9454ab401da28 100644
--- a/frame/support/src/storage/generator/double_map.rs
+++ b/frame/support/src/storage/generator/double_map.rs
@@ -18,7 +18,7 @@
 use sp_std::prelude::*;
 use sp_std::borrow::Borrow;
 use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike};
-use crate::{storage::{self, unhashed, StorageAppend}, Never};
+use crate::{storage::{self, unhashed, StorageAppend, PrefixIterator}, Never};
 use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher};
 
 /// Generator for `StorageDoubleMap` used by `decl_storage`.
@@ -213,10 +213,11 @@ impl<K1, K2, V, G> storage::StorageDoubleMap<K1, K2, V> for G where
 		KArg1: ?Sized + EncodeLike<K1>
 	{
 		let prefix = Self::storage_double_map_final_key1(k1);
-		storage::PrefixIterator::<V> {
+		storage::PrefixIterator {
 			prefix: prefix.clone(),
 			previous_key: prefix,
-			phantom_data: Default::default(),
+			drain: false,
+			closure: |_raw_key, mut raw_value| V::decode(&mut raw_value),
 		}
 	}
 
@@ -322,54 +323,6 @@ impl<K1, K2, V, G> storage::StorageDoubleMap<K1, K2, V> for G where
 	}
 }
 
-/// Iterate over a prefix and decode raw_key and raw_value into `T`.
-pub struct MapIterator<T> {
-	prefix: Vec<u8>,
-	previous_key: Vec<u8>,
-	/// If true then value are removed while iterating
-	drain: bool,
-	/// Function that take `(raw_key_without_prefix, raw_value)` and decode `T`.
-	/// `raw_key_without_prefix` is the raw storage key without the prefix iterated on.
-	closure: fn(&[u8], &[u8]) -> Result<T, codec::Error>,
-}
-
-impl<T> Iterator for MapIterator<T> {
-	type Item = T;
-
-	fn next(&mut self) -> Option<Self::Item> {
-		loop {
-			let maybe_next = sp_io::storage::next_key(&self.previous_key)
-				.filter(|n| n.starts_with(&self.prefix));
-			break match maybe_next {
-				Some(next) => {
-					self.previous_key = next;
-					let raw_value = match unhashed::get_raw(&self.previous_key) {
-						Some(raw_value) => raw_value,
-						None => {
-							frame_support::print("ERROR: next_key returned a key with no value in MapIterator");
-							continue
-						}
-					};
-					if self.drain {
-						unhashed::kill(&self.previous_key)
-					}
-					let raw_key_without_prefix = &self.previous_key[self.prefix.len()..];
-					let item = match (self.closure)(raw_key_without_prefix, &raw_value[..]) {
-						Ok(item) => item,
-						Err(_e) => {
-							frame_support::print("ERROR: (key, value) failed to decode in MapIterator");
-							continue
-						}
-					};
-
-					Some(item)
-				}
-				None => None,
-			}
-		}
-	}
-}
-
 impl<
 	K1: FullCodec,
 	K2: FullCodec,
@@ -379,8 +332,8 @@ impl<
 	G::Hasher1: ReversibleStorageHasher,
 	G::Hasher2: ReversibleStorageHasher
 {
-	type PrefixIterator = MapIterator<(K2, V)>;
-	type Iterator = MapIterator<(K1, K2, V)>;
+	type PrefixIterator = PrefixIterator<(K2, V)>;
+	type Iterator = PrefixIterator<(K1, K2, V)>;
 
 	fn iter_prefix(k1: impl EncodeLike<K1>) -> Self::PrefixIterator {
 		let prefix = G::storage_double_map_final_key1(k1);
@@ -423,23 +376,41 @@ impl<
 		iterator
 	}
 
-	fn translate<O: Decode, F: Fn(O) -> Option<V>>(f: F) {
+	fn translate<O: Decode, F: Fn(K1, K2, O) -> Option<V>>(f: F) {
 		let prefix = G::prefix_hash();
 		let mut previous_key = prefix.clone();
-		loop {
-			match sp_io::storage::next_key(&previous_key).filter(|n| n.starts_with(&prefix)) {
-				Some(next) => {
-					previous_key = next;
-					let maybe_value = unhashed::get::<O>(&previous_key);
-					match maybe_value {
-						Some(value) => match f(value) {
-							Some(new) => unhashed::put::<V>(&previous_key, &new),
-							None => unhashed::kill(&previous_key),
-						},
-						None => continue,
-					}
-				}
-				None => return,
+		while let Some(next) = sp_io::storage::next_key(&previous_key)
+			.filter(|n| n.starts_with(&prefix))
+		{
+			previous_key = next;
+			let value = match unhashed::get::<O>(&previous_key) {
+				Some(value) => value,
+				None => {
+					crate::debug::error!("Invalid translate: fail to decode old value");
+					continue
+				},
+			};
+			let mut key_material = G::Hasher1::reverse(&previous_key[prefix.len()..]);
+			let key1 = match K1::decode(&mut key_material) {
+				Ok(key1) => key1,
+				Err(_) => {
+					crate::debug::error!("Invalid translate: fail to decode key1");
+					continue
+				},
+			};
+
+			let mut key2_material = G::Hasher2::reverse(&key_material);
+			let key2 = match K2::decode(&mut key2_material) {
+				Ok(key2) => key2,
+				Err(_) => {
+					crate::debug::error!("Invalid translate: fail to decode key2");
+					continue
+				},
+			};
+
+			match f(key1, key2, value) {
+				Some(new) => unhashed::put::<V>(&previous_key, &new),
+				None => unhashed::kill(&previous_key),
 			}
 		}
 	}
@@ -447,10 +418,12 @@ impl<
 
 /// Test iterators for StorageDoubleMap
 #[cfg(test)]
-#[allow(dead_code)]
 mod test_iterators {
 	use codec::{Encode, Decode};
-	use crate::storage::{generator::StorageDoubleMap, IterableStorageDoubleMap, unhashed};
+	use crate::{
+		hash::StorageHasher,
+		storage::{generator::StorageDoubleMap, IterableStorageDoubleMap, unhashed},
+	};
 
 	pub trait Trait {
 		type Origin;
@@ -466,7 +439,7 @@ mod test_iterators {
 
 	crate::decl_storage! {
 		trait Store for Module<T: Trait> as Test {
-			DoubleMap: double_map hasher(blake2_128_concat) u16, hasher(blake2_128_concat) u32 => u64;
+			DoubleMap: double_map hasher(blake2_128_concat) u16, hasher(twox_64_concat) u32 => u64;
 		}
 	}
 
@@ -484,11 +457,6 @@ mod test_iterators {
 		prefix
 	}
 
-	fn key_in_prefix(mut prefix: Vec<u8>) -> Vec<u8> {
-		prefix.push(0);
-		prefix
-	}
-
 	#[test]
 	fn double_map_reversible_reversible_iteration() {
 		sp_io::TestExternalities::default().execute_with(|| {
@@ -534,22 +502,59 @@ mod test_iterators {
 
 			assert_eq!(
 				DoubleMap::iter_prefix(k1).collect::<Vec<_>>(),
-				vec![(0, 0), (2, 2), (1, 1), (3, 3)],
+				vec![(1, 1), (2, 2), (0, 0), (3, 3)],
 			);
 
 			assert_eq!(
 				DoubleMap::iter_prefix_values(k1).collect::<Vec<_>>(),
-				vec![0, 2, 1, 3],
+				vec![1, 2, 0, 3],
 			);
 
 			assert_eq!(
 				DoubleMap::drain_prefix(k1).collect::<Vec<_>>(),
-				vec![(0, 0), (2, 2), (1, 1), (3, 3)],
+				vec![(1, 1), (2, 2), (0, 0), (3, 3)],
 			);
 
 			assert_eq!(DoubleMap::iter_prefix(k1).collect::<Vec<_>>(), vec![]);
 			assert_eq!(unhashed::get(&key_before_prefix(prefix.clone())), Some(1u64));
 			assert_eq!(unhashed::get(&key_after_prefix(prefix.clone())), Some(1u64));
+
+			// Translate
+			let prefix = DoubleMap::prefix_hash();
+
+			unhashed::put(&key_before_prefix(prefix.clone()), &1u64);
+			unhashed::put(&key_after_prefix(prefix.clone()), &1u64);
+			for i in 0..4 {
+				DoubleMap::insert(i as u16, i as u32, i as u64);
+			}
+
+			// Wrong key1
+			unhashed::put(
+				&[prefix.clone(), vec![1, 2, 3]].concat(),
+				&3u64.encode()
+			);
+
+			// Wrong key2
+			unhashed::put(
+				&[prefix.clone(), crate::Blake2_128Concat::hash(&1u16.encode())].concat(),
+				&3u64.encode()
+			);
+
+			// Wrong value
+			unhashed::put(
+				&[
+					prefix.clone(),
+					crate::Blake2_128Concat::hash(&1u16.encode()),
+					crate::Twox64Concat::hash(&2u32.encode()),
+				].concat(),
+				&vec![1],
+			);
+
+			DoubleMap::translate(|_k1, _k2, v: u64| Some(v*2));
+			assert_eq!(
+				DoubleMap::iter().collect::<Vec<_>>(),
+				vec![(3, 3, 6), (0, 0, 0), (2, 2, 4), (1, 1, 2)],
+			);
 		})
 	}
 }
diff --git a/frame/support/src/storage/generator/map.rs b/frame/support/src/storage/generator/map.rs
index fe932b797940b..1c13de52e1640 100644
--- a/frame/support/src/storage/generator/map.rs
+++ b/frame/support/src/storage/generator/map.rs
@@ -20,7 +20,7 @@ use sp_std::prelude::*;
 use sp_std::borrow::Borrow;
 use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike};
 use crate::{
-	storage::{self, unhashed, StorageAppend},
+	storage::{self, unhashed, StorageAppend, PrefixIterator},
 	Never, hash::{StorageHasher, Twox128, ReversibleStorageHasher},
 };
 
@@ -139,53 +139,56 @@ impl<
 > storage::IterableStorageMap<K, V> for G where
 	G::Hasher: ReversibleStorageHasher
 {
-	type Iterator = StorageMapIterator<K, V, G::Hasher>;
+	type Iterator = PrefixIterator<(K, V)>;
 
 	/// Enumerate all elements in the map.
 	fn iter() -> Self::Iterator {
 		let prefix = G::prefix_hash();
-		Self::Iterator {
+		PrefixIterator {
 			prefix: prefix.clone(),
 			previous_key: prefix,
 			drain: false,
-			_phantom: Default::default(),
+			closure: |raw_key_without_prefix, mut raw_value| {
+				let mut key_material = G::Hasher::reverse(raw_key_without_prefix);
+				Ok((K::decode(&mut key_material)?, V::decode(&mut raw_value)?))
+			},
 		}
 	}
 
 	/// Enumerate all elements in the map.
 	fn drain() -> Self::Iterator {
-		let prefix = G::prefix_hash();
-		Self::Iterator {
-			prefix: prefix.clone(),
-			previous_key: prefix,
-			drain: true,
-			_phantom: Default::default(),
-		}
+		let mut iterator = Self::iter();
+		iterator.drain = true;
+		iterator
 	}
 
 	fn translate<O: Decode, F: Fn(K, O) -> Option<V>>(f: F) {
 		let prefix = G::prefix_hash();
 		let mut previous_key = prefix.clone();
-		loop {
-			match sp_io::storage::next_key(&previous_key).filter(|n| n.starts_with(&prefix)) {
-				Some(next) => {
-					previous_key = next;
-					let maybe_value = unhashed::get::<O>(&previous_key);
-					match maybe_value {
-						Some(value) => {
-							let mut key_material = G::Hasher::reverse(&previous_key[prefix.len()..]);
-							match K::decode(&mut key_material) {
-								Ok(key) => match f(key, value) {
-									Some(new) => unhashed::put::<V>(&previous_key, &new),
-									None => unhashed::kill(&previous_key),
-								},
-								Err(_) => continue,
-							}
-						}
-						None => continue,
-					}
-				}
-				None => return,
+		while let Some(next) = sp_io::storage::next_key(&previous_key)
+			.filter(|n| n.starts_with(&prefix))
+		{
+			previous_key = next;
+			let value = match unhashed::get::<O>(&previous_key) {
+				Some(value) => value,
+				None => {
+					crate::debug::error!("Invalid translate: fail to decode old value");
+					continue
+				},
+			};
+
+			let mut key_material = G::Hasher::reverse(&previous_key[prefix.len()..]);
+			let key = match K::decode(&mut key_material) {
+				Ok(key) => key,
+				Err(_) => {
+					crate::debug::error!("Invalid translate: fail to decode key");
+					continue
+				},
+			};
+
+			match f(key, value) {
+				Some(new) => unhashed::put::<V>(&previous_key, &new),
+				None => unhashed::kill(&previous_key),
 			}
 		}
 	}
@@ -312,3 +315,91 @@ impl<K: FullEncode, V: FullCodec, G: StorageMap<K, V>> storage::StorageMap<K, V>
 		})
 	}
 }
+
+/// Test iterators for StorageMap
+#[cfg(test)]
+mod test_iterators {
+	use codec::{Encode, Decode};
+	use crate::{
+		hash::StorageHasher,
+		storage::{generator::StorageMap, IterableStorageMap, unhashed},
+	};
+
+	pub trait Trait {
+		type Origin;
+		type BlockNumber;
+	}
+
+	crate::decl_module! {
+		pub struct Module<T: Trait> for enum Call where origin: T::Origin {}
+	}
+
+	#[derive(PartialEq, Eq, Clone, Encode, Decode)]
+	struct NoDef(u32);
+
+	crate::decl_storage! {
+		trait Store for Module<T: Trait> as Test {
+			Map: map hasher(blake2_128_concat) u16 => u64;
+		}
+	}
+
+	fn key_before_prefix(mut prefix: Vec<u8>) -> Vec<u8> {
+		let last = prefix.iter_mut().last().unwrap();
+		assert!(*last != 0, "mock function not implemented for this prefix");
+		*last -= 1;
+		prefix
+	}
+
+	fn key_after_prefix(mut prefix: Vec<u8>) -> Vec<u8> {
+		let last = prefix.iter_mut().last().unwrap();
+		assert!(*last != 255, "mock function not implemented for this prefix");
+		*last += 1;
+		prefix
+	}
+
+	#[test]
+	fn map_reversible_reversible_iteration() {
+		sp_io::TestExternalities::default().execute_with(|| {
+			// All map iterator
+			let prefix = Map::prefix_hash();
+
+			unhashed::put(&key_before_prefix(prefix.clone()), &1u64);
+			unhashed::put(&key_after_prefix(prefix.clone()), &1u64);
+
+			for i in 0..4 {
+				Map::insert(i as u16, i as u64);
+			}
+
+			assert_eq!(Map::iter().collect::<Vec<_>>(), vec![(3, 3), (0, 0), (2, 2), (1, 1)]);
+
+			assert_eq!(Map::iter_values().collect::<Vec<_>>(), vec![3, 0, 2, 1]);
+
+			assert_eq!(Map::drain().collect::<Vec<_>>(), vec![(3, 3), (0, 0), (2, 2), (1, 1)]);
+
+			assert_eq!(Map::iter().collect::<Vec<_>>(), vec![]);
+			assert_eq!(unhashed::get(&key_before_prefix(prefix.clone())), Some(1u64));
+			assert_eq!(unhashed::get(&key_after_prefix(prefix.clone())), Some(1u64));
+
+			// Translate
+			let prefix = Map::prefix_hash();
+
+			unhashed::put(&key_before_prefix(prefix.clone()), &1u64);
+			unhashed::put(&key_after_prefix(prefix.clone()), &1u64);
+			for i in 0..4 {
+				Map::insert(i as u16, i as u64);
+			}
+
+			// Wrong key
+			unhashed::put(&[prefix.clone(), vec![1, 2, 3]].concat(), &3u64.encode());
+
+			// Wrong value
+			unhashed::put(
+				&[prefix.clone(), crate::Blake2_128Concat::hash(&6u16.encode())].concat(),
+				&vec![1],
+			);
+
+			Map::translate(|_k1, v: u64| Some(v*2));
+			assert_eq!(Map::iter().collect::<Vec<_>>(), vec![(3, 6), (0, 0), (2, 4), (1, 2)]);
+		})
+	}
+}
diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs
index 347fd814136d7..717a9a29ad5f5 100644
--- a/frame/support/src/storage/mod.rs
+++ b/frame/support/src/storage/mod.rs
@@ -17,7 +17,7 @@
 
 //! Stuff to do with the runtime's storage.
 
-use sp_std::{prelude::*, marker::PhantomData};
+use sp_std::prelude::*;
 use codec::{FullCodec, FullEncode, Encode, EncodeLike, Decode};
 use crate::hash::{Twox128, StorageHasher};
 use sp_runtime::generic::{Digest, DigestItem};
@@ -251,6 +251,8 @@ pub trait IterableStorageMap<K: FullEncode, V: FullCodec>: StorageMap<K, V> {
 
 	/// Translate the values of all elements by a function `f`, in the map in no particular order.
 	/// By returning `None` from `f` for an element, you'll remove it from the map.
+	///
+	/// NOTE: If a value fail to decode because storage is corrupted then it is skipped.
 	fn translate<O: Decode, F: Fn(K, O) -> Option<V>>(f: F);
 }
 
@@ -286,7 +288,9 @@ pub trait IterableStorageDoubleMap<
 
 	/// Translate the values of all elements by a function `f`, in the map in no particular order.
 	/// By returning `None` from `f` for an element, you'll remove it from the map.
-	fn translate<O: Decode, F: Fn(O) -> Option<V>>(f: F);
+	///
+	/// NOTE: If a value fail to decode because storage is corrupted then it is skipped.
+	fn translate<O: Decode, F: Fn(K1, K2, O) -> Option<V>>(f: F);
 }
 
 /// An implementation of a map with a two keys.
@@ -433,35 +437,58 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 	>(key1: KeyArg1, key2: KeyArg2) -> Option<V>;
 }
 
-/// Iterator for prefixed map.
-pub struct PrefixIterator<Value> {
+/// Iterate over a prefix and decode raw_key and raw_value into `T`.
+///
+/// If any decoding fails it skips it and continues to the next key.
+pub struct PrefixIterator<T> {
 	prefix: Vec<u8>,
 	previous_key: Vec<u8>,
-	phantom_data: PhantomData<Value>,
+	/// If true then value are removed while iterating
+	drain: bool,
+	/// Function that take `(raw_key_without_prefix, raw_value)` and decode `T`.
+	/// `raw_key_without_prefix` is the raw storage key without the prefix iterated on.
+	closure: fn(&[u8], &[u8]) -> Result<T, codec::Error>,
 }
 
-impl<Value: Decode> Iterator for PrefixIterator<Value> {
-	type Item = Value;
+impl<T> Iterator for PrefixIterator<T> {
+	type Item = T;
 
 	fn next(&mut self) -> Option<Self::Item> {
-		match sp_io::storage::next_key(&self.previous_key)
-			.filter(|n| n.starts_with(&self.prefix[..]))
-		{
-			Some(next_key) => {
-				let value = unhashed::get(&next_key);
-
-				if value.is_none() {
-					runtime_print!(
-						"ERROR: returned next_key has no value:\nkey is {:?}\nnext_key is {:?}",
-						&self.previous_key, &next_key,
-					);
+		loop {
+			let maybe_next = sp_io::storage::next_key(&self.previous_key)
+				.filter(|n| n.starts_with(&self.prefix));
+			break match maybe_next {
+				Some(next) => {
+					self.previous_key = next;
+					let raw_value = match unhashed::get_raw(&self.previous_key) {
+						Some(raw_value) => raw_value,
+						None => {
+							crate::debug::error!(
+								"next_key returned a key with no value at {:?}",
+								self.previous_key
+							);
+							continue
+						}
+					};
+					if self.drain {
+						unhashed::kill(&self.previous_key)
+					}
+					let raw_key_without_prefix = &self.previous_key[self.prefix.len()..];
+					let item = match (self.closure)(raw_key_without_prefix, &raw_value[..]) {
+						Ok(item) => item,
+						Err(e) => {
+							crate::debug::error!(
+								"(key, value) failed to decode at {:?}: {:?}",
+								self.previous_key, e
+							);
+							continue
+						}
+					};
+
+					Some(item)
 				}
-
-				self.previous_key = next_key;
-
-				value
-			},
-			_ => None,
+				None => None,
+			}
 		}
 	}
 }
@@ -493,22 +520,22 @@ pub trait StoragePrefixedMap<Value: FullCodec> {
 	}
 
 	/// Iter over all value of the storage.
+	///
+	/// NOTE: If a value failed to decode becaues storage is corrupted then it is skipped.
 	fn iter_values() -> PrefixIterator<Value> {
 		let prefix = Self::final_prefix();
 		PrefixIterator {
 			prefix: prefix.to_vec(),
 			previous_key: prefix.to_vec(),
-			phantom_data: Default::default(),
+			drain: false,
+			closure: |_raw_key, mut raw_value| Value::decode(&mut raw_value),
 		}
 	}
 
-	/// Translate the values from some previous `OldValue` to the current type.
-	///
-	/// `TV` translates values.
+	/// Translate the values of all elements by a function `f`, in the map in no particular order.
+	/// By returning `None` from `f` for an element, you'll remove it from the map.
 	///
-	/// Returns `Err` if the map could not be interpreted as the old type, and Ok if it could.
-	/// The `Err` contains the number of value that couldn't be interpreted, those value are
-	/// removed from the map.
+	/// NOTE: If a value fail to decode because storage is corrupted then it is skipped.
 	///
 	/// # Warning
 	///
@@ -517,33 +544,28 @@ pub trait StoragePrefixedMap<Value: FullCodec> {
 	///
 	/// # Usage
 	///
-	/// This would typically be called inside the module implementation of on_runtime_upgrade, while
-	/// ensuring **no usage of this storage are made before the call to `on_runtime_upgrade`**. (More
-	/// precisely prior initialized modules doesn't make use of this storage).
-	fn translate_values<OldValue, TV>(translate_val: TV) -> Result<(), u32>
-		where OldValue: Decode, TV: Fn(OldValue) -> Value
-	{
+	/// This would typically be called inside the module implementation of on_runtime_upgrade.
+	fn translate_values<OldValue: Decode, F: Fn(OldValue) -> Option<Value>>(f: F) {
 		let prefix = Self::final_prefix();
-		let mut previous_key = prefix.to_vec();
-		let mut errors = 0;
-		while let Some(next_key) = sp_io::storage::next_key(&previous_key)
-			.filter(|n| n.starts_with(&prefix[..]))
+		let mut previous_key = prefix.clone().to_vec();
+		while let Some(next) = sp_io::storage::next_key(&previous_key)
+			.filter(|n| n.starts_with(&prefix))
 		{
-			if let Some(value) = unhashed::get(&next_key) {
-				unhashed::put(&next_key[..], &translate_val(value));
-			} else {
-				// We failed to read the value. Remove the key and increment errors.
-				unhashed::kill(&next_key[..]);
-				errors += 1;
+			previous_key = next;
+			let maybe_value = unhashed::get::<OldValue>(&previous_key);
+			match maybe_value {
+				Some(value) => match f(value) {
+					Some(new) => unhashed::put::<Value>(&previous_key, &new),
+					None => unhashed::kill(&previous_key),
+				},
+				None => {
+					crate::debug::error!(
+						"old key failed to decode at {:?}",
+						previous_key
+					);
+					continue
+				},
 			}
-
-			previous_key = next_key;
-		}
-
-		if errors == 0 {
-			Ok(())
-		} else {
-			Err(errors)
 		}
 	}
 }
@@ -652,7 +674,7 @@ mod test {
 			unhashed::put(&[&k[..], &vec![8][..]].concat(), &2u32);
 
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![]);
-			MyStorage::translate_values(|v: u32| v as u64).unwrap();
+			MyStorage::translate_values(|v: u32| Some(v as u64));
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2]);
 			MyStorage::remove_all();
 
@@ -664,8 +686,8 @@ mod test {
 
 			// (contains some value that successfully decoded to u64)
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2, 3]);
-			assert_eq!(MyStorage::translate_values(|v: u128| v as u64), Err(2));
-			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 3]);
+			MyStorage::translate_values(|v: u128| Some(v as u64));
+			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2, 3]);
 			MyStorage::remove_all();
 
 			// test that other values are not modified.

From 111a110fce78c6777d2eb90bb2095d7f06c2f0b9 Mon Sep 17 00:00:00 2001
From: HarryHong <hong091114@gmail.com>
Date: Tue, 15 Sep 2020 23:17:14 +0800
Subject: [PATCH 089/122] fix js dependancy alert, bumping bl version (#7110)

* fix js dependancy alert, bumping bl version

* fix low severity modules
---
 .maintain/chaostest/package-lock.json | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.maintain/chaostest/package-lock.json b/.maintain/chaostest/package-lock.json
index 8855f221a133d..09468e12fb4f9 100644
--- a/.maintain/chaostest/package-lock.json
+++ b/.maintain/chaostest/package-lock.json
@@ -941,9 +941,9 @@
       }
     },
     "bl": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
-      "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
+      "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
       "dev": true,
       "requires": {
         "buffer": "^5.5.0",
@@ -3836,9 +3836,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.15",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-      "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
+      "version": "4.17.20",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
     },
     "lodash._reinterpolate": {
       "version": "3.0.0",

From f406f499cbafb6e50ea5ff948b782265e2d6fa10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Wed, 16 Sep 2020 00:03:50 +0200
Subject: [PATCH 090/122] Make `transactional` attribute less scope dependent
 (#7112)

* Make `transactional` attribute less scope dependent

The old implementation expected that `frame-support` wasn't imported
under a different name. Besides that the pr removes some whitespaces.

* Update frame/support/procedural/src/lib.rs

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>
---
 frame/support/procedural/src/lib.rs           |  6 ++----
 frame/support/procedural/src/transactional.rs | 19 +++++++++++--------
 frame/support/procedural/tools/src/lib.rs     | 19 +++++++++++++++++++
 3 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs
index 054d90d7bbaeb..060882d1123bf 100644
--- a/frame/support/procedural/src/lib.rs
+++ b/frame/support/procedural/src/lib.rs
@@ -15,9 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// tag::description[]
 //! Proc macro of Support code for the runtime.
-// end::description[]
 
 #![recursion_limit="512"]
 
@@ -296,7 +294,7 @@ pub fn construct_runtime(input: TokenStream) -> TokenStream {
 /// The return type of the annotated function must be `Result`. All changes to storage performed
 /// by the annotated function are discarded if it returns `Err`, or committed if `Ok`.
 ///
-/// #Example
+/// # Example
 ///
 /// ```nocompile
 /// #[transactional]
@@ -313,5 +311,5 @@ pub fn construct_runtime(input: TokenStream) -> TokenStream {
 /// ```
 #[proc_macro_attribute]
 pub fn transactional(attr: TokenStream, input: TokenStream) -> TokenStream {
-	transactional::transactional(attr, input)
+	transactional::transactional(attr, input).unwrap_or_else(|e| e.to_compile_error().into())
 }
diff --git a/frame/support/procedural/src/transactional.rs b/frame/support/procedural/src/transactional.rs
index a001f44c4d482..fbd0c9ca0b3c4 100644
--- a/frame/support/procedural/src/transactional.rs
+++ b/frame/support/procedural/src/transactional.rs
@@ -17,15 +17,17 @@
 
 use proc_macro::TokenStream;
 use quote::quote;
-use syn::{parse_macro_input, ItemFn};
+use syn::{ItemFn, Result};
+use frame_support_procedural_tools::generate_crate_access_2018;
 
-pub fn transactional(_attr: TokenStream, input: TokenStream) -> TokenStream {
-	let ItemFn { attrs, vis, sig, block } = parse_macro_input!(input as ItemFn);
+pub fn transactional(_attr: TokenStream, input: TokenStream) -> Result<TokenStream> {
+	let ItemFn { attrs, vis, sig, block } = syn::parse(input)?;
 
+	let crate_ = generate_crate_access_2018()?;
 	let output = quote! {
 		#(#attrs)*
-        #vis #sig {
-			use frame_support::storage::{with_transaction, TransactionOutcome};
+		#vis #sig {
+			use #crate_::storage::{with_transaction, TransactionOutcome};
 			with_transaction(|| {
 				let r = #block;
 				if r.is_ok() {
@@ -34,7 +36,8 @@ pub fn transactional(_attr: TokenStream, input: TokenStream) -> TokenStream {
 					TransactionOutcome::Rollback(r)
 				}
 			})
-        }
-    };
-    output.into()
+		}
+	};
+
+	Ok(output.into())
 }
diff --git a/frame/support/procedural/tools/src/lib.rs b/frame/support/procedural/tools/src/lib.rs
index 0033787a7c045..c5a27c809aff8 100644
--- a/frame/support/procedural/tools/src/lib.rs
+++ b/frame/support/procedural/tools/src/lib.rs
@@ -46,6 +46,25 @@ pub fn generate_crate_access(unique_id: &str, def_crate: &str) -> TokenStream {
 	}
 }
 
+/// Generate the crate access for the `frame-support` crate using 2018 syntax.
+///
+/// Output will for example be `frame_support`.
+pub fn generate_crate_access_2018() -> Result<TokenStream, Error> {
+	if std::env::var("CARGO_PKG_NAME").unwrap() == "frame-support" {
+		Ok(quote::quote!( frame_support ))
+	} else {
+		match crate_name("frame-support") {
+			Ok(name) => {
+				let name = Ident::new(&name, Span::call_site());
+				Ok(quote!( #name ))
+			},
+			Err(e) => {
+				Err(Error::new(Span::call_site(), &e))
+			}
+		}
+	}
+}
+
 /// Generates the hidden includes that are required to make the macro independent from its scope.
 pub fn generate_hidden_includes(unique_id: &str, def_crate: &str) -> TokenStream {
 	if std::env::var("CARGO_PKG_NAME").unwrap() == def_crate {

From d2c7c1f61d150b289c3b53265f5b63db9b124ffa Mon Sep 17 00:00:00 2001
From: joe petrowski <25483142+joepetrowski@users.noreply.github.com>
Date: Wed, 16 Sep 2020 11:32:11 +0200
Subject: [PATCH 091/122] Add SS58 Registry (#7020)

* add SS58 registry

* formatting

* description -> displayName

* Update ss58-registry.json

Co-authored-by: Jaco Greeff <jacogr@gmail.com>

* make numbers literal, tokens can have different denominations

* add dock

* add dark

* add websites and tokens

* add KLP decimals

* add acala and laminar info

Co-authored-by: Jaco Greeff <jacogr@gmail.com>
---
 ss58-registry.json | 302 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 302 insertions(+)
 create mode 100644 ss58-registry.json

diff --git a/ss58-registry.json b/ss58-registry.json
new file mode 100644
index 0000000000000..db3ab18d9854f
--- /dev/null
+++ b/ss58-registry.json
@@ -0,0 +1,302 @@
+{
+	"specification": "https://github.com/paritytech/substrate/wiki/External-Address-Format-(SS58)",
+	"schema": {
+		"prefix": "The address prefix. Must be an integer and unique.",
+		"network": "Unique identifier for the network that will use this prefix, string, no spaces. To integrate with CLI tools, e.g. `--network polkadot`.",
+		"displayName": "The name of the network that will use this prefix, in a format friendly for display.",
+		"symbols": "Array of symbols of any tokens the chain uses, usually 2-5 characters. Most chains will only have one. Chains that have multiple instances of the Balances pallet should order the array by instance.",
+		"decimals": "Array of integers representing the number of decimals that represent a single unit to the end user. Must be same length as `symbols` to represent each token's denomination.",
+		"standardAccount": "Signing curve for standard account. Substrate supports ed25519, sr25519, and secp256k1.",
+		"website": "A website or Github repo associated with the network."
+	},
+	"registry": [
+		{
+			"prefix": 0,
+			"network": "polkadot",
+			"displayName": "Polkadot Relay Chain",
+			"symbols": ["DOT"],
+			"decimals": [10],
+			"standardAccount": "*25519",
+			"website": "https://polkadot.network"
+		},
+		{
+			"prefix": 1,
+			"network": "reserved1",
+			"displayName": "This prefix is reserved.",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": null,
+			"website": null
+		},
+		{
+			"prefix": 2,
+			"network": "kusama",
+			"displayName": "Kusama Relay Chain",
+			"symbols": ["KSM"],
+			"decimals": [12],
+			"standardAccount": "*25519",
+			"website": "https://kusama.network"
+		},
+		{
+			"prefix": 3,
+			"network": "reserved3",
+			"displayName": "This prefix is reserved.",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": null,
+			"website": null
+		},
+		{
+			"prefix": 4,
+			"network": "katalchain",
+			"displayName": "Katal Chain",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 5,
+			"network": "plasm",
+			"displayName": "Plasm Network",
+			"symbols": ["PLM"],
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 6,
+			"network": "bitfrost",
+			"displayName": "Bitfrost",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 7,
+			"network": "edgeware",
+			"displayName": "Edgeware",
+			"symbols": ["EDG"],
+			"decimals": [18],
+			"standardAccount": "*25519",
+			"website": "https://edgewa.re"
+		},
+		{
+			"prefix": 8,
+			"network": "karura",
+			"displayName": "Acala Karura Canary",
+			"symbols": ["KAR"],
+			"decimals": [18],
+			"standardAccount": "*25519",
+			"website": "https://acala.network/"
+		},
+		{
+			"prefix": 9,
+			"network": "reynolds",
+			"displayName": "Laminar Reynolds Canary",
+			"symbols": ["REY"],
+			"decimals": [18],
+			"standardAccount": "*25519",
+			"website": ["http://laminar.network/"]
+		},
+		{
+			"prefix": 10,
+			"network": "acala",
+			"displayName": "Acala",
+			"symbols": ["ACA"],
+			"decimals": [18],
+			"standardAccount": "*25519",
+			"website": "https://acala.network/"
+		},
+		{
+			"prefix": 11,
+			"network": "laminar",
+			"displayName": "Laminar",
+			"symbols": ["LAMI"],
+			"decimals": [18],
+			"standardAccount": "*25519",
+			"website": ["http://laminar.network/"]
+		},
+		{
+			"prefix": 12,
+			"network": "polymath",
+			"displayName": "Polymath",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 13,
+			"network": "substratee",
+			"displayName": "SubstraTEE",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": "https://www.substratee.com"
+		},
+		{
+			"prefix": 16,
+			"network": "kulupu",
+			"displayName": "Kulupu",
+			"symbols": ["KLP"],
+			"decimals": [12],
+			"standardAccount": "*25519",
+			"website": "https://kulupu.network/"
+		},
+		{
+			"prefix": 17,
+			"network": "dark",
+			"displayName": "Dark Mainnet",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 18,
+			"network": "darwinia",
+			"displayName": "Darwinia Chain",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 20,
+			"network": "stafi",
+			"displayName": "Stafi",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 21,
+			"network": "dock-testnet",
+			"displayName": "Dock Testnet",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 22,
+			"network": "dock-mainnet",
+			"displayName": "Dock Mainnet",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 23,
+			"network": "shift",
+			"displayName": "ShiftNrg",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 28,
+			"network": "subsocial",
+			"displayName": "Subsocial",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 30,
+			"network": "phala",
+			"displayName": "Phala Network",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 32,
+			"network": "robonomics",
+			"displayName": "Robonomics Network",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 33,
+			"network": "datahighway",
+			"displayName": "DataHighway",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 36,
+			"network": "centrifuge",
+			"displayName": "Centrifuge Chain",
+			"symbols": ["RAD"],
+			"decimals": [18],
+			"standardAccount": "*25519",
+			"website": "https://centrifuge.io/"
+		},
+		{
+			"prefix": 42,
+			"network": "substrate",
+			"displayName": "Substrate",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": "https://substrate.dev/"
+		},
+		{
+			"prefix": 43,
+			"network": "reserved43",
+			"displayName": "This prefix is reserved.",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": null,
+			"website": null
+		},
+		{
+			"prefix": 44,
+			"network": "chainx",
+			"displayName": "ChainX",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": "*25519",
+			"website": null
+		},
+		{
+			"prefix": 46,
+			"network": "reserved46",
+			"displayName": "This prefix is reserved.",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": null,
+			"website": null
+		},
+		{
+			"prefix": 47,
+			"network": "reserved47",
+			"displayName": "This prefix is reserved.",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": null,
+			"website": null
+		},
+		{
+			"prefix": 48,
+			"network": "reserved48",
+			"displayName": "All prefixes 48 and higher are reserved and cannot be allocated.",
+			"symbols": null,
+			"decimals": null,
+			"standardAccount": null,
+			"website": null
+		}
+	]
+}

From 0ddcd66e25715ee2fef6d6e3e0b4ab9305423bd0 Mon Sep 17 00:00:00 2001
From: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Date: Wed, 16 Sep 2020 14:25:31 +0200
Subject: [PATCH 092/122] Move Staking Weights to T::WeightInfo (#7007)

* Fix the benchmarks

* Migrate staking to weightInfo

* Fix global benchmarks

* re-calculate the submit solution weight.

* Fix some refund.

* Get rid of all the extra parameters.

* Fix staking tests.

* new values from the bench machine.

* Fix some grumbles

* better macro

* Some better doc

* Move to interpreted wasm

* Make it work temporarily

* Final fix of default ones.

* Fix payout benchmarks

* Fix payout stuff

* One last fix

* use benchmarking machine for numbers

* update weight docs

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
---
 frame/session/benchmarking/src/lib.rs        |  11 +-
 frame/staking/src/benchmarking.rs            | 268 +++++++++------
 frame/staking/src/default_weights.rs         | 169 ++++++++++
 frame/staking/src/lib.rs                     | 329 ++++++-------------
 frame/staking/src/testing_utils.rs           |  31 +-
 frame/staking/src/tests.rs                   |  57 +++-
 primitives/npos-elections/compact/src/lib.rs |  98 +++++-
 primitives/npos-elections/src/tests.rs       |  71 +++-
 8 files changed, 658 insertions(+), 376 deletions(-)
 create mode 100644 frame/staking/src/default_weights.rs

diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs
index cc471893356d5..ee66223fc0b04 100644
--- a/frame/session/benchmarking/src/lib.rs
+++ b/frame/session/benchmarking/src/lib.rs
@@ -35,7 +35,7 @@ use frame_system::RawOrigin;
 use pallet_session::{historical::Module as Historical, Module as Session, *};
 use pallet_staking::{
 	benchmarking::create_validator_with_nominators, testing_utils::create_validators,
-	MAX_NOMINATIONS,
+	MAX_NOMINATIONS, RewardDestination,
 };
 use sp_runtime::traits::{One, StaticLookup};
 
@@ -55,7 +55,12 @@ benchmarks! {
 
 	set_keys {
 		let n in 1 .. MAX_NOMINATIONS as u32;
-		let v_stash = create_validator_with_nominators::<T>(n, MAX_NOMINATIONS as u32, false)?;
+		let v_stash = create_validator_with_nominators::<T>(
+			n,
+			MAX_NOMINATIONS as u32,
+			false,
+			RewardDestination::Staked,
+		)?;
 		let v_controller = pallet_staking::Module::<T>::bonded(&v_stash).ok_or("not stash")?;
 		let keys = T::Keys::default();
 		let proof: Vec<u8> = vec![0,1,2,3];
@@ -63,7 +68,7 @@ benchmarks! {
 
 	purge_keys {
 		let n in 1 .. MAX_NOMINATIONS as u32;
-		let v_stash = create_validator_with_nominators::<T>(n, MAX_NOMINATIONS as u32, false)?;
+		let v_stash = create_validator_with_nominators::<T>(n, MAX_NOMINATIONS as u32, false, RewardDestination::Staked)?;
 		let v_controller = pallet_staking::Module::<T>::bonded(&v_stash).ok_or("not stash")?;
 		let keys = T::Keys::default();
 		let proof: Vec<u8> = vec![0,1,2,3];
diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs
index 156b2f81c8429..afda58db4672f 100644
--- a/frame/staking/src/benchmarking.rs
+++ b/frame/staking/src/benchmarking.rs
@@ -29,6 +29,14 @@ const MAX_SPANS: u32 = 100;
 const MAX_VALIDATORS: u32 = 1000;
 const MAX_SLASHES: u32 = 1000;
 
+macro_rules! do_whitelist {
+	($acc:ident) => {
+		frame_benchmarking::benchmarking::add_to_whitelist(
+			frame_system::Account::<T>::hashed_key_for(&$acc).into()
+		);
+	}
+}
+
 // Add slashing spans to a user account. Not relevant for actual use, only to benchmark
 // read and write operations.
 fn add_slashing_spans<T: Trait>(who: &T::AccountId, spans: u32) {
@@ -51,11 +59,12 @@ pub fn create_validator_with_nominators<T: Trait>(
 	n: u32,
 	upper_bound: u32,
 	dead: bool,
+	destination: RewardDestination<T::AccountId>
 ) -> Result<T::AccountId, &'static str> {
 	let mut points_total = 0;
 	let mut points_individual = Vec::new();
 
-	let (v_stash, v_controller) = create_stash_controller::<T>(0, 100)?;
+	let (v_stash, v_controller) = create_stash_controller::<T>(0, 100, destination.clone())?;
 	let validator_prefs = ValidatorPrefs {
 		commission: Perbill::from_percent(50),
 	};
@@ -68,9 +77,9 @@ pub fn create_validator_with_nominators<T: Trait>(
 	// Give the validator n nominators, but keep total users in the system the same.
 	for i in 0 .. upper_bound {
 		let (_n_stash, n_controller) = if !dead {
-			create_stash_controller::<T>(u32::max_value() - i, 100)?
+			create_stash_controller::<T>(u32::max_value() - i, 100, destination.clone())?
 		} else {
-			create_stash_and_dead_controller::<T>(u32::max_value() - i, 100)?
+			create_stash_and_dead_controller::<T>(u32::max_value() - i, 100, destination.clone())?
 		};
 		if i < n {
 			Staking::<T>::nominate(RawOrigin::Signed(n_controller.clone()).into(), vec![stash_lookup.clone()])?;
@@ -100,19 +109,18 @@ pub fn create_validator_with_nominators<T: Trait>(
 	Ok(v_stash)
 }
 
+const USER_SEED: u32 = 999666;
+
 benchmarks! {
-	_{
-		// User account seed
-		let u in 0 .. 1000 => ();
-	}
+	_{}
 
 	bond {
-		let u in ...;
-		let stash = create_funded_user::<T>("stash", u, 100);
-		let controller = create_funded_user::<T>("controller", u, 100);
+		let stash = create_funded_user::<T>("stash", USER_SEED, 100);
+		let controller = create_funded_user::<T>("controller", USER_SEED, 100);
 		let controller_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(controller.clone());
 		let reward_destination = RewardDestination::Staked;
 		let amount = T::Currency::minimum_balance() * 10.into();
+		do_whitelist!(stash);
 	}: _(RawOrigin::Signed(stash.clone()), controller_lookup, amount, reward_destination)
 	verify {
 		assert!(Bonded::<T>::contains_key(stash));
@@ -120,11 +128,11 @@ benchmarks! {
 	}
 
 	bond_extra {
-		let u in ...;
-		let (stash, controller) = create_stash_controller::<T>(u, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(USER_SEED, 100, Default::default())?;
 		let max_additional = T::Currency::minimum_balance() * 10.into();
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created before")?;
 		let original_bonded: BalanceOf<T> = ledger.active;
+		do_whitelist!(stash);
 	}: _(RawOrigin::Signed(stash), max_additional)
 	verify {
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created after")?;
@@ -133,11 +141,11 @@ benchmarks! {
 	}
 
 	unbond {
-		let u in ...;
-		let (_, controller) = create_stash_controller::<T>(u, 100)?;
+		let (_, controller) = create_stash_controller::<T>(USER_SEED, 100, Default::default())?;
 		let amount = T::Currency::minimum_balance() * 10.into();
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created before")?;
 		let original_bonded: BalanceOf<T> = ledger.active;
+		do_whitelist!(controller);
 	}: _(RawOrigin::Signed(controller.clone()), amount)
 	verify {
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created after")?;
@@ -149,13 +157,14 @@ benchmarks! {
 	withdraw_unbonded_update {
 		// Slashing Spans
 		let s in 0 .. MAX_SPANS;
-		let (stash, controller) = create_stash_controller::<T>(0, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(0, 100, Default::default())?;
 		add_slashing_spans::<T>(&stash, s);
 		let amount = T::Currency::minimum_balance() * 5.into(); // Half of total
 		Staking::<T>::unbond(RawOrigin::Signed(controller.clone()).into(), amount)?;
 		CurrentEra::put(EraIndex::max_value());
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created before")?;
 		let original_total: BalanceOf<T> = ledger.total;
+		do_whitelist!(controller);
 	}: withdraw_unbonded(RawOrigin::Signed(controller.clone()), s)
 	verify {
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created after")?;
@@ -167,22 +176,23 @@ benchmarks! {
 	withdraw_unbonded_kill {
 		// Slashing Spans
 		let s in 0 .. MAX_SPANS;
-		let (stash, controller) = create_stash_controller::<T>(0, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(0, 100, Default::default())?;
 		add_slashing_spans::<T>(&stash, s);
 		let amount = T::Currency::minimum_balance() * 10.into();
 		Staking::<T>::unbond(RawOrigin::Signed(controller.clone()).into(), amount)?;
 		CurrentEra::put(EraIndex::max_value());
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created before")?;
 		let original_total: BalanceOf<T> = ledger.total;
+		do_whitelist!(controller);
 	}: withdraw_unbonded(RawOrigin::Signed(controller.clone()), s)
 	verify {
 		assert!(!Ledger::<T>::contains_key(controller));
 	}
 
 	validate {
-		let u in ...;
-		let (stash, controller) = create_stash_controller::<T>(u, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(USER_SEED, 100, Default::default())?;
 		let prefs = ValidatorPrefs::default();
+		do_whitelist!(controller);
 	}: _(RawOrigin::Signed(controller), prefs)
 	verify {
 		assert!(Validators::<T>::contains_key(stash));
@@ -191,51 +201,52 @@ benchmarks! {
 	// Worst case scenario, MAX_NOMINATIONS
 	nominate {
 		let n in 1 .. MAX_NOMINATIONS as u32;
-		let (stash, controller) = create_stash_controller::<T>(n + 1, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(n + 1, 100, Default::default())?;
 		let validators = create_validators::<T>(n, 100)?;
+		do_whitelist!(controller);
 	}: _(RawOrigin::Signed(controller), validators)
 	verify {
 		assert!(Nominators::<T>::contains_key(stash));
 	}
 
 	chill {
-		let u in ...;
-		let (_, controller) = create_stash_controller::<T>(u, 100)?;
+		let (_, controller) = create_stash_controller::<T>(USER_SEED, 100, Default::default())?;
+		do_whitelist!(controller);
 	}: _(RawOrigin::Signed(controller))
 
 	set_payee {
-		let u in ...;
-		let (stash, controller) = create_stash_controller::<T>(u, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(USER_SEED, 100, Default::default())?;
 		assert_eq!(Payee::<T>::get(&stash), RewardDestination::Staked);
+		do_whitelist!(controller);
 	}: _(RawOrigin::Signed(controller), RewardDestination::Controller)
 	verify {
 		assert_eq!(Payee::<T>::get(&stash), RewardDestination::Controller);
 	}
 
 	set_controller {
-		let u in ...;
-		let (stash, _) = create_stash_controller::<T>(u, 100)?;
-		let new_controller = create_funded_user::<T>("new_controller", u, 100);
+		let (stash, _) = create_stash_controller::<T>(USER_SEED, 100, Default::default())?;
+		let new_controller = create_funded_user::<T>("new_controller", USER_SEED, 100);
 		let new_controller_lookup = T::Lookup::unlookup(new_controller.clone());
+		do_whitelist!(stash);
 	}: _(RawOrigin::Signed(stash), new_controller_lookup)
 	verify {
 		assert!(Ledger::<T>::contains_key(&new_controller));
 	}
 
 	set_validator_count {
-		let c in 0 .. MAX_VALIDATORS;
-	}: _(RawOrigin::Root, c)
+		let validator_count = MAX_VALIDATORS;
+	}: _(RawOrigin::Root, validator_count)
 	verify {
-		assert_eq!(ValidatorCount::get(), c);
+		assert_eq!(ValidatorCount::get(), validator_count);
 	}
 
-	force_no_eras { let i in 0 .. 1; }: _(RawOrigin::Root)
+	force_no_eras {}: _(RawOrigin::Root)
 	verify { assert_eq!(ForceEra::get(), Forcing::ForceNone); }
 
-	force_new_era {let i in 0 .. 1; }: _(RawOrigin::Root)
+	force_new_era {}: _(RawOrigin::Root)
 	verify { assert_eq!(ForceEra::get(), Forcing::ForceNew); }
 
-	force_new_era_always { let i in 0 .. 1; }: _(RawOrigin::Root)
+	force_new_era_always {}: _(RawOrigin::Root)
 	verify { assert_eq!(ForceEra::get(), Forcing::ForceAlways); }
 
 	// Worst case scenario, the list of invulnerables is very long.
@@ -253,7 +264,7 @@ benchmarks! {
 	force_unstake {
 		// Slashing Spans
 		let s in 0 .. MAX_SPANS;
-		let (stash, controller) = create_stash_controller::<T>(0, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(0, 100, Default::default())?;
 		add_slashing_spans::<T>(&stash, s);
 	}: _(RawOrigin::Root, stash, s)
 	verify {
@@ -275,37 +286,60 @@ benchmarks! {
 		assert_eq!(UnappliedSlashes::<T>::get(&era).len(), (MAX_SLASHES - s) as usize);
 	}
 
-	payout_stakers {
+	payout_stakers_dead_controller {
 		let n in 1 .. T::MaxNominatorRewardedPerValidator::get() as u32;
-		let validator = create_validator_with_nominators::<T>(n, T::MaxNominatorRewardedPerValidator::get() as u32, true)?;
+		let validator = create_validator_with_nominators::<T>(
+			n,
+			T::MaxNominatorRewardedPerValidator::get() as u32,
+			true,
+			RewardDestination::Controller,
+		)?;
 
 		let current_era = CurrentEra::get().unwrap();
+		// set the commission for this particular era as well.
+		<ErasValidatorPrefs<T>>::insert(current_era, validator.clone(), <Staking<T>>::validators(&validator));
+
 		let caller = whitelisted_caller();
-		let balance_before = T::Currency::free_balance(&validator);
-	}: _(RawOrigin::Signed(caller), validator.clone(), current_era)
+		let validator_controller = <Bonded<T>>::get(&validator).unwrap();
+		let balance_before = T::Currency::free_balance(&validator_controller);
+	}: payout_stakers(RawOrigin::Signed(caller), validator.clone(), current_era)
 	verify {
-		// Validator has been paid!
-		let balance_after = T::Currency::free_balance(&validator);
-		assert!(balance_before < balance_after);
+		let balance_after = T::Currency::free_balance(&validator_controller);
+		assert!(
+			balance_before < balance_after,
+			"Balance of controller {:?} should have increased after payout.",
+			validator,
+		);
 	}
 
-	payout_stakers_alive_controller {
+	payout_stakers_alive_staked {
 		let n in 1 .. T::MaxNominatorRewardedPerValidator::get() as u32;
-		let validator = create_validator_with_nominators::<T>(n, T::MaxNominatorRewardedPerValidator::get() as u32, false)?;
+		let validator = create_validator_with_nominators::<T>(
+			n,
+			T::MaxNominatorRewardedPerValidator::get() as u32,
+			false,
+			RewardDestination::Staked,
+		)?;
 
 		let current_era = CurrentEra::get().unwrap();
+		// set the commission for this particular era as well.
+		<ErasValidatorPrefs<T>>::insert(current_era, validator.clone(), <Staking<T>>::validators(&validator));
+
 		let caller = whitelisted_caller();
 		let balance_before = T::Currency::free_balance(&validator);
 	}: payout_stakers(RawOrigin::Signed(caller), validator.clone(), current_era)
 	verify {
-		// Validator has been paid!
 		let balance_after = T::Currency::free_balance(&validator);
-		assert!(balance_before < balance_after);
+		assert!(
+			balance_before < balance_after,
+			"Balance of stash {:?} should have increased after payout.",
+			validator,
+		);
 	}
 
 	rebond {
 		let l in 1 .. MAX_UNLOCKING_CHUNKS as u32;
-		let (_, controller) = create_stash_controller::<T>(u, 100)?;
+		let (_, controller) = create_stash_controller::<T>(USER_SEED, 100, Default::default())?;
 		let mut staking_ledger = Ledger::<T>::get(controller.clone()).unwrap();
 		let unlock_chunk = UnlockChunk::<BalanceOf<T>> {
 			value: 1.into(),
@@ -316,6 +350,7 @@ benchmarks! {
 		}
 		Ledger::<T>::insert(controller.clone(), staking_ledger.clone());
 		let original_bonded: BalanceOf<T> = staking_ledger.active;
+		do_whitelist!(controller);
 	}: _(RawOrigin::Signed(controller.clone()), (l + 100).into())
 	verify {
 		let ledger = Ledger::<T>::get(&controller).ok_or("ledger not created after")?;
@@ -343,9 +378,10 @@ benchmarks! {
 
 	reap_stash {
 		let s in 1 .. MAX_SPANS;
-		let (stash, controller) = create_stash_controller::<T>(0, 100)?;
+		let (stash, controller) = create_stash_controller::<T>(0, 100, Default::default())?;
 		add_slashing_spans::<T>(&stash, s);
 		T::Currency::make_free_balance_be(&stash, 0.into());
+		do_whitelist!(controller);
 	}: _(RawOrigin::Signed(controller), stash.clone(), s)
 	verify {
 		assert!(!Bonded::<T>::contains_key(&stash));
@@ -362,32 +398,7 @@ benchmarks! {
 		assert!(validators.len() == v as usize);
 	}
 
-	do_slash {
-		let l in 1 .. MAX_UNLOCKING_CHUNKS as u32;
-		let (stash, controller) = create_stash_controller::<T>(0, 100)?;
-		let mut staking_ledger = Ledger::<T>::get(controller.clone()).unwrap();
-		let unlock_chunk = UnlockChunk::<BalanceOf<T>> {
-			value: 1.into(),
-			era: EraIndex::zero(),
-		};
-		for _ in 0 .. l {
-			staking_ledger.unlocking.push(unlock_chunk.clone())
-		}
-		Ledger::<T>::insert(controller, staking_ledger);
-		let slash_amount = T::Currency::minimum_balance() * 10.into();
-		let balance_before = T::Currency::free_balance(&stash);
-	}: {
-		crate::slashing::do_slash::<T>(
-			&stash,
-			slash_amount,
-			&mut BalanceOf::<T>::zero(),
-			&mut NegativeImbalanceOf::<T>::zero()
-		);
-	} verify {
-		let balance_after = T::Currency::free_balance(&stash);
-		assert!(balance_before > balance_after);
-	}
-
+	#[extra]
 	payout_all {
 		let v in 1 .. 10;
 		let n in 1 .. 100;
@@ -426,18 +437,45 @@ benchmarks! {
 		}
 	}
 
-	// This benchmark create `v` validators intent, `n` nominators intent, each nominator nominate
-	// MAX_NOMINATIONS in the set of the first `w` validators.
-	// It builds a solution with `w` winners composed of nominated validators randomly nominated,
-	// `a` assignment with MAX_NOMINATIONS.
+	#[extra]
+	do_slash {
+		let l in 1 .. MAX_UNLOCKING_CHUNKS as u32;
+		let (stash, controller) = create_stash_controller::<T>(0, 100, Default::default())?;
+		let mut staking_ledger = Ledger::<T>::get(controller.clone()).unwrap();
+		let unlock_chunk = UnlockChunk::<BalanceOf<T>> {
+			value: 1.into(),
+			era: EraIndex::zero(),
+		};
+		for _ in 0 .. l {
+			staking_ledger.unlocking.push(unlock_chunk.clone())
+		}
+		Ledger::<T>::insert(controller, staking_ledger);
+		let slash_amount = T::Currency::minimum_balance() * 10.into();
+		let balance_before = T::Currency::free_balance(&stash);
+	}: {
+		crate::slashing::do_slash::<T>(
+			&stash,
+			slash_amount,
+			&mut BalanceOf::<T>::zero(),
+			&mut NegativeImbalanceOf::<T>::zero()
+		);
+	} verify {
+		let balance_after = T::Currency::free_balance(&stash);
+		assert!(balance_before > balance_after);
+	}
+
+	// This benchmark create `v` validators intent, `n` nominators intent, in total creating `e`
+	// edges.
+	#[extra]
 	submit_solution_initial {
-		// number of validator intent
-		let v in 1000 .. 2000;
-		// number of nominator intent
-		let n in 1000 .. 2000;
-		// number of assignments. Basically, number of active nominators.
-		let a in 200 .. 500;
-		// number of winners, also ValidatorCount
+		// number of validator intention. This will be equal to `ElectionSize::validators`.
+		let v in 200 .. 400;
+		// number of nominator intention. This will be equal to `ElectionSize::nominators`.
+		let n in 500 .. 1000;
+		// number of assignments. Basically, number of active nominators. This will be equal to
+		// `compact.len()`.
+		let a in 200 .. 400;
+		// number of winners, also ValidatorCount. This will be equal to `winner.len()`.
 		let w in 16 .. 100;
 
 		ensure!(w as usize >= MAX_NOMINATIONS, "doesn't support lower value");
@@ -466,15 +504,19 @@ benchmarks! {
 			size
 		) = offchain_election::prepare_submission::<T>(assignments, winners, false).unwrap();
 
+		assert_eq!(
+			winners.len(), compact.unique_targets().len(),
+			"unique targets ({}) and winners ({}) count not same. This solution is not valid.",
+			compact.unique_targets().len(),
+			winners.len(),
+		);
+
 		// needed for the solution to be accepted
 		<EraElectionStatus<T>>::put(ElectionStatus::Open(T::BlockNumber::from(1u32)));
 
 		let era = <Staking<T>>::current_era().unwrap_or(0);
 		let caller: T::AccountId = account("caller", n, SEED);
-
-		// Whitelist caller account from further DB operations.
-		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
-		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
+		do_whitelist!(caller);
 	}: {
 		let result = <Staking<T>>::submit_election_solution(
 			RawOrigin::Signed(caller.clone()).into(),
@@ -493,13 +535,13 @@ benchmarks! {
 
 	// same as submit_solution_initial but we place a very weak solution on chian first.
 	submit_solution_better {
-		// number of validator intent
-		let v in 1000 .. 2000;
-		// number of nominator intent
-		let n in 1000 .. 2000;
+		// number of validator intention.
+		let v in 200 .. 400;
+		// number of nominator intention.
+		let n in 500 .. 1000;
 		// number of assignments. Basically, number of active nominators.
-		let a in 200 .. 500;
-		// number of winners, also ValidatorCount
+		let a in 200 .. 400;
+		// number of winners, also ValidatorCount.
 		let w in 16 .. 100;
 
 		ensure!(w as usize >= MAX_NOMINATIONS, "doesn't support lower value");
@@ -530,15 +572,19 @@ benchmarks! {
 			size
 		) = offchain_election::prepare_submission::<T>(assignments, winners, false).unwrap();
 
+		assert_eq!(
+			winners.len(), compact.unique_targets().len(),
+			"unique targets ({}) and winners ({}) count not same. This solution is not valid.",
+			compact.unique_targets().len(),
+			winners.len(),
+		);
+
 		// needed for the solution to be accepted
 		<EraElectionStatus<T>>::put(ElectionStatus::Open(T::BlockNumber::from(1u32)));
 
 		let era = <Staking<T>>::current_era().unwrap_or(0);
 		let caller: T::AccountId = account("caller", n, SEED);
-
-		// Whitelist caller account from further DB operations.
-		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
-		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
+		do_whitelist!(caller);
 
 		// submit a very bad solution on-chain
 		{
@@ -576,11 +622,12 @@ benchmarks! {
 	}
 
 	// This will be early rejected based on the score.
+	#[extra]
 	submit_solution_weaker {
-		// number of validator intent
-		let v in 1000 .. 2000;
-		// number of nominator intent
-		let n in 1000 .. 2000;
+		// number of validator intention.
+		let v in 200 .. 400;
+		// number of nominator intention.
+		let n in 500 .. 1000;
 
 		create_validators_with_nominators_for_era::<T>(v, n, MAX_NOMINATIONS, false, None)?;
 
@@ -589,16 +636,19 @@ benchmarks! {
 
 		// needed for the solution to be accepted
 		<EraElectionStatus<T>>::put(ElectionStatus::Open(T::BlockNumber::from(1u32)));
-		let caller: T::AccountId = account("caller", n, SEED);
 		let era = <Staking<T>>::current_era().unwrap_or(0);
-
-		// Whitelist caller account from further DB operations.
-		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
-		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
+		let caller: T::AccountId = account("caller", n, SEED);
+		do_whitelist!(caller);
 
 		// submit a seq-phragmen with all the good stuff on chain.
 		{
 			let (winners, compact, score, size) = get_seq_phragmen_solution::<T>(true);
+			assert_eq!(
+				winners.len(), compact.unique_targets().len(),
+				"unique targets ({}) and winners ({}) count not same. This solution is not valid.",
+				compact.unique_targets().len(),
+				winners.len(),
+			);
 			assert!(
 				<Staking<T>>::submit_election_solution(
 					RawOrigin::Signed(caller.clone()).into(),
@@ -662,6 +712,7 @@ mod tests {
 				n,
 				<Test as Trait>::MaxNominatorRewardedPerValidator::get() as u32,
 				false,
+				RewardDestination::Staked,
 			).unwrap();
 
 			let current_era = CurrentEra::get().unwrap();
@@ -683,6 +734,7 @@ mod tests {
 				n,
 				<Test as Trait>::MaxNominatorRewardedPerValidator::get() as u32,
 				false,
+				RewardDestination::Staked,
 			).unwrap();
 
 			// Add 20 slashing spans
@@ -743,8 +795,8 @@ mod tests {
 			assert_ok!(test_benchmark_set_invulnerables::<Test>());
 			assert_ok!(test_benchmark_force_unstake::<Test>());
 			assert_ok!(test_benchmark_cancel_deferred_slash::<Test>());
-			assert_ok!(test_benchmark_payout_stakers::<Test>());
-			assert_ok!(test_benchmark_payout_stakers_alive_controller::<Test>());
+			assert_ok!(test_benchmark_payout_stakers_dead_controller::<Test>());
+			assert_ok!(test_benchmark_payout_stakers_alive_staked::<Test>());
 			assert_ok!(test_benchmark_rebond::<Test>());
 			assert_ok!(test_benchmark_set_history_depth::<Test>());
 			assert_ok!(test_benchmark_reap_stash::<Test>());
diff --git a/frame/staking/src/default_weights.rs b/frame/staking/src/default_weights.rs
new file mode 100644
index 0000000000000..fa5a05f63824e
--- /dev/null
+++ b/frame/staking/src/default_weights.rs
@@ -0,0 +1,169 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Default weights of pallet-staking.
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn bond() -> Weight {
+		(144278000 as Weight)
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn bond_extra() -> Weight {
+		(110715000 as Weight)
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn unbond() -> Weight {
+		(99840000 as Weight)
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn withdraw_unbonded_update(s: u32, ) -> Weight {
+		(100728000 as Weight)
+			.saturating_add((63000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn withdraw_unbonded_kill(s: u32, ) -> Weight {
+		(168879000 as Weight)
+			.saturating_add((6666000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(7 as Weight))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn validate() -> Weight {
+		(35539000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn nominate(n: u32, ) -> Weight {
+		(48596000 as Weight)
+			.saturating_add((308000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn chill() -> Weight {
+		(35144000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn set_payee() -> Weight {
+		(24255000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_controller() -> Weight {
+		(52294000 as Weight)
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn set_validator_count() -> Weight {
+		(5185000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_no_eras() -> Weight {
+		(5907000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_new_era() -> Weight {
+		(5917000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_new_era_always() -> Weight {
+		(5952000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_invulnerables(v: u32, ) -> Weight {
+		(6324000 as Weight)
+			.saturating_add((9000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_unstake(s: u32, ) -> Weight {
+		(119691000 as Weight)
+			.saturating_add((6681000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn cancel_deferred_slash(s: u32, ) -> Weight {
+		(5820201000 as Weight)
+			.saturating_add((34672000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn payout_stakers_dead_controller(n: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((92486000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight)))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(n as Weight)))
+	}
+	fn payout_stakers_alive_staked(n: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((117324000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight)))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(n as Weight)))
+	}
+	fn rebond(l: u32, ) -> Weight {
+		(71316000 as Weight)
+			.saturating_add((142000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn set_history_depth(e: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((51901000 as Weight).saturating_mul(e as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+			.saturating_add(DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight)))
+	}
+	fn reap_stash(s: u32, ) -> Weight {
+		(147166000 as Weight)
+			.saturating_add((6661000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn new_era(v: u32, n: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((1440459000 as Weight).saturating_mul(v as Weight))
+			.saturating_add((182580000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads(10 as Weight))
+			.saturating_add(DbWeight::get().reads((4 as Weight).saturating_mul(v as Weight)))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight)))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight)))
+	}
+	fn submit_solution_better(v: u32, n: u32, a: u32, w: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((964000 as Weight).saturating_mul(v as Weight))
+			.saturating_add((432000 as Weight).saturating_mul(n as Weight))
+			.saturating_add((204294000 as Weight).saturating_mul(a as Weight))
+			.saturating_add((9546000 as Weight).saturating_mul(w as Weight))
+			.saturating_add(DbWeight::get().reads(6 as Weight))
+			.saturating_add(DbWeight::get().reads((4 as Weight).saturating_mul(a as Weight)))
+			.saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(w as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+}
diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs
index 279b6bb1dec7d..7061832b0460c 100644
--- a/frame/staking/src/lib.rs
+++ b/frame/staking/src/lib.rs
@@ -279,6 +279,7 @@ pub mod benchmarking;
 pub mod slashing;
 pub mod offchain_election;
 pub mod inflation;
+pub mod default_weights;
 
 use sp_std::{
 	result,
@@ -356,6 +357,8 @@ pub type ValidatorIndex = u16;
 // Ensure the size of both ValidatorIndex and NominatorIndex. They both need to be well below usize.
 static_assertions::const_assert!(size_of::<ValidatorIndex>() <= size_of::<usize>());
 static_assertions::const_assert!(size_of::<NominatorIndex>() <= size_of::<usize>());
+static_assertions::const_assert!(size_of::<ValidatorIndex>() <= size_of::<u32>());
+static_assertions::const_assert!(size_of::<NominatorIndex>() <= size_of::<u32>());
 
 /// Maximum number of stakers that can be stored in a snapshot.
 pub(crate) const MAX_VALIDATORS: usize = ValidatorIndex::max_value() as usize;
@@ -766,132 +769,31 @@ impl<T: Trait> SessionInterface<<T as frame_system::Trait>::AccountId> for T whe
 	}
 }
 
-pub mod weight {
-	use super::*;
-
-	/// All weight notes are pertaining to the case of a better solution, in which we execute
-	/// the longest code path.
-	/// Weight: 0 + (0.63 μs * v) + (0.36 μs * n) + (96.53 μs * a ) + (8 μs * w ) with:
-	/// * v validators in snapshot validators,
-	/// * n nominators in snapshot nominators,
-	/// * a assignment in the submitted solution
-	/// * w winners in the submitted solution
-	///
-	/// State reads:
-	/// 	- Initial checks:
-	/// 		- ElectionState, CurrentEra, QueuedScore
-	/// 		- SnapshotValidators.len() + SnapShotNominators.len()
-	/// 		- ValidatorCount
-	/// 		- SnapshotValidators
-	/// 		- SnapshotNominators
-	/// 	- Iterate over nominators:
-	/// 		- compact.len() * Nominators(who)
-	/// 		- (non_self_vote_edges) * SlashingSpans
-	/// 	- For `assignment_ratio_to_staked`: Basically read the staked value of each stash.
-	/// 		- (winners.len() + compact.len()) * (Ledger + Bonded)
-	/// 		- TotalIssuance (read a gzillion times potentially, but well it is cached.)
-	/// - State writes:
-	/// 	- QueuedElected, QueuedScore
-	pub fn weight_for_submit_solution<T: Trait>(
-		winners: &Vec<ValidatorIndex>,
-		compact: &CompactAssignments,
-		size: &ElectionSize,
-	) -> Weight {
-		(630 * WEIGHT_PER_NANOS).saturating_mul(size.validators as Weight)
-			.saturating_add((360 * WEIGHT_PER_NANOS).saturating_mul(size.nominators as Weight))
-			.saturating_add((96 * WEIGHT_PER_MICROS).saturating_mul(compact.len() as Weight))
-			.saturating_add((8 * WEIGHT_PER_MICROS).saturating_mul(winners.len() as Weight))
-			// Initial checks
-			.saturating_add(T::DbWeight::get().reads(8))
-			// Nominators
-			.saturating_add(T::DbWeight::get().reads(compact.len() as Weight))
-			// SlashingSpans (upper bound for invalid solution)
-			.saturating_add(T::DbWeight::get().reads(compact.edge_count() as Weight))
-			// `assignment_ratio_to_staked`
-			.saturating_add(T::DbWeight::get().reads(2 * ((winners.len() + compact.len()) as Weight)))
-			.saturating_add(T::DbWeight::get().reads(1))
-			// write queued score and elected
-			.saturating_add(T::DbWeight::get().writes(2))
-	}
-
-	/// Weight of `submit_solution` in case of a correct submission.
-	///
-	/// refund: we charged compact.len() * read(1) for SlashingSpans. A valid solution only reads
-	/// winners.len().
-	pub fn weight_for_correct_submit_solution<T: Trait>(
-		winners: &Vec<ValidatorIndex>,
-		compact: &CompactAssignments,
-		size: &ElectionSize,
-	) -> Weight {
-		// NOTE: for consistency, we re-compute the original weight to maintain their relation and
-		// prevent any foot-guns.
-		let original_weight = weight_for_submit_solution::<T>(winners, compact, size);
-		original_weight
-			.saturating_sub(T::DbWeight::get().reads(compact.edge_count() as Weight))
-			.saturating_add(T::DbWeight::get().reads(winners.len() as Weight))
-	}
-}
-
 pub trait WeightInfo {
-	fn bond(u: u32, ) -> Weight;
-	fn bond_extra(u: u32, ) -> Weight;
-	fn unbond(u: u32, ) -> Weight;
+	fn bond() -> Weight;
+	fn bond_extra() -> Weight;
+	fn unbond() -> Weight;
 	fn withdraw_unbonded_update(s: u32, ) -> Weight;
 	fn withdraw_unbonded_kill(s: u32, ) -> Weight;
-	fn validate(u: u32, ) -> Weight;
+	fn validate() -> Weight;
 	fn nominate(n: u32, ) -> Weight;
-	fn chill(u: u32, ) -> Weight;
-	fn set_payee(u: u32, ) -> Weight;
-	fn set_controller(u: u32, ) -> Weight;
-	fn set_validator_count(c: u32, ) -> Weight;
-	fn force_no_eras(i: u32, ) -> Weight;
-	fn force_new_era(i: u32, ) -> Weight;
-	fn force_new_era_always(i: u32, ) -> Weight;
+	fn chill() -> Weight;
+	fn set_payee() -> Weight;
+	fn set_controller() -> Weight;
+	fn set_validator_count() -> Weight;
+	fn force_no_eras() -> Weight;
+	fn force_new_era() -> Weight;
+	fn force_new_era_always() -> Weight;
 	fn set_invulnerables(v: u32, ) -> Weight;
 	fn force_unstake(s: u32, ) -> Weight;
 	fn cancel_deferred_slash(s: u32, ) -> Weight;
-	fn payout_stakers(n: u32, ) -> Weight;
-	fn payout_stakers_alive_controller(n: u32, ) -> Weight;
+	fn payout_stakers_alive_staked(n: u32, ) -> Weight;
+	fn payout_stakers_dead_controller(n: u32, ) -> Weight;
 	fn rebond(l: u32, ) -> Weight;
 	fn set_history_depth(e: u32, ) -> Weight;
 	fn reap_stash(s: u32, ) -> Weight;
 	fn new_era(v: u32, n: u32, ) -> Weight;
-	fn do_slash(l: u32, ) -> Weight;
-	fn payout_all(v: u32, n: u32, ) -> Weight;
-	fn submit_solution_initial(v: u32, n: u32, a: u32, w: u32, ) -> Weight;
 	fn submit_solution_better(v: u32, n: u32, a: u32, w: u32, ) -> Weight;
-	fn submit_solution_weaker(v: u32, n: u32, ) -> Weight;
-}
-
-impl WeightInfo for () {
-	fn bond(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn bond_extra(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn unbond(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn withdraw_unbonded_update(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn withdraw_unbonded_kill(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn validate(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn nominate(_n: u32, ) -> Weight { 1_000_000_000 }
-	fn chill(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn set_payee(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn set_controller(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn set_validator_count(_c: u32, ) -> Weight { 1_000_000_000 }
-	fn force_no_eras(_i: u32, ) -> Weight { 1_000_000_000 }
-	fn force_new_era(_i: u32, ) -> Weight { 1_000_000_000 }
-	fn force_new_era_always(_i: u32, ) -> Weight { 1_000_000_000 }
-	fn set_invulnerables(_v: u32, ) -> Weight { 1_000_000_000 }
-	fn force_unstake(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn cancel_deferred_slash(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn payout_stakers(_n: u32, ) -> Weight { 1_000_000_000 }
-	fn payout_stakers_alive_controller(_n: u32, ) -> Weight { 1_000_000_000 }
-	fn rebond(_l: u32, ) -> Weight { 1_000_000_000 }
-	fn set_history_depth(_e: u32, ) -> Weight { 1_000_000_000 }
-	fn reap_stash(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn new_era(_v: u32, _n: u32, ) -> Weight { 1_000_000_000 }
-	fn do_slash(_l: u32, ) -> Weight { 1_000_000_000 }
-	fn payout_all(_v: u32, _n: u32, ) -> Weight { 1_000_000_000 }
-	fn submit_solution_initial(_v: u32, _n: u32, _a: u32, _w: u32, ) -> Weight { 1_000_000_000 }
-	fn submit_solution_better(_v: u32, _n: u32, _a: u32, _w: u32, ) -> Weight { 1_000_000_000 }
-	fn submit_solution_weaker(_v: u32, _n: u32, ) -> Weight { 1_000_000_000 }
 }
 
 pub trait Trait: frame_system::Trait + SendTransactionTypes<Call<Self>> {
@@ -1489,12 +1391,12 @@ decl_module! {
 		/// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned
 		/// unless the `origin` falls below _existential deposit_ and gets removed as dust.
 		/// ------------------
-		/// Base Weight: 67.87 µs
+		/// Weight: O(1)
 		/// DB Weight:
 		/// - Read: Bonded, Ledger, [Origin Account], Current Era, History Depth, Locks
 		/// - Write: Bonded, Payee, [Origin Account], Locks, Ledger
 		/// # </weight>
-		#[weight = 67 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(5, 4)]
+		#[weight = T::WeightInfo::bond()]
 		pub fn bond(origin,
 			controller: <T::Lookup as StaticLookup>::Source,
 			#[compact] value: BalanceOf<T>,
@@ -1558,12 +1460,11 @@ decl_module! {
 		/// - O(1).
 		/// - One DB entry.
 		/// ------------
-		/// Base Weight: 54.88 µs
 		/// DB Weight:
 		/// - Read: Era Election Status, Bonded, Ledger, [Origin Account], Locks
 		/// - Write: [Origin Account], Locks, Ledger
 		/// # </weight>
-		#[weight = 55 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(4, 2)]
+		#[weight = T::WeightInfo::bond_extra()]
 		fn bond_extra(origin, #[compact] max_additional: BalanceOf<T>) {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			let stash = ensure_signed(origin)?;
@@ -1609,12 +1510,12 @@ decl_module! {
 		///   `withdraw_unbonded`.
 		/// - One DB entry.
 		/// ----------
-		/// Base Weight: 50.34 µs
+		/// Weight: O(1)
 		/// DB Weight:
-		/// - Read: Era Election Status, Ledger, Current Era, Locks, [Origin Account]
-		/// - Write: [Origin Account], Locks, Ledger
+		/// - Read: EraElectionStatus, Ledger, CurrentEra, Locks, BalanceOf Stash,
+		/// - Write: Locks, Ledger, BalanceOf Stash,
 		/// </weight>
-		#[weight = 50 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(4, 2)]
+		#[weight = T::WeightInfo::unbond()]
 		fn unbond(origin, #[compact] value: BalanceOf<T>) {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			let controller = ensure_signed(origin)?;
@@ -1663,25 +1564,18 @@ decl_module! {
 		/// - Writes are limited to the `origin` account key.
 		/// ---------------
 		/// Complexity O(S) where S is the number of slashing spans to remove
-		/// Base Weight:
-		/// Update: 50.52 + .028 * S µs
+		/// Update:
 		/// - Reads: EraElectionStatus, Ledger, Current Era, Locks, [Origin Account]
 		/// - Writes: [Origin Account], Locks, Ledger
-		/// Kill: 79.41 + 2.366 * S µs
-		/// - Reads: EraElectionStatus, Ledger, Current Era, Bonded, Slashing Spans, [Origin Account], Locks
-		/// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, [Origin Account], Locks
+		/// Kill:
+		/// - Reads: EraElectionStatus, Ledger, Current Era, Bonded, Slashing Spans, [Origin
+		///   Account], Locks, BalanceOf stash
+		/// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators,
+		///   [Origin Account], Locks, BalanceOf stash.
 		/// - Writes Each: SpanSlash * S
 		/// NOTE: Weight annotation is the kill scenario, we refund otherwise.
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(6, 6)
-			.saturating_add(80 * WEIGHT_PER_MICROS)
-			.saturating_add(
-				(2 * WEIGHT_PER_MICROS).saturating_mul(Weight::from(*num_slashing_spans))
-			)
-			.saturating_add(T::DbWeight::get().writes(Weight::from(*num_slashing_spans)))
-			// if slashing spans is non-zero, add 1 more write
-			.saturating_add(T::DbWeight::get().writes(Weight::from(*num_slashing_spans).min(1)))
-		]
+		#[weight = T::WeightInfo::withdraw_unbonded_kill(*num_slashing_spans)]
 		fn withdraw_unbonded(origin, num_slashing_spans: u32) -> DispatchResultWithPostInfo {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			let controller = ensure_signed(origin)?;
@@ -1703,8 +1597,9 @@ decl_module! {
 			} else {
 				// This was the consequence of a partial unbond. just update the ledger and move on.
 				Self::update_ledger(&controller, &ledger);
-				// This is only an update, so we use less overall weight
-				Some(50 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(4, 2))
+
+				// This is only an update, so we use less overall weight.
+				Some(T::WeightInfo::withdraw_unbonded_update(num_slashing_spans))
 			};
 
 			// `old_total` should never be less than the new total because
@@ -1730,12 +1625,12 @@ decl_module! {
 		/// - Contains a limited number of reads.
 		/// - Writes are limited to the `origin` account key.
 		/// -----------
-		/// Base Weight: 17.13 µs
+		/// Weight: O(1)
 		/// DB Weight:
 		/// - Read: Era Election Status, Ledger
 		/// - Write: Nominators, Validators
 		/// # </weight>
-		#[weight = 17 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(2, 2)]
+		#[weight = T::WeightInfo::validate()]
 		pub fn validate(origin, prefs: ValidatorPrefs) {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			let controller = ensure_signed(origin)?;
@@ -1758,16 +1653,13 @@ decl_module! {
 		/// which is capped at CompactAssignments::LIMIT (MAX_NOMINATIONS).
 		/// - Both the reads and writes follow a similar pattern.
 		/// ---------
-		/// Base Weight: 22.34 + .36 * N µs
+		/// Weight: O(N)
 		/// where N is the number of targets
 		/// DB Weight:
 		/// - Reads: Era Election Status, Ledger, Current Era
 		/// - Writes: Validators, Nominators
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(3, 2)
-			.saturating_add(22 * WEIGHT_PER_MICROS)
-			.saturating_add((360 * WEIGHT_PER_NANOS).saturating_mul(targets.len() as Weight))
-		]
+		#[weight = T::WeightInfo::nominate(targets.len() as u32)]
 		pub fn nominate(origin, targets: Vec<<T::Lookup as StaticLookup>::Source>) {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			let controller = ensure_signed(origin)?;
@@ -1802,12 +1694,12 @@ decl_module! {
 		/// - Contains one read.
 		/// - Writes are limited to the `origin` account key.
 		/// --------
-		/// Base Weight: 16.53 µs
+		/// Weight: O(1)
 		/// DB Weight:
 		/// - Read: EraElectionStatus, Ledger
 		/// - Write: Validators, Nominators
 		/// # </weight>
-		#[weight = 16 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(2, 2)]
+		#[weight = T::WeightInfo::chill()]
 		fn chill(origin) {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			let controller = ensure_signed(origin)?;
@@ -1826,12 +1718,12 @@ decl_module! {
 		/// - Contains a limited number of reads.
 		/// - Writes are limited to the `origin` account key.
 		/// ---------
-		/// - Base Weight: 11.33 µs
+		/// - Weight: O(1)
 		/// - DB Weight:
 		///     - Read: Ledger
 		///     - Write: Payee
 		/// # </weight>
-		#[weight = 11 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(1, 1)]
+		#[weight = T::WeightInfo::set_payee()]
 		fn set_payee(origin, payee: RewardDestination<T::AccountId>) {
 			let controller = ensure_signed(origin)?;
 			let ledger = Self::ledger(&controller).ok_or(Error::<T>::NotController)?;
@@ -1850,12 +1742,12 @@ decl_module! {
 		/// - Contains a limited number of reads.
 		/// - Writes are limited to the `origin` account key.
 		/// ----------
-		/// Base Weight: 25.22 µs
+		/// Weight: O(1)
 		/// DB Weight:
 		/// - Read: Bonded, Ledger New Controller, Ledger Old Controller
 		/// - Write: Bonded, Ledger New Controller, Ledger Old Controller
 		/// # </weight>
-		#[weight = 25 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(3, 3)]
+		#[weight = T::WeightInfo::set_controller()]
 		fn set_controller(origin, controller: <T::Lookup as StaticLookup>::Source) {
 			let stash = ensure_signed(origin)?;
 			let old_controller = Self::bonded(&stash).ok_or(Error::<T>::NotStash)?;
@@ -1876,10 +1768,10 @@ decl_module! {
 		/// The dispatch origin must be Root.
 		///
 		/// # <weight>
-		/// Base Weight: 1.717 µs
+		/// Weight: O(1)
 		/// Write: Validator Count
 		/// # </weight>
-		#[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().writes(1)]
+		#[weight = T::WeightInfo::set_validator_count()]
 		fn set_validator_count(origin, #[compact] new: u32) {
 			ensure_root(origin)?;
 			ValidatorCount::put(new);
@@ -1890,10 +1782,9 @@ decl_module! {
 		/// The dispatch origin must be Root.
 		///
 		/// # <weight>
-		/// Base Weight: 1.717 µs
-		/// Read/Write: Validator Count
+		/// Same as [`set_validator_count`].
 		/// # </weight>
-		#[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(1, 1)]
+		#[weight = T::WeightInfo::set_validator_count()]
 		fn increase_validator_count(origin, #[compact] additional: u32) {
 			ensure_root(origin)?;
 			ValidatorCount::mutate(|n| *n += additional);
@@ -1904,10 +1795,9 @@ decl_module! {
 		/// The dispatch origin must be Root.
 		///
 		/// # <weight>
-		/// Base Weight: 1.717 µs
-		/// Read/Write: Validator Count
+		/// Same as [`set_validator_count`].
 		/// # </weight>
-		#[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(1, 1)]
+		#[weight = T::WeightInfo::set_validator_count()]
 		fn scale_validator_count(origin, factor: Percent) {
 			ensure_root(origin)?;
 			ValidatorCount::mutate(|n| *n += factor * *n);
@@ -1919,10 +1809,10 @@ decl_module! {
 		///
 		/// # <weight>
 		/// - No arguments.
-		/// - Base Weight: 1.857 µs
+		/// - Weight: O(1)
 		/// - Write: ForceEra
 		/// # </weight>
-		#[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().writes(1)]
+		#[weight = T::WeightInfo::force_no_eras()]
 		fn force_no_eras(origin) {
 			ensure_root(origin)?;
 			ForceEra::put(Forcing::ForceNone);
@@ -1935,10 +1825,10 @@ decl_module! {
 		///
 		/// # <weight>
 		/// - No arguments.
-		/// - Base Weight: 1.959 µs
+		/// - Weight: O(1)
 		/// - Write ForceEra
 		/// # </weight>
-		#[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().writes(1)]
+		#[weight = T::WeightInfo::force_new_era()]
 		fn force_new_era(origin) {
 			ensure_root(origin)?;
 			ForceEra::put(Forcing::ForceNew);
@@ -1950,16 +1840,12 @@ decl_module! {
 		///
 		/// # <weight>
 		/// - O(V)
-		/// - Base Weight: 2.208 + .006 * V µs
 		/// - Write: Invulnerables
 		/// # </weight>
-		#[weight = T::DbWeight::get().writes(1)
-			.saturating_add(2 * WEIGHT_PER_MICROS)
-			.saturating_add((6 * WEIGHT_PER_NANOS).saturating_mul(validators.len() as Weight))
-		]
-		fn set_invulnerables(origin, validators: Vec<T::AccountId>) {
+		#[weight = T::WeightInfo::set_invulnerables(invulnerables.len() as u32)]
+		fn set_invulnerables(origin, invulnerables: Vec<T::AccountId>) {
 			ensure_root(origin)?;
-			<Invulnerables<T>>::put(validators);
+			<Invulnerables<T>>::put(invulnerables);
 		}
 
 		/// Force a current staker to become completely unstaked, immediately.
@@ -1968,20 +1854,11 @@ decl_module! {
 		///
 		/// # <weight>
 		/// O(S) where S is the number of slashing spans to be removed
-		/// Base Weight: 53.07 + 2.365 * S µs
 		/// Reads: Bonded, Slashing Spans, Account, Locks
 		/// Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Account, Locks
 		/// Writes Each: SpanSlash * S
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(4, 7)
-			.saturating_add(53 * WEIGHT_PER_MICROS)
-			.saturating_add(
-				WEIGHT_PER_MICROS.saturating_mul(2).saturating_mul(Weight::from(*num_slashing_spans))
-			)
-			.saturating_add(T::DbWeight::get().writes(Weight::from(*num_slashing_spans)))
-			// if slashing spans is non-zero, add 1 more write
-			.saturating_add(T::DbWeight::get().writes(Weight::from(*num_slashing_spans > 0)))
-		]
+		#[weight = T::WeightInfo::force_unstake(*num_slashing_spans)]
 		fn force_unstake(origin, stash: T::AccountId, num_slashing_spans: u32) {
 			ensure_root(origin)?;
 
@@ -1997,10 +1874,10 @@ decl_module! {
 		/// The dispatch origin must be Root.
 		///
 		/// # <weight>
-		/// - Base Weight: 2.05 µs
+		/// - Weight: O(1)
 		/// - Write: ForceEra
 		/// # </weight>
-		#[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().writes(1)]
+		#[weight = T::WeightInfo::force_new_era_always()]
 		fn force_new_era_always(origin) {
 			ensure_root(origin)?;
 			ForceEra::put(Forcing::ForceAlways);
@@ -2016,14 +1893,10 @@ decl_module! {
 		/// Complexity: O(U + S)
 		/// with U unapplied slashes weighted with U=1000
 		/// and S is the number of slash indices to be canceled.
-		/// - Base: 5870 + 34.61 * S µs
 		/// - Read: Unapplied Slashes
 		/// - Write: Unapplied Slashes
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 1)
-			.saturating_add(5_870 * WEIGHT_PER_MICROS)
-			.saturating_add((35 * WEIGHT_PER_MICROS).saturating_mul(slash_indices.len() as Weight))
-		]
+		#[weight = T::WeightInfo::cancel_deferred_slash(slash_indices.len() as u32)]
 		fn cancel_deferred_slash(origin, era: EraIndex, slash_indices: Vec<u32>) {
 			T::SlashCancelOrigin::ensure_origin(origin)?;
 
@@ -2058,22 +1931,19 @@ decl_module! {
 		/// - Contains a limited number of reads and writes.
 		/// -----------
 		/// N is the Number of payouts for the validator (including the validator)
-		/// Base Weight:
-		/// - Reward Destination Staked: 110 + 54.2 * N µs (Median Slopes)
-		/// - Reward Destination Controller (Creating): 120 + 41.95 * N µs (Median Slopes)
+		/// Weight:
+		/// - Reward Destination Staked: O(N)
+		/// - Reward Destination Controller (Creating): O(N)
 		/// DB Weight:
 		/// - Read: EraElectionStatus, CurrentEra, HistoryDepth, ErasValidatorReward,
 		///         ErasStakersClipped, ErasRewardPoints, ErasValidatorPrefs (8 items)
 		/// - Read Each: Bonded, Ledger, Payee, Locks, System Account (5 items)
 		/// - Write Each: System Account, Locks, Ledger (3 items)
+		///
+		///   NOTE: weights are assuming that payouts are made to alive stash account (Staked).
+		///   Paying even a dead controller is cheaper weight-wise. We don't do any refunds here.
 		/// # </weight>
-		#[weight =
-			120 * WEIGHT_PER_MICROS
-			+ 54 * WEIGHT_PER_MICROS * Weight::from(T::MaxNominatorRewardedPerValidator::get())
-			+ T::DbWeight::get().reads(7)
-			+ T::DbWeight::get().reads(5)  * Weight::from(T::MaxNominatorRewardedPerValidator::get() + 1)
-			+ T::DbWeight::get().writes(3) * Weight::from(T::MaxNominatorRewardedPerValidator::get() + 1)
-		]
+		#[weight = T::WeightInfo::payout_stakers_alive_staked(T::MaxNominatorRewardedPerValidator::get())]
 		fn payout_stakers(origin, validator_stash: T::AccountId, era: EraIndex) -> DispatchResult {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			ensure_signed(origin)?;
@@ -2090,16 +1960,11 @@ decl_module! {
 		/// - Bounded by `MAX_UNLOCKING_CHUNKS`.
 		/// - Storage changes: Can't increase storage, only decrease it.
 		/// ---------------
-		/// - Base Weight: 34.51 µs * .048 L µs
 		/// - DB Weight:
 		///     - Reads: EraElectionStatus, Ledger, Locks, [Origin Account]
 		///     - Writes: [Origin Account], Locks, Ledger
 		/// # </weight>
-		#[weight =
-			35 * WEIGHT_PER_MICROS
-			+ 50 * WEIGHT_PER_NANOS * (MAX_UNLOCKING_CHUNKS as Weight)
-			+ T::DbWeight::get().reads_writes(3, 2)
-		]
+		#[weight = T::WeightInfo::rebond(MAX_UNLOCKING_CHUNKS as u32)]
 		fn rebond(origin, #[compact] value: BalanceOf<T>) -> DispatchResultWithPostInfo {
 			ensure!(Self::era_election_status().is_closed(), Error::<T>::CallNotAllowed);
 			let controller = ensure_signed(origin)?;
@@ -2129,19 +1994,14 @@ decl_module! {
 		///
 		/// # <weight>
 		/// - E: Number of history depths removed, i.e. 10 -> 7 = 3
-		/// - Base Weight: 29.13 * E µs
+		/// - Weight: O(E)
 		/// - DB Weight:
 		///     - Reads: Current Era, History Depth
 		///     - Writes: History Depth
 		///     - Clear Prefix Each: Era Stakers, EraStakersClipped, ErasValidatorPrefs
 		///     - Writes Each: ErasValidatorReward, ErasRewardPoints, ErasTotalStake, ErasStartSessionIndex
 		/// # </weight>
-		#[weight = {
-			let items = Weight::from(*_era_items_deleted);
-			T::DbWeight::get().reads_writes(2, 1)
-				.saturating_add(T::DbWeight::get().reads_writes(items, items))
-
-		}]
+		#[weight = T::WeightInfo::set_history_depth(*_era_items_deleted)]
 		fn set_history_depth(origin,
 			#[compact] new_history_depth: EraIndex,
 			#[compact] _era_items_deleted: u32,
@@ -2169,21 +2029,12 @@ decl_module! {
 		///
 		/// # <weight>
 		/// Complexity: O(S) where S is the number of slashing spans on the account.
-		/// Base Weight: 75.94 + 2.396 * S µs
 		/// DB Weight:
 		/// - Reads: Stash Account, Bonded, Slashing Spans, Locks
 		/// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Stash Account, Locks
 		/// - Writes Each: SpanSlash * S
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(4, 7)
-			.saturating_add(76 * WEIGHT_PER_MICROS)
-			.saturating_add(
-				WEIGHT_PER_MICROS.saturating_mul(2).saturating_mul(Weight::from(*num_slashing_spans))
-			)
-			.saturating_add(T::DbWeight::get().writes(Weight::from(*num_slashing_spans)))
-			// if slashing spans is non-zero, add 1 more write
-			.saturating_add(T::DbWeight::get().writes(Weight::from(*num_slashing_spans).min(1)))
-		]
+		#[weight = T::WeightInfo::reap_stash(*num_slashing_spans)]
 		fn reap_stash(_origin, stash: T::AccountId, num_slashing_spans: u32) {
 			ensure!(T::Currency::total_balance(&stash).is_zero(), Error::<T>::FundedTarget);
 			Self::kill_stash(&stash, num_slashing_spans)?;
@@ -2235,9 +2086,16 @@ decl_module! {
 		///    minimized (to ensure less variance)
 		///
 		/// # <weight>
-		/// See `crate::weight` module.
+		/// The transaction is assumed to be the longest path, a better solution.
+		///   - Initial solution is almost the same.
+		///   - Worse solution is retraced in pre-dispatch-checks which sets its own weight.
 		/// # </weight>
-		#[weight = weight::weight_for_submit_solution::<T>(winners, compact, size)]
+		#[weight = T::WeightInfo::submit_solution_better(
+			size.validators.into(),
+			size.nominators.into(),
+			compact.len() as u32,
+			winners.len() as u32,
+		)]
 		pub fn submit_election_solution(
 			origin,
 			winners: Vec<ValidatorIndex>,
@@ -2266,7 +2124,12 @@ decl_module! {
 		/// # <weight>
 		/// See `crate::weight` module.
 		/// # </weight>
-		#[weight = weight::weight_for_submit_solution::<T>(winners, compact, size)]
+		#[weight = T::WeightInfo::submit_solution_better(
+			size.validators.into(),
+			size.nominators.into(),
+			compact.len() as u32,
+			winners.len() as u32,
+		)]
 		pub fn submit_election_solution_unsigned(
 			origin,
 			winners: Vec<ValidatorIndex>,
@@ -2580,13 +2443,16 @@ impl<T: Trait> Module<T> {
 		election_size: ElectionSize,
 	) -> DispatchResultWithPostInfo {
 		// Do the basic checks. era, claimed score and window open.
-		Self::pre_dispatch_checks(claimed_score, era)?;
-		// the weight that we will refund in case of a correct submission. We compute this now
-		// because the data needed for it will be consumed further down.
-		let adjusted_weight = weight::weight_for_correct_submit_solution::<T>(
-			&winners,
-			&compact_assignments,
-			&election_size,
+		let _ = Self::pre_dispatch_checks(claimed_score, era)?;
+
+		// before we read any further state, we check that the unique targets in compact is same as
+		// compact. is a all in-memory check and easy to do. Moreover, it ensures that the solution
+		// is not full of bogus edges that can cause lots of reads to SlashingSpans. Thus, we can
+		// assume that the storage access of this function is always O(|winners|), not
+		// O(|compact.edge_count()|).
+		ensure!(
+			compact_assignments.unique_targets().len() == winners.len(),
+			Error::<T>::OffchainElectionBogusWinnerCount,
 		);
 
 		// Check that the number of presented winners is sane. Most often we have more candidates
@@ -2744,7 +2610,7 @@ impl<T: Trait> Module<T> {
 		// emit event.
 		Self::deposit_event(RawEvent::SolutionStored(compute));
 
-		Ok(Some(adjusted_weight).into())
+		Ok(None.into())
 	}
 
 	/// Start a session potentially starting an era.
@@ -2862,7 +2728,6 @@ impl<T: Trait> Module<T> {
 		maybe_new_validators
 	}
 
-
 	/// Remove all the storage items associated with the election.
 	fn close_election_window() {
 		// Close window.
diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs
index 02acd135e63e4..6354014232d50 100644
--- a/frame/staking/src/testing_utils.rs
+++ b/frame/staking/src/testing_utils.rs
@@ -20,7 +20,7 @@
 
 use crate::*;
 use crate::Module as Staking;
-use frame_benchmarking::{account};
+use frame_benchmarking::account;
 use frame_system::RawOrigin;
 use sp_io::hashing::blake2_256;
 use rand_chacha::{rand_core::{RngCore, SeedableRng}, ChaChaRng};
@@ -29,7 +29,11 @@ use sp_npos_elections::*;
 const SEED: u32 = 0;
 
 /// Grab a funded user.
-pub fn create_funded_user<T: Trait>(string: &'static str, n: u32, balance_factor: u32) -> T::AccountId {
+pub fn create_funded_user<T: Trait>(
+	string: &'static str,
+	n: u32,
+	balance_factor: u32,
+) -> T::AccountId {
 	let user = account(string, n, SEED);
 	let balance = T::Currency::minimum_balance() * balance_factor.into();
 	T::Currency::make_free_balance_be(&user, balance);
@@ -39,30 +43,36 @@ pub fn create_funded_user<T: Trait>(string: &'static str, n: u32, balance_factor
 }
 
 /// Create a stash and controller pair.
-pub fn create_stash_controller<T: Trait>(n: u32, balance_factor: u32)
+pub fn create_stash_controller<T: Trait>(
+	n: u32,
+	balance_factor: u32,
+	destination: RewardDestination<T::AccountId>,
+)
 	-> Result<(T::AccountId, T::AccountId), &'static str>
 {
 	let stash = create_funded_user::<T>("stash", n, balance_factor);
 	let controller = create_funded_user::<T>("controller", n, balance_factor);
 	let controller_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(controller.clone());
-	let reward_destination = RewardDestination::Staked;
 	let amount = T::Currency::minimum_balance() * (balance_factor / 10).max(1).into();
-	Staking::<T>::bond(RawOrigin::Signed(stash.clone()).into(), controller_lookup, amount, reward_destination)?;
+	Staking::<T>::bond(RawOrigin::Signed(stash.clone()).into(), controller_lookup, amount, destination)?;
 	return Ok((stash, controller))
 }
 
 /// Create a stash and controller pair, where the controller is dead, and payouts go to controller.
 /// This is used to test worst case payout scenarios.
-pub fn create_stash_and_dead_controller<T: Trait>(n: u32, balance_factor: u32)
+pub fn create_stash_and_dead_controller<T: Trait>(
+	n: u32,
+	balance_factor: u32,
+	destination: RewardDestination<T::AccountId>,
+)
 	-> Result<(T::AccountId, T::AccountId), &'static str>
 {
 	let stash = create_funded_user::<T>("stash", n, balance_factor);
 	// controller has no funds
 	let controller = create_funded_user::<T>("controller", n, 0);
 	let controller_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(controller.clone());
-	let reward_destination = RewardDestination::Controller;
 	let amount = T::Currency::minimum_balance() * (balance_factor / 10).max(1).into();
-	Staking::<T>::bond(RawOrigin::Signed(stash.clone()).into(), controller_lookup, amount, reward_destination)?;
+	Staking::<T>::bond(RawOrigin::Signed(stash.clone()).into(), controller_lookup, amount, destination)?;
 	return Ok((stash, controller))
 }
 
@@ -73,7 +83,7 @@ pub fn create_validators<T: Trait>(
 ) -> Result<Vec<<T::Lookup as StaticLookup>::Source>, &'static str> {
 	let mut validators: Vec<<T::Lookup as StaticLookup>::Source> = Vec::with_capacity(max as usize);
 	for i in 0 .. max {
-		let (stash, controller) = create_stash_controller::<T>(i, balance_factor)?;
+		let (stash, controller) = create_stash_controller::<T>(i, balance_factor, RewardDestination::Staked)?;
 		let validator_prefs = ValidatorPrefs {
 			commission: Perbill::from_percent(50),
 		};
@@ -110,7 +120,7 @@ pub fn create_validators_with_nominators_for_era<T: Trait>(
 	// Create validators
 	for i in 0 .. validators {
 		let balance_factor = if randomize_stake { rng.next_u32() % 255 + 10 } else { 100u32 };
-		let (v_stash, v_controller) = create_stash_controller::<T>(i, balance_factor)?;
+		let (v_stash, v_controller) = create_stash_controller::<T>(i, balance_factor, RewardDestination::Staked)?;
 		let validator_prefs = ValidatorPrefs {
 			commission: Perbill::from_percent(50),
 		};
@@ -128,6 +138,7 @@ pub fn create_validators_with_nominators_for_era<T: Trait>(
 		let (_n_stash, n_controller) = create_stash_controller::<T>(
 			u32::max_value() - j,
 			balance_factor,
+			RewardDestination::Staked,
 		)?;
 
 		// Have them randomly validate
diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs
index d27654d1feae6..0f5d08a3a8ccc 100644
--- a/frame/staking/src/tests.rs
+++ b/frame/staking/src/tests.rs
@@ -3615,7 +3615,7 @@ mod offchain_election {
 		// A validator index which is out of bound
 		ExtBuilder::default()
 			.offchain_election_ext()
-			.validator_count(4)
+			.validator_count(2)
 			.has_stakers(false)
 			.build()
 			.execute_with(|| {
@@ -3627,7 +3627,7 @@ mod offchain_election {
 				let (mut compact, winners, score) = prepare_submission_with(true, 2, |_| {});
 
 				// index 4 doesn't exist.
-				compact.votes1.push((3, 4));
+				compact.votes1.iter_mut().for_each(|(_, vidx)| if *vidx == 1 { *vidx = 4 });
 
 				// The error type sadly cannot be more specific now.
 				assert_noop!(
@@ -3688,12 +3688,57 @@ mod offchain_election {
 
 				assert_eq!(Staking::snapshot_nominators().unwrap().len(), 5 + 4);
 				assert_eq!(Staking::snapshot_validators().unwrap().len(), 4);
+				let (compact, winners, score) = prepare_submission_with(true, 2, |a| {
+					// swap all 11 and 41s in the distribution with non-winners. Note that it is
+					// important that the count of winners and the count of unique targets remain
+					// valid.
+					a.iter_mut().for_each(| StakedAssignment { who, distribution } |
+						distribution.iter_mut().for_each(|(t, _)| {
+							if *t == 41 { *t = 31 } else { *t = 21 }
+							// if it is self vote, correct that.
+							if *who == 41 { *who = 31 }
+							if *who == 11 { *who = 21 }
+						})
+					);
+				});
+
+				assert_noop!(
+					submit_solution(
+						Origin::signed(10),
+						winners,
+						compact,
+						score,
+					),
+					Error::<Test>::OffchainElectionBogusNomination,
+				);
+			})
+	}
+
+	#[test]
+	fn offchain_election_unique_target_count_is_checked() {
+		// Number of unique targets and and winners.len must match.
+		ExtBuilder::default()
+			.offchain_election_ext()
+			.validator_count(2) // we select only 2.
+			.has_stakers(false)
+			.build()
+			.execute_with(|| {
+				build_offchain_election_test_ext();
+				run_to_block(12);
+
+				assert_eq!(Staking::snapshot_nominators().unwrap().len(), 5 + 4);
+				assert_eq!(Staking::snapshot_validators().unwrap().len(), 4);
+
 				let (compact, winners, score) = prepare_submission_with(true, 2, |a| {
 					a.iter_mut()
 						.find(|x| x.who == 5)
-						// all 3 cannot be among the winners. Although, all of them are validator
-						// candidates.
-						.map(|x| x.distribution = vec![(21, 50), (41, 30), (31, 20)]);
+						// just add any new target.
+						.map(|x| {
+							// old value.
+							assert_eq!(x.distribution, vec![(41, 100)]);
+							// new value.
+							x.distribution = vec![(21, 50), (41, 50)]
+						});
 				});
 
 				assert_noop!(
@@ -3703,7 +3748,7 @@ mod offchain_election {
 						compact,
 						score,
 					),
-					Error::<Test>::OffchainElectionBogusEdge,
+					Error::<Test>::OffchainElectionBogusWinnerCount,
 				);
 			})
 	}
diff --git a/primitives/npos-elections/compact/src/lib.rs b/primitives/npos-elections/compact/src/lib.rs
index 54c94b6df65c2..134f3f59ff177 100644
--- a/primitives/npos-elections/compact/src/lib.rs
+++ b/primitives/npos-elections/compact/src/lib.rs
@@ -149,22 +149,9 @@ fn struct_def(
 		)
 	}).collect::<TokenStream2>();
 
-
-	let len_impl = (1..=count).map(|c| {
-		let field_name = field_name_for(c);
-		quote!(
-			all_len = all_len.saturating_add(self.#field_name.len());
-		)
-	}).collect::<TokenStream2>();
-
-	let edge_count_impl = (1..=count).map(|c| {
-		let field_name = field_name_for(c);
-		quote!(
-			all_edges = all_edges.saturating_add(
-				self.#field_name.len().saturating_mul(#c as usize)
-			);
-		)
-	}).collect::<TokenStream2>();
+	let len_impl = len_impl(count);
+	let edge_count_impl = edge_count_impl(count);
+	let unique_targets_impl = unique_targets_impl(count);
 
 	let derives_and_maybe_compact_encoding = if compact_encoding {
 		// custom compact encoding.
@@ -209,6 +196,26 @@ fn struct_def(
 				all_edges
 			}
 
+			/// Get the number of unique targets in the whole struct.
+			///
+			/// Once presented with a list of winners, this set and the set of winners must be
+			/// equal.
+			///
+			/// The resulting indices are sorted.
+			pub fn unique_targets(&self) -> Vec<#target_type> {
+				let mut all_targets: Vec<#target_type> = Vec::with_capacity(self.average_edge_count());
+				let mut maybe_insert_target = |t: #target_type| {
+					match all_targets.binary_search(&t) {
+						Ok(_) => (),
+						Err(pos) => all_targets.insert(pos, t)
+					}
+				};
+
+				#unique_targets_impl
+
+				all_targets
+			}
+
 			/// Get the average edge count.
 			pub fn average_edge_count(&self) -> usize {
 				self.edge_count().checked_div(self.len()).unwrap_or(0)
@@ -217,6 +224,65 @@ fn struct_def(
 	))
 }
 
+fn len_impl(count: usize) -> TokenStream2 {
+	(1..=count).map(|c| {
+		let field_name = field_name_for(c);
+		quote!(
+			all_len = all_len.saturating_add(self.#field_name.len());
+		)
+	}).collect::<TokenStream2>()
+}
+
+fn edge_count_impl(count: usize) -> TokenStream2 {
+	(1..=count).map(|c| {
+		let field_name = field_name_for(c);
+		quote!(
+			all_edges = all_edges.saturating_add(
+				self.#field_name.len().saturating_mul(#c as usize)
+			);
+		)
+	}).collect::<TokenStream2>()
+}
+
+fn unique_targets_impl(count: usize) -> TokenStream2 {
+	let unique_targets_impl_single = {
+		let field_name = field_name_for(1);
+		quote! {
+			self.#field_name.iter().for_each(|(_, t)| {
+				maybe_insert_target(*t);
+			});
+		}
+	};
+
+	let unique_targets_impl_double = {
+		let field_name = field_name_for(2);
+		quote! {
+			self.#field_name.iter().for_each(|(_, (t1, _), t2)| {
+				maybe_insert_target(*t1);
+				maybe_insert_target(*t2);
+			});
+		}
+	};
+
+	let unique_targets_impl_rest = (3..=count).map(|c| {
+		let field_name = field_name_for(c);
+		quote! {
+			self.#field_name.iter().for_each(|(_, inners, t_last)| {
+				inners.iter().for_each(|(t, _)| {
+					maybe_insert_target(*t);
+				});
+				maybe_insert_target(*t_last);
+			});
+		}
+	}).collect::<TokenStream2>();
+
+	quote! {
+		#unique_targets_impl_single
+		#unique_targets_impl_double
+		#unique_targets_impl_rest
+	}
+}
+
 fn imports() -> Result<TokenStream2> {
 	if std::env::var("CARGO_PKG_NAME").unwrap() == "sp-npos-elections" {
 		Ok(quote! {
diff --git a/primitives/npos-elections/src/tests.rs b/primitives/npos-elections/src/tests.rs
index 8e99d2222e885..d1769acd08144 100644
--- a/primitives/npos-elections/src/tests.rs
+++ b/primitives/npos-elections/src/tests.rs
@@ -1002,6 +1002,7 @@ mod solution_type {
 		);
 		assert_eq!(compact.len(), 4);
 		assert_eq!(compact.edge_count(), 2 + 4);
+		assert_eq!(compact.unique_targets(), vec![10, 11, 20, 40, 50, 51]);
 	}
 
 	#[test]
@@ -1097,6 +1098,11 @@ mod solution_type {
 			}
 		);
 
+		assert_eq!(
+			compacted.unique_targets(),
+			vec![0, 1, 2, 3, 4, 5, 6, 7, 8],
+		);
+
 		let voter_at = |a: u32| -> Option<AccountId> {
 			voters.get(<u32 as TryInto<usize>>::try_into(a).unwrap()).cloned()
 		};
@@ -1110,6 +1116,69 @@ mod solution_type {
 		);
 	}
 
+	#[test]
+	fn unique_targets_len_edge_count_works() {
+		const ACC: TestAccuracy = TestAccuracy::from_percent(10);
+
+		// we don't really care about voters here so all duplicates. This is not invalid per se.
+		let compact = TestSolutionCompact {
+			votes1: vec![(99, 1), (99, 2)],
+			votes2: vec![
+				(99, (3, ACC.clone()), 7),
+				(99, (4, ACC.clone()), 8),
+			],
+			votes3: vec![
+				(99, [(11, ACC.clone()), (12, ACC.clone())], 13),
+			],
+			// ensure the last one is also counted.
+			votes16: vec![
+				(
+					99,
+					[
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+						(66, ACC.clone()),
+					],
+					67,
+				)
+			],
+			..Default::default()
+		};
+
+		assert_eq!(compact.unique_targets(), vec![1, 2, 3, 4, 7, 8, 11, 12, 13, 66, 67]);
+		assert_eq!(compact.edge_count(), 2 + (2 * 2) + 3 + 16);
+		assert_eq!(compact.len(), 6);
+
+		// this one has some duplicates.
+		let compact = TestSolutionCompact {
+			votes1: vec![(99, 1), (99, 1)],
+			votes2: vec![
+				(99, (3, ACC.clone()), 7),
+				(99, (4, ACC.clone()), 8),
+			],
+			votes3: vec![
+				(99, [(11, ACC.clone()), (11, ACC.clone())], 13),
+			],
+			..Default::default()
+		};
+
+		assert_eq!(compact.unique_targets(), vec![1, 3, 4, 7, 8, 11, 13]);
+		assert_eq!(compact.edge_count(), 2 + (2 * 2) + 3);
+		assert_eq!(compact.len(), 5);
+	}
+
 	#[test]
 	fn compact_into_assignment_must_report_overflow() {
 		// in votes2
@@ -1165,7 +1234,7 @@ mod solution_type {
 		assert_eq!(compacted.unwrap_err(), PhragmenError::CompactTargetOverflow);
 	}
 
-		#[test]
+	#[test]
 	fn zero_target_count_is_ignored() {
 		let voters = vec![1 as AccountId, 2];
 		let targets = vec![10 as AccountId, 11];

From 4e4c321217cc0c08a707068d2535a1e688543e97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Wed, 16 Sep 2020 16:43:43 +0200
Subject: [PATCH 093/122] Send import notification always for re-orgs (#7118)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Send import notification always for re-orgs

This pr changes the behavior of sending import notifications. Before we
only send notifications when importing blocks on the tip of the chain or
on similar conditions. However we did not send a notification when we
for example being in a state where we import multiple blocks to catch
up. If we re-org in this process, systems like the transaction pool
would not be notified about this re-org. This means, that we would also
not resubmit transactions of these retracted blocks. This pr fixes this,
by always sending a notification on a re-org.

See
https://github.com/substrate-developer-hub/substrate-node-template/issues/82
for some context about the bug.

* Update client/service/src/client/client.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
---
 client/service/src/client/client.rs   |  3 +-
 client/service/test/src/client/mod.rs | 54 +++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs
index d0859f4ee0392..fd76084988dbc 100644
--- a/client/service/src/client/client.rs
+++ b/client/service/src/client/client.rs
@@ -802,7 +802,8 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 
 		operation.op.insert_aux(aux)?;
 
-		if make_notifications {
+		// we only notify when we are already synced to the tip of the chain or if this import triggers a re-org
+		if make_notifications || tree_route.is_some() {
 			if finalized {
 				operation.notify_finalized.push(hash);
 			}
diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs
index 8d073df272fd9..ea3eaa7ffbabf 100644
--- a/client/service/test/src/client/mod.rs
+++ b/client/service/test/src/client/mod.rs
@@ -1802,3 +1802,57 @@ fn cleans_up_closed_notification_sinks_on_block_import() {
 	assert_eq!(client.finality_notification_sinks().lock().len(), 0);
 }
 
+/// Test that ensures that we always send an import notification for re-orgs.
+#[test]
+fn reorg_triggers_a_notification_even_for_sources_that_should_not_trigger_notifications() {
+	let mut client = TestClientBuilder::new().build();
+
+	let mut notification_stream = futures::executor::block_on_stream(
+		client.import_notification_stream()
+	);
+
+	let a1 = client.new_block_at(
+		&BlockId::Number(0),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
+	client.import(BlockOrigin::NetworkInitialSync, a1.clone()).unwrap();
+
+	let a2 = client.new_block_at(
+		&BlockId::Hash(a1.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
+	client.import(BlockOrigin::NetworkInitialSync, a2.clone()).unwrap();
+
+	let mut b1 = client.new_block_at(
+		&BlockId::Number(0),
+		Default::default(),
+		false,
+	).unwrap();
+	// needed to make sure B1 gets a different hash from A1
+	b1.push_transfer(Transfer {
+		from: AccountKeyring::Alice.into(),
+		to: AccountKeyring::Ferdie.into(),
+		amount: 1,
+		nonce: 0,
+	}).unwrap();
+	let b1 = b1.build().unwrap().block;
+	client.import(BlockOrigin::NetworkInitialSync, b1.clone()).unwrap();
+
+	let b2 = client.new_block_at(
+		&BlockId::Hash(b1.hash()),
+		Default::default(),
+		false,
+	).unwrap().build().unwrap().block;
+
+	// Should trigger a notification because we reorg
+	client.import_as_best(BlockOrigin::NetworkInitialSync, b2.clone()).unwrap();
+
+	// There should be one notification
+	let notification = notification_stream.next().unwrap();
+
+	// We should have a tree route of the re-org
+	let tree_route = notification.tree_route.unwrap();
+	assert_eq!(tree_route.enacted()[0].hash, b1.hash());
+}

From 9167fe0e6c91e9cd77fbebba0f0447ee3203cd90 Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Wed, 16 Sep 2020 21:48:10 +0200
Subject: [PATCH 094/122] WeightInfo for Vesting Pallet (#7103)

* WeightInfo for Vesting Pallet

* clean up weight docs

* Update lib.rs

* try to pipe max locks

* Update for new type

* add warning when locks > MaxLocks

* Update lib.rs

* fix compile

* remove aliasing, fix trait def

* Update
---
 bin/node-template/runtime/src/lib.rs          |  4 +-
 bin/node/runtime/src/lib.rs                   |  6 +-
 bin/node/runtime/src/weights/mod.rs           |  1 +
 .../runtime/src/weights/pallet_vesting.rs     | 63 +++++++++++++++++
 frame/atomic-swap/src/tests.rs                |  1 +
 frame/babe/src/mock.rs                        |  1 +
 frame/balances/src/lib.rs                     | 18 +++++
 frame/balances/src/tests_composite.rs         |  2 +
 frame/balances/src/tests_local.rs             |  4 ++
 frame/contracts/src/tests.rs                  |  1 +
 frame/democracy/src/tests.rs                  |  1 +
 frame/elections-phragmen/src/lib.rs           |  1 +
 frame/elections/src/mock.rs                   |  1 +
 frame/evm/src/tests.rs                        |  1 +
 frame/example/src/lib.rs                      |  1 +
 frame/executive/src/lib.rs                    |  1 +
 frame/generic-asset/src/lib.rs                |  2 +
 frame/grandpa/src/mock.rs                     | 70 +++++++++----------
 frame/identity/src/lib.rs                     |  1 +
 frame/indices/src/mock.rs                     |  1 +
 frame/multisig/src/tests.rs                   |  1 +
 frame/nicks/src/lib.rs                        |  1 +
 frame/offences/benchmarking/src/mock.rs       |  1 +
 frame/proxy/src/tests.rs                      |  1 +
 frame/recovery/src/mock.rs                    |  1 +
 frame/scored-pool/src/mock.rs                 |  1 +
 frame/session/benchmarking/src/mock.rs        |  1 +
 frame/society/src/mock.rs                     |  1 +
 frame/staking/fuzzer/src/mock.rs              |  1 +
 frame/staking/src/mock.rs                     |  1 +
 frame/support/src/traits.rs                   |  3 +
 frame/transaction-payment/src/lib.rs          |  1 +
 frame/treasury/src/tests.rs                   |  1 +
 frame/utility/src/tests.rs                    |  1 +
 frame/vesting/src/benchmarking.rs             | 44 ++++++++++--
 frame/vesting/src/default_weights.rs          | 62 ++++++++++++++++
 frame/vesting/src/lib.rs                      | 41 +++++------
 37 files changed, 273 insertions(+), 71 deletions(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_vesting.rs
 create mode 100644 frame/vesting/src/default_weights.rs

diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs
index 06e34e4551673..91fffb7f8e155 100644
--- a/bin/node-template/runtime/src/lib.rs
+++ b/bin/node-template/runtime/src/lib.rs
@@ -227,9 +227,11 @@ impl pallet_timestamp::Trait for Runtime {
 
 parameter_types! {
 	pub const ExistentialDeposit: u128 = 500;
+	pub const MaxLocks: u32 = 50;
 }
 
 impl pallet_balances::Trait for Runtime {
+	type MaxLocks = MaxLocks;
 	/// The type for recording an account's balance.
 	type Balance = Balance;
 	/// The ubiquitous event type.
@@ -423,7 +425,7 @@ impl_runtime_apis! {
 			None
 		}
 	}
-	
+
 	impl frame_system_rpc_runtime_api::AccountNonceApi<Block, AccountId, Index> for Runtime {
 		fn account_nonce(account: AccountId) -> Index {
 			System::account_nonce(account)
diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index eeac6d83b8777..5737fcfd2e2f1 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -325,9 +325,13 @@ impl pallet_indices::Trait for Runtime {
 
 parameter_types! {
 	pub const ExistentialDeposit: Balance = 1 * DOLLARS;
+	// For weight estimation, we assume that the most locks on an individual account will be 50.
+	// This number may need to be adjusted in the future if this assumption no longer holds true.
+	pub const MaxLocks: u32 = 50;
 }
 
 impl pallet_balances::Trait for Runtime {
+	type MaxLocks = MaxLocks;
 	type Balance = Balance;
 	type DustRemoval = ();
 	type Event = Event;
@@ -856,7 +860,7 @@ impl pallet_vesting::Trait for Runtime {
 	type Currency = Balances;
 	type BlockNumberToBalance = ConvertInto;
 	type MinVestedTransfer = MinVestedTransfer;
-	type WeightInfo = ();
+	type WeightInfo = weights::pallet_vesting::WeightInfo;
 }
 
 construct_runtime!(
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index 372b13a093e27..86cab773b18e3 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -22,3 +22,4 @@ pub mod pallet_democracy;
 pub mod pallet_proxy;
 pub mod pallet_timestamp;
 pub mod pallet_utility;
+pub mod pallet_vesting;
diff --git a/bin/node/runtime/src/weights/pallet_vesting.rs b/bin/node/runtime/src/weights/pallet_vesting.rs
new file mode 100644
index 0000000000000..b2a4b57e64415
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_vesting.rs
@@ -0,0 +1,63 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_vesting::WeightInfo for WeightInfo {
+	fn vest_locked(l: u32, ) -> Weight {
+		(82109000 as Weight)
+			.saturating_add((332000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn vest_unlocked(l: u32, ) -> Weight {
+		(88419000 as Weight)
+			.saturating_add((3000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn vest_other_locked(l: u32, ) -> Weight {
+		(81277000 as Weight)
+			.saturating_add((321000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn vest_other_unlocked(l: u32, ) -> Weight {
+		(87584000 as Weight)
+			.saturating_add((19000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn vested_transfer(l: u32, ) -> Weight {
+		(185916000 as Weight)
+			.saturating_add((625000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn force_vested_transfer(l: u32, ) -> Weight {
+		(185916000 as Weight)
+			.saturating_add((625000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+}
diff --git a/frame/atomic-swap/src/tests.rs b/frame/atomic-swap/src/tests.rs
index 6690a24d364d6..528203fc39096 100644
--- a/frame/atomic-swap/src/tests.rs
+++ b/frame/atomic-swap/src/tests.rs
@@ -55,6 +55,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type DustRemoval = ();
 	type Event = ();
diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs
index 8a0356d8da7e8..34e9ff113d42c 100644
--- a/frame/babe/src/mock.rs
+++ b/frame/babe/src/mock.rs
@@ -152,6 +152,7 @@ parameter_types! {
 }
 
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u128;
 	type DustRemoval = ();
 	type Event = ();
diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs
index 331c5a27dfa74..471efb90bf3ff 100644
--- a/frame/balances/src/lib.rs
+++ b/frame/balances/src/lib.rs
@@ -200,6 +200,10 @@ pub trait Subtrait<I: Instance = DefaultInstance>: frame_system::Trait {
 
 	/// Weight information for the extrinsics in this pallet.
 	type WeightInfo: WeightInfo;
+
+	/// The maximum number of locks that should exist on an account.
+	/// Not strictly enforced, but used for weight estimation.
+	type MaxLocks: Get<u32>;
 }
 
 pub trait Trait<I: Instance = DefaultInstance>: frame_system::Trait {
@@ -221,6 +225,10 @@ pub trait Trait<I: Instance = DefaultInstance>: frame_system::Trait {
 
 	/// Weight information for extrinsics in this pallet.
 	type WeightInfo: WeightInfo;
+
+	/// The maximum number of locks that should exist on an account.
+	/// Not strictly enforced, but used for weight estimation.
+	type MaxLocks: Get<u32>;
 }
 
 impl<T: Trait<I>, I: Instance> Subtrait<I> for T {
@@ -228,6 +236,7 @@ impl<T: Trait<I>, I: Instance> Subtrait<I> for T {
 	type ExistentialDeposit = T::ExistentialDeposit;
 	type AccountStore = T::AccountStore;
 	type WeightInfo = <T as Trait<I>>::WeightInfo;
+	type MaxLocks = T::MaxLocks;
 }
 
 decl_event!(
@@ -663,6 +672,12 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 
 	/// Update the account entry for `who`, given the locks.
 	fn update_locks(who: &T::AccountId, locks: &[BalanceLock<T::Balance>]) {
+		if locks.len() as u32 > T::MaxLocks::get() {
+			frame_support::debug::warn!(
+				"Warning: A user has more currency locks than expected. \
+				A runtime configuration adjustment may be needed."
+			);
+		}
 		Self::mutate_account(who, |b| {
 			b.misc_frozen = Zero::zero();
 			b.fee_frozen = Zero::zero();
@@ -900,6 +915,7 @@ impl<T: Subtrait<I>, I: Instance> Trait<I> for ElevatedTrait<T, I> {
 	type ExistentialDeposit = T::ExistentialDeposit;
 	type AccountStore = T::AccountStore;
 	type WeightInfo = <T as Subtrait<I>>::WeightInfo;
+	type MaxLocks = T::MaxLocks;
 }
 
 impl<T: Trait<I>, I: Instance> Currency<T::AccountId> for Module<T, I> where
@@ -1285,6 +1301,8 @@ where
 {
 	type Moment = T::BlockNumber;
 
+	type MaxLocks = T::MaxLocks;
+
 	// Set a lock on the balance of `who`.
 	// Is a no-op if lock amount is zero or `reasons` `is_none()`.
 	fn set_lock(
diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs
index 8e764112ba24c..90ad145f22521 100644
--- a/frame/balances/src/tests_composite.rs
+++ b/frame/balances/src/tests_composite.rs
@@ -103,12 +103,14 @@ impl pallet_transaction_payment::Trait for Test {
 	type WeightToFee = IdentityFee<u64>;
 	type FeeMultiplierUpdate = ();
 }
+
 impl Trait for Test {
 	type Balance = u64;
 	type DustRemoval = ();
 	type Event = Event;
 	type ExistentialDeposit = ExistentialDeposit;
 	type AccountStore = system::Module<Test>;
+	type MaxLocks = ();
 	type WeightInfo = ();
 }
 
diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs
index 86abc2b6044ce..75813c6b1bc9a 100644
--- a/frame/balances/src/tests_local.rs
+++ b/frame/balances/src/tests_local.rs
@@ -103,6 +103,9 @@ impl pallet_transaction_payment::Trait for Test {
 	type WeightToFee = IdentityFee<u64>;
 	type FeeMultiplierUpdate = ();
 }
+parameter_types! {
+	pub const MaxLocks: u32 = 50;
+}
 impl Trait for Test {
 	type Balance = u64;
 	type DustRemoval = ();
@@ -114,6 +117,7 @@ impl Trait for Test {
 		system::CallKillAccount<Test>,
 		u64, super::AccountData<u64>
 	>;
+	type MaxLocks = MaxLocks;
 	type WeightInfo = ();
 }
 
diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs
index bd1242ff6701a..1c30021793146 100644
--- a/frame/contracts/src/tests.rs
+++ b/frame/contracts/src/tests.rs
@@ -144,6 +144,7 @@ impl frame_system::Trait for Test {
 	type SystemWeightInfo = ();
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type Event = MetaEvent;
 	type DustRemoval = ();
diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs
index 13c6a09a04bc1..aed6739c77ebb 100644
--- a/frame/democracy/src/tests.rs
+++ b/frame/democracy/src/tests.rs
@@ -134,6 +134,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type Event = Event;
 	type DustRemoval = ();
diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs
index 0b93dd6c13b9c..93b11e8d95c57 100644
--- a/frame/elections-phragmen/src/lib.rs
+++ b/frame/elections-phragmen/src/lib.rs
@@ -1156,6 +1156,7 @@ mod tests {
 		type DustRemoval = ();
 		type ExistentialDeposit = ExistentialDeposit;
 		type AccountStore = frame_system::Module<Test>;
+		type MaxLocks = ();
 		type WeightInfo = ();
 	}
 
diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs
index c9b2523c4bc8a..adde24c25d32f 100644
--- a/frame/elections/src/mock.rs
+++ b/frame/elections/src/mock.rs
@@ -70,6 +70,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type DustRemoval = ();
 	type Event = Event;
diff --git a/frame/evm/src/tests.rs b/frame/evm/src/tests.rs
index 652d6c723b9d3..f741e3e4fc048 100644
--- a/frame/evm/src/tests.rs
+++ b/frame/evm/src/tests.rs
@@ -63,6 +63,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type DustRemoval = ();
 	type Event = ();
diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs
index b41c8196c018f..e2b00daf31cd7 100644
--- a/frame/example/src/lib.rs
+++ b/frame/example/src/lib.rs
@@ -774,6 +774,7 @@ mod tests {
 		pub const ExistentialDeposit: u64 = 1;
 	}
 	impl pallet_balances::Trait for Test {
+		type MaxLocks = ();
 		type Balance = u64;
 		type DustRemoval = ();
 		type Event = ();
diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs
index 24dccf8b0b4a4..cd9642fb82c9d 100644
--- a/frame/executive/src/lib.rs
+++ b/frame/executive/src/lib.rs
@@ -584,6 +584,7 @@ mod tests {
 		type DustRemoval = ();
 		type ExistentialDeposit = ExistentialDeposit;
 		type AccountStore = System;
+		type MaxLocks = ();
 		type WeightInfo = ();
 	}
 
diff --git a/frame/generic-asset/src/lib.rs b/frame/generic-asset/src/lib.rs
index 534a97cf5372a..6c3683312d0a8 100644
--- a/frame/generic-asset/src/lib.rs
+++ b/frame/generic-asset/src/lib.rs
@@ -1340,6 +1340,8 @@ where
 {
 	type Moment = T::BlockNumber;
 
+	type MaxLocks = ();
+
 	fn set_lock(
 		id: LockIdentifier,
 		who: &T::AccountId,
diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs
index 684712df7d078..81026c756273a 100644
--- a/frame/grandpa/src/mock.rs
+++ b/frame/grandpa/src/mock.rs
@@ -41,21 +41,14 @@ use sp_runtime::{
 };
 use sp_staking::SessionIndex;
 
-use frame_system as system;
-use pallet_balances as balances;
-use pallet_offences as offences;
-use pallet_session as session;
-use pallet_staking as staking;
-use pallet_timestamp as timestamp;
-
 impl_outer_origin! {
 	pub enum Origin for Test {}
 }
 
 impl_outer_dispatch! {
 	pub enum Call for Test where origin: Origin {
-		grandpa::Grandpa,
-		staking::Staking,
+		pallet_grandpa::Grandpa,
+		pallet_staking::Staking,
 	}
 }
 
@@ -67,12 +60,12 @@ impl_opaque_keys! {
 
 impl_outer_event! {
 	pub enum TestEvent for Test {
-		system<T>,
-		balances<T>,
-		grandpa,
-		offences,
-		session,
-		staking<T>,
+		frame_system<T>,
+		pallet_balances<T>,
+		pallet_grandpa,
+		pallet_offences,
+		pallet_session,
+		pallet_staking<T>,
 	}
 }
 
@@ -108,13 +101,13 @@ impl frame_system::Trait for Test {
 	type AvailableBlockRatio = AvailableBlockRatio;
 	type Version = ();
 	type ModuleToIndex = ();
-	type AccountData = balances::AccountData<u128>;
+	type AccountData = pallet_balances::AccountData<u128>;
 	type OnNewAccount = ();
 	type OnKilledAccount = ();
 	type SystemWeightInfo = ();
 }
 
-impl<C> system::offchain::SendTransactionTypes<C> for Test
+impl<C> frame_system::offchain::SendTransactionTypes<C> for Test
 where
 	Call: From<C>,
 {
@@ -129,22 +122,22 @@ parameter_types! {
 }
 
 /// Custom `SessionHandler` since we use `TestSessionKeys` as `Keys`.
-impl session::Trait for Test {
+impl pallet_session::Trait for Test {
 	type Event = TestEvent;
 	type ValidatorId = u64;
-	type ValidatorIdOf = staking::StashOf<Self>;
-	type ShouldEndSession = session::PeriodicSessions<Period, Offset>;
-	type NextSessionRotation = session::PeriodicSessions<Period, Offset>;
-	type SessionManager = session::historical::NoteHistoricalRoot<Self, Staking>;
+	type ValidatorIdOf = pallet_staking::StashOf<Self>;
+	type ShouldEndSession = pallet_session::PeriodicSessions<Period, Offset>;
+	type NextSessionRotation = pallet_session::PeriodicSessions<Period, Offset>;
+	type SessionManager = pallet_session::historical::NoteHistoricalRoot<Self, Staking>;
 	type SessionHandler = <TestSessionKeys as OpaqueKeys>::KeyTypeIdProviders;
 	type Keys = TestSessionKeys;
 	type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
 	type WeightInfo = ();
 }
 
-impl session::historical::Trait for Test {
-	type FullIdentification = staking::Exposure<u64, u128>;
-	type FullIdentificationOf = staking::ExposureOf<Self>;
+impl pallet_session::historical::Trait for Test {
+	type FullIdentification = pallet_staking::Exposure<u64, u128>;
+	type FullIdentificationOf = pallet_staking::ExposureOf<Self>;
 }
 
 parameter_types! {
@@ -162,7 +155,8 @@ parameter_types! {
 	pub const ExistentialDeposit: u128 = 1;
 }
 
-impl balances::Trait for Test {
+impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u128;
 	type DustRemoval = ();
 	type Event = TestEvent;
@@ -175,7 +169,7 @@ parameter_types! {
 	pub const MinimumPeriod: u64 = 3;
 }
 
-impl timestamp::Trait for Test {
+impl pallet_timestamp::Trait for Test {
 	type Moment = u64;
 	type OnTimestampSet = ();
 	type MinimumPeriod = MinimumPeriod;
@@ -218,7 +212,7 @@ impl Convert<u128, u64> for CurrencyToVoteHandler {
 	}
 }
 
-impl staking::Trait for Test {
+impl pallet_staking::Trait for Test {
 	type RewardRemainder = ();
 	type CurrencyToVote = CurrencyToVoteHandler;
 	type Event = TestEvent;
@@ -228,9 +222,9 @@ impl staking::Trait for Test {
 	type SessionsPerEra = SessionsPerEra;
 	type BondingDuration = BondingDuration;
 	type SlashDeferDuration = SlashDeferDuration;
-	type SlashCancelOrigin = system::EnsureRoot<Self::AccountId>;
+	type SlashCancelOrigin = frame_system::EnsureRoot<Self::AccountId>;
 	type SessionInterface = Self;
-	type UnixTime = timestamp::Module<Test>;
+	type UnixTime = pallet_timestamp::Module<Test>;
 	type RewardCurve = RewardCurve;
 	type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator;
 	type NextNewSession = Session;
@@ -246,9 +240,9 @@ parameter_types! {
 	pub OffencesWeightSoftLimit: Weight = Perbill::from_percent(60) * MaximumBlockWeight::get();
 }
 
-impl offences::Trait for Test {
+impl pallet_offences::Trait for Test {
 	type Event = TestEvent;
-	type IdentificationTuple = session::historical::IdentificationTuple<Self>;
+	type IdentificationTuple = pallet_session::historical::IdentificationTuple<Self>;
 	type OnOffenceHandler = Staking;
 	type WeightSoftLimit = OffencesWeightSoftLimit;
 	type WeightInfo = ();
@@ -271,7 +265,7 @@ impl Trait for Test {
 	type HandleEquivocation = super::EquivocationHandler<Self::KeyOwnerIdentification, Offences>;
 }
 
-mod grandpa {
+mod pallet_grandpa {
 	pub use crate::Event;
 }
 
@@ -331,7 +325,7 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx
 				i as u64,
 				i as u64 + 1000,
 				10_000,
-				staking::StakerStatus::<u64>::Validator,
+				pallet_staking::StakerStatus::<u64>::Validator,
 			)
 		})
 		.collect();
@@ -342,18 +336,18 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx
 
 	// NOTE: this will initialize the grandpa authorities
 	// through OneSessionHandler::on_genesis_session
-	session::GenesisConfig::<Test> { keys: session_keys }
+	pallet_session::GenesisConfig::<Test> { keys: session_keys }
 		.assimilate_storage(&mut t)
 		.unwrap();
 
-	balances::GenesisConfig::<Test> { balances }
+	pallet_balances::GenesisConfig::<Test> { balances }
 		.assimilate_storage(&mut t)
 		.unwrap();
 
-	let staking_config = staking::GenesisConfig::<Test> {
+	let staking_config = pallet_staking::GenesisConfig::<Test> {
 		stakers,
 		validator_count: 8,
-		force_era: staking::Forcing::ForceNew,
+		force_era: pallet_staking::Forcing::ForceNew,
 		minimum_validator_count: 0,
 		invulnerables: vec![],
 		..Default::default()
diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs
index 65f1597622c56..e69255ab1980b 100644
--- a/frame/identity/src/lib.rs
+++ b/frame/identity/src/lib.rs
@@ -1387,6 +1387,7 @@ mod tests {
 		pub const ExistentialDeposit: u64 = 1;
 	}
 	impl pallet_balances::Trait for Test {
+		type MaxLocks = ();
 		type Balance = u64;
 		type Event = ();
 		type DustRemoval = ();
diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs
index 97e7a954f8f58..a47e1251d63db 100644
--- a/frame/indices/src/mock.rs
+++ b/frame/indices/src/mock.rs
@@ -82,6 +82,7 @@ parameter_types! {
 }
 
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type DustRemoval = ();
 	type Event = MetaEvent;
diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs
index 888dcecb3a8fd..b727ec8cdb41d 100644
--- a/frame/multisig/src/tests.rs
+++ b/frame/multisig/src/tests.rs
@@ -90,6 +90,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type Event = TestEvent;
 	type DustRemoval = ();
diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs
index a1faedaf1cee6..ca90da1750b3c 100644
--- a/frame/nicks/src/lib.rs
+++ b/frame/nicks/src/lib.rs
@@ -293,6 +293,7 @@ mod tests {
 		pub const ExistentialDeposit: u64 = 1;
 	}
 	impl pallet_balances::Trait for Test {
+		type MaxLocks = ();
 		type Balance = u64;
 		type Event = ();
 		type DustRemoval = ();
diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs
index ad6e8a14d5622..12a14e90b0e53 100644
--- a/frame/offences/benchmarking/src/mock.rs
+++ b/frame/offences/benchmarking/src/mock.rs
@@ -72,6 +72,7 @@ parameter_types! {
 	pub const ExistentialDeposit: Balance = 10;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = Balance;
 	type Event = Event;
 	type DustRemoval = ();
diff --git a/frame/proxy/src/tests.rs b/frame/proxy/src/tests.rs
index 00d84e65ad1d6..ea9b321ee0a8e 100644
--- a/frame/proxy/src/tests.rs
+++ b/frame/proxy/src/tests.rs
@@ -92,6 +92,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type Event = TestEvent;
 	type DustRemoval = ();
diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs
index 6b8ef169c0076..9256ec9425de5 100644
--- a/frame/recovery/src/mock.rs
+++ b/frame/recovery/src/mock.rs
@@ -91,6 +91,7 @@ parameter_types! {
 }
 
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u128;
 	type DustRemoval = ();
 	type Event = TestEvent;
diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs
index 4581f49bbbcb6..2341832748f68 100644
--- a/frame/scored-pool/src/mock.rs
+++ b/frame/scored-pool/src/mock.rs
@@ -78,6 +78,7 @@ impl frame_system::Trait for Test {
 }
 
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type Event = ();
 	type DustRemoval = ();
diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs
index d4eac4247734f..c1f75ec4e0965 100644
--- a/frame/session/benchmarking/src/mock.rs
+++ b/frame/session/benchmarking/src/mock.rs
@@ -88,6 +88,7 @@ parameter_types! {
 	pub const ExistentialDeposit: Balance = 10;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = Balance;
 	type Event = ();
 	type DustRemoval = ();
diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs
index 1ca828bf37196..03fa9b60f74dd 100644
--- a/frame/society/src/mock.rs
+++ b/frame/society/src/mock.rs
@@ -89,6 +89,7 @@ impl frame_system::Trait for Test {
 }
 
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type Event = ();
 	type DustRemoval = ();
diff --git a/frame/staking/fuzzer/src/mock.rs b/frame/staking/fuzzer/src/mock.rs
index 1f5b29b56b6b6..766f088f40388 100644
--- a/frame/staking/fuzzer/src/mock.rs
+++ b/frame/staking/fuzzer/src/mock.rs
@@ -87,6 +87,7 @@ parameter_types! {
 	pub const ExistentialDeposit: Balance = 10;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = Balance;
 	type Event = ();
 	type DustRemoval = ();
diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs
index dcdacfbaacb04..31e41e21360a0 100644
--- a/frame/staking/src/mock.rs
+++ b/frame/staking/src/mock.rs
@@ -227,6 +227,7 @@ impl frame_system::Trait for Test {
 	type SystemWeightInfo = ();
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = Balance;
 	type Event = MetaEvent;
 	type DustRemoval = ();
diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs
index 6f50f38a23369..32983b414d304 100644
--- a/frame/support/src/traits.rs
+++ b/frame/support/src/traits.rs
@@ -1110,6 +1110,9 @@ pub trait LockableCurrency<AccountId>: Currency<AccountId> {
 	/// The quantity used to denote time; usually just a `BlockNumber`.
 	type Moment;
 
+	/// The maximum number of locks a user should have on their account.
+	type MaxLocks: Get<u32>;
+
 	/// Create a new balance lock on account `who`.
 	///
 	/// If the new lock is valid (i.e. not already expired), it will push the struct to
diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs
index 4e4bc5311dacd..bfd69ea29e3b9 100644
--- a/frame/transaction-payment/src/lib.rs
+++ b/frame/transaction-payment/src/lib.rs
@@ -671,6 +671,7 @@ mod tests {
 		type DustRemoval = ();
 		type ExistentialDeposit = ExistentialDeposit;
 		type AccountStore = System;
+		type MaxLocks = ();
 		type WeightInfo = ();
 	}
 	thread_local! {
diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs
index f9928c37b36a3..a4e1e3d8d77b2 100644
--- a/frame/treasury/src/tests.rs
+++ b/frame/treasury/src/tests.rs
@@ -90,6 +90,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type Event = Event;
 	type DustRemoval = ();
diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs
index 611c42907ca04..cf5b0dd7568e7 100644
--- a/frame/utility/src/tests.rs
+++ b/frame/utility/src/tests.rs
@@ -89,6 +89,7 @@ parameter_types! {
 	pub const ExistentialDeposit: u64 = 1;
 }
 impl pallet_balances::Trait for Test {
+	type MaxLocks = ();
 	type Balance = u64;
 	type DustRemoval = ();
 	type Event = TestEvent;
diff --git a/frame/vesting/src/benchmarking.rs b/frame/vesting/src/benchmarking.rs
index 974289aac3218..7c5478472f8ab 100644
--- a/frame/vesting/src/benchmarking.rs
+++ b/frame/vesting/src/benchmarking.rs
@@ -28,7 +28,6 @@ use sp_runtime::traits::Bounded;
 use crate::Module as Vesting;
 
 const SEED: u32 = 0;
-const MAX_LOCKS: u32 = 20;
 
 type BalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
 
@@ -62,7 +61,7 @@ benchmarks! {
 	_ { }
 
 	vest_locked {
-		let l in 0 .. MAX_LOCKS;
+		let l in 0 .. MaxLocksOf::<T>::get();
 
 		let caller = whitelisted_caller();
 		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
@@ -86,7 +85,7 @@ benchmarks! {
 	}
 
 	vest_unlocked {
-		let l in 0 .. MAX_LOCKS;
+		let l in 0 .. MaxLocksOf::<T>::get();
 
 		let caller = whitelisted_caller();
 		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
@@ -110,7 +109,7 @@ benchmarks! {
 	}
 
 	vest_other_locked {
-		let l in 0 .. MAX_LOCKS;
+		let l in 0 .. MaxLocksOf::<T>::get();
 
 		let other: T::AccountId = account("other", 0, SEED);
 		let other_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(other.clone());
@@ -137,7 +136,7 @@ benchmarks! {
 	}
 
 	vest_other_unlocked {
-		let l in 0 .. MAX_LOCKS;
+		let l in 0 .. MaxLocksOf::<T>::get();
 
 		let other: T::AccountId = account("other", 0, SEED);
 		let other_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(other.clone());
@@ -164,7 +163,7 @@ benchmarks! {
 	}
 
 	vested_transfer {
-		let l in 0 .. MAX_LOCKS;
+		let l in 0 .. MaxLocksOf::<T>::get();
 
 		let caller: T::AccountId = whitelisted_caller();
 		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
@@ -193,6 +192,38 @@ benchmarks! {
 			"Lock not created",
 		);
 	}
+
+	force_vested_transfer {
+		let l in 0 .. MaxLocksOf::<T>::get();
+
+		let source: T::AccountId = account("source", 0, SEED);
+		let source_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(source.clone());
+		T::Currency::make_free_balance_be(&source, BalanceOf::<T>::max_value());
+		let target: T::AccountId = account("target", 0, SEED);
+		let target_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(target.clone());
+		// Give target existing locks
+		add_locks::<T>(&target, l as u8);
+
+		let transfer_amount = T::MinVestedTransfer::get();
+
+		let vesting_schedule = VestingInfo {
+			locked: transfer_amount,
+			per_block: 10.into(),
+			starting_block: 1.into(),
+		};
+	}: _(RawOrigin::Root, source_lookup, target_lookup, vesting_schedule)
+	verify {
+		assert_eq!(
+			T::MinVestedTransfer::get(),
+			T::Currency::free_balance(&target),
+			"Transfer didn't happen",
+		);
+		assert_eq!(
+			Vesting::<T>::vesting_balance(&target),
+			Some(T::MinVestedTransfer::get()),
+			"Lock not created",
+		);
+	}
 }
 
 #[cfg(test)]
@@ -209,6 +240,7 @@ mod tests {
 			assert_ok!(test_benchmark_vest_other_locked::<Test>());
 			assert_ok!(test_benchmark_vest_other_unlocked::<Test>());
 			assert_ok!(test_benchmark_vested_transfer::<Test>());
+			assert_ok!(test_benchmark_force_vested_transfer::<Test>());
 		});
 	}
 }
diff --git a/frame/vesting/src/default_weights.rs b/frame/vesting/src/default_weights.rs
new file mode 100644
index 0000000000000..dac9224d69ab0
--- /dev/null
+++ b/frame/vesting/src/default_weights.rs
@@ -0,0 +1,62 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn vest_locked(l: u32, ) -> Weight {
+		(82109000 as Weight)
+			.saturating_add((332000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn vest_unlocked(l: u32, ) -> Weight {
+		(88419000 as Weight)
+			.saturating_add((3000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn vest_other_locked(l: u32, ) -> Weight {
+		(81277000 as Weight)
+			.saturating_add((321000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn vest_other_unlocked(l: u32, ) -> Weight {
+		(87584000 as Weight)
+			.saturating_add((19000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn vested_transfer(l: u32, ) -> Weight {
+		(185916000 as Weight)
+			.saturating_add((625000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn force_vested_transfer(l: u32, ) -> Weight {
+		(185916000 as Weight)
+			.saturating_add((625000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+}
diff --git a/frame/vesting/src/lib.rs b/frame/vesting/src/lib.rs
index 2fe8e033bb25e..223dc168645f4 100644
--- a/frame/vesting/src/lib.rs
+++ b/frame/vesting/src/lib.rs
@@ -61,8 +61,10 @@ use frame_support::traits::{
 use frame_system::{ensure_signed, ensure_root};
 
 mod benchmarking;
+mod default_weights;
 
 type BalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
+type MaxLocksOf<T> = <<T as Trait>::Currency as LockableCurrency<<T as frame_system::Trait>::AccountId>>::MaxLocks;
 
 pub trait WeightInfo {
 	fn vest_locked(l: u32, ) -> Weight;
@@ -70,14 +72,7 @@ pub trait WeightInfo {
 	fn vest_other_locked(l: u32, ) -> Weight;
 	fn vest_other_unlocked(l: u32, ) -> Weight;
 	fn vested_transfer(l: u32, ) -> Weight;
-}
-
-impl WeightInfo for () {
-	fn vest_locked(_l: u32, ) -> Weight { 1_000_000_000 }
-	fn vest_unlocked(_l: u32, ) -> Weight { 1_000_000_000 }
-	fn vest_other_locked(_l: u32, ) -> Weight { 1_000_000_000 }
-	fn vest_other_unlocked(_l: u32, ) -> Weight { 1_000_000_000 }
-	fn vested_transfer(_l: u32, ) -> Weight { 1_000_000_000 }
+	fn force_vested_transfer(l: u32, ) -> Weight;
 }
 
 pub trait Trait: frame_system::Trait {
@@ -171,7 +166,7 @@ decl_storage! {
 decl_event!(
 	pub enum Event<T> where AccountId = <T as frame_system::Trait>::AccountId, Balance = BalanceOf<T> {
 		/// The amount vested has been updated. This could indicate more funds are available. The
-		/// balance given is the amount which is left unvested (and thus locked). 
+		/// balance given is the amount which is left unvested (and thus locked).
 		/// \[account, unvested\]
 		VestingUpdated(AccountId, Balance),
 		/// An \[account\] has become fully vested. No further vesting can happen.
@@ -213,12 +208,10 @@ decl_module! {
 		/// - DbWeight: 2 Reads, 2 Writes
 		///     - Reads: Vesting Storage, Balances Locks, [Sender Account]
 		///     - Writes: Vesting Storage, Balances Locks, [Sender Account]
-		/// - Benchmark:
-		///     - Unlocked: 48.76 + .048 * l µs (min square analysis)
-		///     - Locked: 44.43 + .284 * l µs (min square analysis)
-		/// - Using 50 µs fixed. Assuming less than 50 locks on any user, else we may want factor in number of locks.
 		/// # </weight>
-		#[weight = 50_000_000 + T::DbWeight::get().reads_writes(2, 2)]
+		#[weight = T::WeightInfo::vest_locked(MaxLocksOf::<T>::get())
+			.max(T::WeightInfo::vest_unlocked(MaxLocksOf::<T>::get()))
+		]
 		fn vest(origin) -> DispatchResult {
 			let who = ensure_signed(origin)?;
 			Self::update_lock(who)
@@ -238,12 +231,10 @@ decl_module! {
 		/// - DbWeight: 3 Reads, 3 Writes
 		///     - Reads: Vesting Storage, Balances Locks, Target Account
 		///     - Writes: Vesting Storage, Balances Locks, Target Account
-		/// - Benchmark:
-		///     - Unlocked: 44.3 + .294 * l µs (min square analysis)
-		///     - Locked: 48.16 + .103 * l µs (min square analysis)
-		/// - Using 50 µs fixed. Assuming less than 50 locks on any user, else we may want factor in number of locks.
 		/// # </weight>
-		#[weight = 50_000_000 + T::DbWeight::get().reads_writes(3, 3)]
+		#[weight = T::WeightInfo::vest_other_locked(MaxLocksOf::<T>::get())
+			.max(T::WeightInfo::vest_other_unlocked(MaxLocksOf::<T>::get()))
+		]
 		fn vest_other(origin, target: <T::Lookup as StaticLookup>::Source) -> DispatchResult {
 			ensure_signed(origin)?;
 			Self::update_lock(T::Lookup::lookup(target)?)
@@ -264,10 +255,8 @@ decl_module! {
 		/// - DbWeight: 3 Reads, 3 Writes
 		///     - Reads: Vesting Storage, Balances Locks, Target Account, [Sender Account]
 		///     - Writes: Vesting Storage, Balances Locks, Target Account, [Sender Account]
-		/// - Benchmark: 100.3 + .365 * l µs (min square analysis)
-		/// - Using 100 µs fixed. Assuming less than 50 locks on any user, else we may want factor in number of locks.
 		/// # </weight>
-		#[weight = 100_000_000 + T::DbWeight::get().reads_writes(3, 3)]
+		#[weight = T::WeightInfo::vested_transfer(MaxLocksOf::<T>::get())]
 		pub fn vested_transfer(
 			origin,
 			target: <T::Lookup as StaticLookup>::Source,
@@ -303,10 +292,8 @@ decl_module! {
 		/// - DbWeight: 4 Reads, 4 Writes
 		///     - Reads: Vesting Storage, Balances Locks, Target Account, Source Account
 		///     - Writes: Vesting Storage, Balances Locks, Target Account, Source Account
-		/// - Benchmark: 100.3 + .365 * l µs (min square analysis)
-		/// - Using 100 µs fixed. Assuming less than 50 locks on any user, else we may want factor in number of locks.
 		/// # </weight>
-		#[weight = 100_000_000 + T::DbWeight::get().reads_writes(4, 4)]
+		#[weight = T::WeightInfo::force_vested_transfer(MaxLocksOf::<T>::get())]
 		pub fn force_vested_transfer(
 			origin,
 			source: <T::Lookup as StaticLookup>::Source,
@@ -463,12 +450,16 @@ mod tests {
 		type OnKilledAccount = ();
 		type SystemWeightInfo = ();
 	}
+	parameter_types! {
+		pub const MaxLocks: u32 = 10;
+	}
 	impl pallet_balances::Trait for Test {
 		type Balance = u64;
 		type DustRemoval = ();
 		type Event = ();
 		type ExistentialDeposit = ExistentialDeposit;
 		type AccountStore = System;
+		type MaxLocks = MaxLocks;
 		type WeightInfo = ();
 	}
 	parameter_types! {

From 59b4668ec76a18c2101c4ba66e5f5adb361eaca5 Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Thu, 17 Sep 2020 10:09:59 +0200
Subject: [PATCH 095/122] Add benchmarking pipeline to node-template (#7122)

---
 Cargo.lock                            |  5 ++++
 bin/node-template/node/Cargo.toml     | 10 ++++++++
 bin/node-template/node/src/cli.rs     |  4 +++
 bin/node-template/node/src/command.rs | 25 +++++++++++++------
 bin/node-template/node/src/service.rs |  1 +
 bin/node-template/runtime/Cargo.toml  | 19 +++++++++++++--
 bin/node-template/runtime/src/lib.rs  | 35 +++++++++++++++++++++++++++
 bin/node/cli/src/command.rs           |  5 ++--
 8 files changed, 91 insertions(+), 13 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d8602d2c6f3f0..1049d23f93763 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3912,6 +3912,8 @@ dependencies = [
 name = "node-template"
 version = "2.0.0-rc6"
 dependencies = [
+ "frame-benchmarking",
+ "frame-benchmarking-cli",
  "jsonrpc-core",
  "node-template-runtime",
  "pallet-transaction-payment-rpc",
@@ -3945,10 +3947,13 @@ dependencies = [
 name = "node-template-runtime"
 version = "2.0.0-rc6"
 dependencies = [
+ "frame-benchmarking",
  "frame-executive",
  "frame-support",
  "frame-system",
+ "frame-system-benchmarking",
  "frame-system-rpc-runtime-api",
+ "hex-literal",
  "pallet-aura",
  "pallet-balances",
  "pallet-grandpa",
diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml
index d8cc9478bbda8..eb3b63b926f00 100644
--- a/bin/node-template/node/Cargo.toml
+++ b/bin/node-template/node/Cargo.toml
@@ -45,7 +45,17 @@ sc-basic-authorship = { version = "0.8.0-rc6", path = "../../../client/basic-aut
 substrate-frame-rpc-system = { version = "2.0.0-rc6", path = "../../../utils/frame/rpc/system" }
 pallet-transaction-payment-rpc = { version = "2.0.0-rc6", path = "../../../frame/transaction-payment/rpc/" }
 
+# These dependencies are used for runtime benchmarking
+frame-benchmarking = { version = "2.0.0-rc6", path = "../../../frame/benchmarking" }
+frame-benchmarking-cli = { version = "2.0.0-rc6", path = "../../../utils/frame/benchmarking-cli" }
+
 node-template-runtime = { version = "2.0.0-rc6", path = "../runtime" }
 
 [build-dependencies]
 substrate-build-script-utils = { version = "2.0.0-rc6", path = "../../../utils/build-script-utils" }
+
+[features]
+default = []
+runtime-benchmarks = [
+	"node-template-runtime/runtime-benchmarks",
+]
diff --git a/bin/node-template/node/src/cli.rs b/bin/node-template/node/src/cli.rs
index f3667fa79d19e..f2faf17e4ddf4 100644
--- a/bin/node-template/node/src/cli.rs
+++ b/bin/node-template/node/src/cli.rs
@@ -32,4 +32,8 @@ pub enum Subcommand {
 
 	/// Revert the chain to a previous state.
 	Revert(sc_cli::RevertCmd),
+
+	/// The custom benchmark subcommmand benchmarking runtime pallets.
+	#[structopt(name = "benchmark", about = "Benchmark runtime pallets.")]
+	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 }
diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs
index 98c56e948300e..2efca03837108 100644
--- a/bin/node-template/node/src/command.rs
+++ b/bin/node-template/node/src/command.rs
@@ -15,12 +15,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::chain_spec;
+use crate::{chain_spec, service};
 use crate::cli::{Cli, Subcommand};
-use crate::service;
 use sc_cli::{SubstrateCli, RuntimeVersion, Role, ChainSpec};
 use sc_service::PartialComponents;
-use crate::service::new_partial;
+use node_template_runtime::Block;
 
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
@@ -75,7 +74,7 @@ pub fn run() -> sc_cli::Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {
 				let PartialComponents { client, task_manager, import_queue, ..}
-					= new_partial(&config)?;
+					= service::new_partial(&config)?;
 				Ok((cmd.run(client, import_queue), task_manager))
 			})
 		},
@@ -83,7 +82,7 @@ pub fn run() -> sc_cli::Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {
 				let PartialComponents { client, task_manager, ..}
-					= new_partial(&config)?;
+					= service::new_partial(&config)?;
 				Ok((cmd.run(client, config.database), task_manager))
 			})
 		},
@@ -91,7 +90,7 @@ pub fn run() -> sc_cli::Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {
 				let PartialComponents { client, task_manager, ..}
-					= new_partial(&config)?;
+					= service::new_partial(&config)?;
 				Ok((cmd.run(client, config.chain_spec), task_manager))
 			})
 		},
@@ -99,7 +98,7 @@ pub fn run() -> sc_cli::Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {
 				let PartialComponents { client, task_manager, import_queue, ..}
-					= new_partial(&config)?;
+					= service::new_partial(&config)?;
 				Ok((cmd.run(client, import_queue), task_manager))
 			})
 		},
@@ -111,10 +110,20 @@ pub fn run() -> sc_cli::Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {
 				let PartialComponents { client, task_manager, backend, ..}
-					= new_partial(&config)?;
+					= service::new_partial(&config)?;
 				Ok((cmd.run(client, backend), task_manager))
 			})
 		},
+		Some(Subcommand::Benchmark(cmd)) => {
+			if cfg!(feature = "runtime-benchmarks") {
+				let runner = cli.create_runner(cmd)?;
+
+				runner.sync_run(|config| cmd.run::<Block, service::Executor>(config))
+			} else {
+				Err("Benchmarking wasn't enabled when building the node. \
+				You can enable it with `--features runtime-benchmarks`.".into())
+			}
+		},
 		None => {
 			let runner = cli.create_runner(&cli.run)?;
 			runner.run_node_until_exit(|config| match config.role {
diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs
index 8fa935c375021..3de31dc61ab51 100644
--- a/bin/node-template/node/src/service.rs
+++ b/bin/node-template/node/src/service.rs
@@ -16,6 +16,7 @@ native_executor_instance!(
 	pub Executor,
 	node_template_runtime::api::dispatch,
 	node_template_runtime::native_version,
+	frame_benchmarking::benchmarking::HostFunctions,
 );
 
 type FullClient = sc_service::TFullClient<Block, RuntimeApi, Executor>;
diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml
index 3cb0754089d91..09b46f4a56fa9 100644
--- a/bin/node-template/runtime/Cargo.toml
+++ b/bin/node-template/runtime/Cargo.toml
@@ -40,6 +40,11 @@ sp-version = { version = "2.0.0-rc6", default-features = false, path = "../../..
 frame-system-rpc-runtime-api = { version = "2.0.0-rc6", default-features = false, path = "../../../frame/system/rpc/runtime-api/" }
 pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-rc6", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" }
 
+# Used for runtime benchmarking
+frame-benchmarking = { version = "2.0.0-rc6", default-features = false, path = "../../../frame/benchmarking", optional = true }
+frame-system-benchmarking = { version = "2.0.0-rc6", default-features = false, path = "../../../frame/system/benchmarking", optional = true }
+hex-literal = { version = "0.3.1", optional = true }
+
 template = { version = "2.0.0-rc6", default-features = false, path = "../pallets/template", package = "pallet-template" }
 
 [build-dependencies]
@@ -58,7 +63,7 @@ std = [
 	"pallet-sudo/std",
 	"pallet-timestamp/std",
 	"pallet-transaction-payment/std",
-  "pallet-transaction-payment-rpc-runtime-api/std",
+	"pallet-transaction-payment-rpc-runtime-api/std",
 	"serde",
 	"sp-api/std",
 	"sp-block-builder/std",
@@ -72,6 +77,16 @@ std = [
 	"sp-transaction-pool/std",
 	"sp-version/std",
 	"frame-system/std",
-  "frame-system-rpc-runtime-api/std",
+	"frame-system-rpc-runtime-api/std",
 	"template/std",
 ]
+runtime-benchmarks = [
+	"sp-runtime/runtime-benchmarks",
+	"frame-benchmarking",
+	"frame-support/runtime-benchmarks",
+	"frame-system-benchmarking",
+	"hex-literal",
+	"frame-system/runtime-benchmarks",
+	"pallet-balances/runtime-benchmarks",
+	"pallet-timestamp/runtime-benchmarks",
+]
diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs
index 91fffb7f8e155..9612394cc7fbf 100644
--- a/bin/node-template/runtime/src/lib.rs
+++ b/bin/node-template/runtime/src/lib.rs
@@ -440,4 +440,39 @@ impl_runtime_apis! {
 			TransactionPayment::query_info(uxt, len)
 		}
 	}
+
+	#[cfg(feature = "runtime-benchmarks")]
+	impl frame_benchmarking::Benchmark<Block> for Runtime {
+		fn dispatch_benchmark(
+			config: frame_benchmarking::BenchmarkConfig
+		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
+			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey};
+
+			use frame_system_benchmarking::Module as SystemBench;
+			impl frame_system_benchmarking::Trait for Runtime {}
+
+			let whitelist: Vec<TrackedStorageKey> = vec![
+				// Block Number
+				hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec().into(),
+				// Total Issuance
+				hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec().into(),
+				// Execution Phase
+				hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a").to_vec().into(),
+				// Event Count
+				hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850").to_vec().into(),
+				// System Events
+				hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef780d41e5e16056765bc8461851072c9d7").to_vec().into(),
+			];
+
+			let mut batches = Vec::<BenchmarkBatch>::new();
+			let params = (&config, &whitelist);
+
+			add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
+			add_benchmark!(params, batches, pallet_balances, Balances);
+			add_benchmark!(params, batches, pallet_timestamp, Timestamp);
+
+			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
+			Ok(batches)
+		}
+	}
 }
diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs
index 4772d6e4be605..7b84ff5a0583b 100644
--- a/bin/node/cli/src/command.rs
+++ b/bin/node/cli/src/command.rs
@@ -88,9 +88,8 @@ pub fn run() -> Result<()> {
 
 				runner.sync_run(|config| cmd.run::<Block, Executor>(config))
 			} else {
-				println!("Benchmarking wasn't enabled when building the node. \
-				You can enable it with `--features runtime-benchmarks`.");
-				Ok(())
+				Err("Benchmarking wasn't enabled when building the node. \
+				You can enable it with `--features runtime-benchmarks`.".into())
 			}
 		}
 		Some(Subcommand::Key(cmd)) => cmd.run(),

From 2c650364c95132db2a3e3b1833ad8ef31dfeabce Mon Sep 17 00:00:00 2001
From: Wei Tang <wei@that.world>
Date: Thu, 17 Sep 2020 11:04:43 +0200
Subject: [PATCH 096/122] Use tracing-based subscriber logging (#6825)

* init_logger: switch from log-based to tracing-based and add compatibility layer

* Move tracing profiling subscriber related config realization

* sp-tracing: change profiling to be a layer instead of a subscriber

* Enable profiling layer in cli

* Change all test env_logger init to sp_tracing::try_init_simple

* Remove all local env_logger dependency

* Add missing tracing-subscriber dependency

* frame-sudo: fix tests

* frame-support: fix tests

* Fix frame/pallet and executor tests

* Fix the remaining tests

* Use subscriber's try_init as recommended by @davidbarsky

* Be explict that the tracing-log feature is needed

* Set subscriber writer to stderr

* Shorter line width

* Update cargo lock tracing version

* Fix sc_tracing crate compile

* Fix sc_authority_discovery crate test

* unremove default-features

* Leave enabled to default true

* Warn if global default cannot be set

* Fix unused import

* Remove unused PROXY_TARGET

* Change all reference from rc5 to rc6

* Change all reference of rc2 to rc6

* Fix styling

* Fix typo

* make logger init error'ing

* re-fixing the test issue

Co-authored-by: Benjamin Kampmann <ben@parity.io>
---
 Cargo.lock                                    |  33 +++--
 bin/node/bench/Cargo.toml                     |   1 +
 bin/node/bench/src/main.rs                    |   2 +-
 bin/node/rpc-client/Cargo.toml                |   2 +-
 bin/node/rpc-client/src/main.rs               |   2 +-
 client/authority-discovery/Cargo.toml         |   2 +-
 .../authority-discovery/src/worker/tests.rs   |   4 +-
 client/cli/Cargo.toml                         |   4 +-
 client/cli/src/config.rs                      |   8 +-
 client/cli/src/lib.rs                         | 131 +++++++++---------
 client/consensus/aura/Cargo.toml              |   2 +-
 client/consensus/aura/src/lib.rs              |   2 +-
 client/consensus/babe/Cargo.toml              |   2 +-
 client/consensus/babe/src/tests.rs            |  14 +-
 client/consensus/manual-seal/Cargo.toml       |  29 ++--
 client/db/Cargo.toml                          |   2 +-
 client/db/src/lib.rs                          |   2 +-
 client/db/src/storage_cache.rs                |   4 +-
 client/executor/Cargo.toml                    |   1 +
 client/executor/src/integration_tests/mod.rs  |   8 +-
 client/finality-grandpa/Cargo.toml            |   2 +-
 .../src/communication/tests.rs                |   2 +-
 client/finality-grandpa/src/tests.rs          |  18 +--
 client/informant/Cargo.toml                   |   4 +-
 client/light/Cargo.toml                       |  16 +--
 client/network/Cargo.toml                     |   2 +-
 client/network/src/light_client_handler.rs    |   2 +-
 .../src/protocol/sync/extra_requests.rs       |   2 +-
 client/network/test/Cargo.toml                |   2 +-
 client/network/test/src/sync.rs               |  52 +++----
 client/offchain/Cargo.toml                    |   2 +-
 client/offchain/src/api.rs                    |   2 +-
 client/offchain/src/lib.rs                    |   2 +-
 client/service/Cargo.toml                     |   2 +-
 client/service/src/builder.rs                 |  13 +-
 client/service/test/Cargo.toml                |   2 +-
 client/service/test/src/client/mod.rs         |   8 +-
 client/service/test/src/lib.rs                |   2 +-
 client/state-db/Cargo.toml                    |   3 -
 client/tracing/Cargo.toml                     |   3 +-
 client/tracing/src/lib.rs                     |  61 ++++----
 frame/elections-phragmen/src/lib.rs           |  76 +++++-----
 frame/elections/src/tests.rs                  |  14 +-
 frame/example/src/lib.rs                      |   2 +-
 frame/scored-pool/src/tests.rs                |   2 +-
 frame/staking/Cargo.toml                      |   2 +-
 frame/staking/src/mock.rs                     |   2 +-
 frame/staking/src/tests.rs                    |   4 +-
 frame/sudo/src/tests.rs                       |  30 ++--
 frame/support/src/storage/mod.rs              |   6 +-
 frame/support/test/tests/decl_storage.rs      |  10 +-
 primitives/consensus/slots/Cargo.toml         |   2 +-
 primitives/tracing/Cargo.toml                 |   3 +-
 primitives/tracing/src/lib.rs                 |   9 +-
 utils/frame/rpc/system/Cargo.toml             |   2 +-
 utils/frame/rpc/system/src/lib.rs             |   8 +-
 56 files changed, 309 insertions(+), 320 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1049d23f93763..e80ba143d3de8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3638,6 +3638,7 @@ dependencies = [
  "sp-runtime",
  "sp-state-machine",
  "sp-timestamp",
+ "sp-tracing",
  "sp-transaction-pool",
  "sp-trie",
  "structopt",
@@ -3830,13 +3831,13 @@ dependencies = [
 name = "node-rpc-client"
 version = "2.0.0-rc6"
 dependencies = [
- "env_logger",
  "futures 0.1.29",
  "hyper 0.12.35",
  "jsonrpc-core-client",
  "log",
  "node-primitives",
  "sc-rpc",
+ "sp-tracing",
 ]
 
 [[package]]
@@ -4876,7 +4877,6 @@ dependencies = [
 name = "pallet-staking"
 version = "2.0.0-rc6"
 dependencies = [
- "env_logger",
  "frame-benchmarking",
  "frame-support",
  "frame-system",
@@ -4898,6 +4898,7 @@ dependencies = [
  "sp-staking",
  "sp-std",
  "sp-storage",
+ "sp-tracing",
  "static_assertions",
  "substrate-test-utils",
 ]
@@ -6242,7 +6243,6 @@ dependencies = [
  "bytes 0.5.6",
  "derive_more",
  "either",
- "env_logger",
  "futures 0.3.5",
  "futures-timer 3.0.2",
  "libp2p",
@@ -6262,6 +6262,7 @@ dependencies = [
  "sp-blockchain",
  "sp-core",
  "sp-runtime",
+ "sp-tracing",
  "substrate-prometheus-endpoint",
  "substrate-test-runtime-client",
 ]
@@ -6345,7 +6346,6 @@ dependencies = [
  "bip39",
  "chrono",
  "derive_more",
- "env_logger",
  "fdlimit",
  "futures 0.3.5",
  "hex",
@@ -6383,6 +6383,9 @@ dependencies = [
  "tempfile",
  "time",
  "tokio 0.2.22",
+ "tracing",
+ "tracing-log",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -6428,7 +6431,6 @@ name = "sc-client-db"
 version = "0.8.0-rc6"
 dependencies = [
  "blake2-rfc",
- "env_logger",
  "hash-db",
  "kvdb",
  "kvdb-memorydb",
@@ -6451,6 +6453,7 @@ dependencies = [
  "sp-keyring",
  "sp-runtime",
  "sp-state-machine",
+ "sp-tracing",
  "sp-trie",
  "substrate-prometheus-endpoint",
  "substrate-test-runtime-client",
@@ -6472,7 +6475,6 @@ name = "sc-consensus-aura"
 version = "0.8.0-rc6"
 dependencies = [
  "derive_more",
- "env_logger",
  "futures 0.3.5",
  "futures-timer 3.0.2",
  "log",
@@ -6499,6 +6501,7 @@ dependencies = [
  "sp-keyring",
  "sp-runtime",
  "sp-timestamp",
+ "sp-tracing",
  "sp-version",
  "substrate-prometheus-endpoint",
  "substrate-test-runtime-client",
@@ -6510,7 +6513,6 @@ name = "sc-consensus-babe"
 version = "0.8.0-rc6"
 dependencies = [
  "derive_more",
- "env_logger",
  "fork-tree",
  "futures 0.3.5",
  "futures-timer 3.0.2",
@@ -6551,6 +6553,7 @@ dependencies = [
  "sp-keyring",
  "sp-runtime",
  "sp-timestamp",
+ "sp-tracing",
  "sp-utils",
  "sp-version",
  "substrate-prometheus-endpoint",
@@ -6604,7 +6607,6 @@ version = "0.8.0-rc6"
 dependencies = [
  "assert_matches",
  "derive_more",
- "env_logger",
  "futures 0.3.5",
  "jsonrpc-core",
  "jsonrpc-core-client",
@@ -6725,6 +6727,7 @@ dependencies = [
  "substrate-test-runtime",
  "test-case",
  "tracing",
+ "tracing-subscriber",
  "wasmi",
  "wat",
 ]
@@ -6783,7 +6786,6 @@ version = "0.8.0-rc6"
 dependencies = [
  "assert_matches",
  "derive_more",
- "env_logger",
  "finality-grandpa",
  "fork-tree",
  "futures 0.3.5",
@@ -6815,6 +6817,7 @@ dependencies = [
  "sp-keyring",
  "sp-runtime",
  "sp-state-machine",
+ "sp-tracing",
  "sp-utils",
  "substrate-prometheus-endpoint",
  "substrate-test-runtime-client",
@@ -6914,7 +6917,6 @@ dependencies = [
  "bytes 0.5.6",
  "derive_more",
  "either",
- "env_logger",
  "erased-serde",
  "fnv",
  "fork-tree",
@@ -6951,6 +6953,7 @@ dependencies = [
  "sp-keyring",
  "sp-runtime",
  "sp-test-primitives",
+ "sp-tracing",
  "sp-utils",
  "substrate-prometheus-endpoint",
  "substrate-test-runtime",
@@ -6985,7 +6988,6 @@ dependencies = [
 name = "sc-network-test"
 version = "0.8.0-rc6"
 dependencies = [
- "env_logger",
  "futures 0.3.5",
  "futures-timer 3.0.2",
  "libp2p",
@@ -7002,6 +7004,7 @@ dependencies = [
  "sp-consensus-babe",
  "sp-core",
  "sp-runtime",
+ "sp-tracing",
  "substrate-test-runtime",
  "substrate-test-runtime-client",
  "tempfile",
@@ -7012,7 +7015,6 @@ name = "sc-offchain"
 version = "2.0.0-rc6"
 dependencies = [
  "bytes 0.5.6",
- "env_logger",
  "fnv",
  "futures 0.3.5",
  "futures-timer 3.0.2",
@@ -7033,6 +7035,7 @@ dependencies = [
  "sp-core",
  "sp-offchain",
  "sp-runtime",
+ "sp-tracing",
  "sp-transaction-pool",
  "sp-utils",
  "substrate-test-runtime-client",
@@ -7221,7 +7224,6 @@ dependencies = [
 name = "sc-service-test"
 version = "2.0.0-rc6"
 dependencies = [
- "env_logger",
  "fdlimit",
  "futures 0.1.29",
  "futures 0.3.5",
@@ -7245,6 +7247,7 @@ dependencies = [
  "sp-runtime",
  "sp-state-machine",
  "sp-storage",
+ "sp-tracing",
  "sp-transaction-pool",
  "sp-trie",
  "substrate-test-runtime",
@@ -7257,7 +7260,6 @@ dependencies = [
 name = "sc-state-db"
 version = "0.8.0-rc6"
 dependencies = [
- "env_logger",
  "log",
  "parity-scale-codec",
  "parity-util-mem",
@@ -8432,6 +8434,7 @@ dependencies = [
  "log",
  "rental",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -8692,7 +8695,6 @@ dependencies = [
 name = "substrate-frame-rpc-system"
 version = "2.0.0-rc6"
 dependencies = [
- "env_logger",
  "frame-system-rpc-runtime-api",
  "futures 0.3.5",
  "jsonrpc-core",
@@ -8709,6 +8711,7 @@ dependencies = [
  "sp-blockchain",
  "sp-core",
  "sp-runtime",
+ "sp-tracing",
  "sp-transaction-pool",
  "substrate-test-runtime-client",
 ]
diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml
index 1914f460be0f1..ec797e32de3f2 100644
--- a/bin/node/bench/Cargo.toml
+++ b/bin/node/bench/Cargo.toml
@@ -31,6 +31,7 @@ sc-basic-authorship = { version = "0.8.0-rc6", path = "../../../client/basic-aut
 sp-inherents = { version = "2.0.0-rc6", path = "../../../primitives/inherents" }
 sp-finality-tracker = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/finality-tracker" }
 sp-timestamp = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/timestamp" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../../primitives/tracing" }
 hash-db = "0.15.2"
 tempfile = "3.1.0"
 fs_extra = "1"
diff --git a/bin/node/bench/src/main.rs b/bin/node/bench/src/main.rs
index 96ef1d920c1f5..46b659dd88387 100644
--- a/bin/node/bench/src/main.rs
+++ b/bin/node/bench/src/main.rs
@@ -79,7 +79,7 @@ fn main() {
 	let opt = Opt::from_args();
 
 	if !opt.json {
-		sc_cli::init_logger("");
+		sp_tracing::try_init_simple();
 	}
 
 	let mut import_benchmarks = Vec::new();
diff --git a/bin/node/rpc-client/Cargo.toml b/bin/node/rpc-client/Cargo.toml
index 698aa8f08aea5..92e1e1d3af15c 100644
--- a/bin/node/rpc-client/Cargo.toml
+++ b/bin/node/rpc-client/Cargo.toml
@@ -11,10 +11,10 @@ repository = "https://github.com/paritytech/substrate/"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-env_logger = "0.7.0"
 futures = "0.1.29"
 hyper = "0.12.35"
 jsonrpc-core-client = { version = "14.2.0", default-features = false, features = ["http"] }
 log = "0.4.8"
 node-primitives = { version = "2.0.0-rc6", path = "../primitives" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../../primitives/tracing" }
 sc-rpc = { version = "2.0.0-rc6", path = "../../../client/rpc" }
diff --git a/bin/node/rpc-client/src/main.rs b/bin/node/rpc-client/src/main.rs
index eadd1c8d47247..31f1efa28ccd0 100644
--- a/bin/node/rpc-client/src/main.rs
+++ b/bin/node/rpc-client/src/main.rs
@@ -35,7 +35,7 @@ use jsonrpc_core_client::{
 };
 
 fn main() {
-	env_logger::init();
+	sp_tracing::try_init_simple();
 
 	rt::run(rt::lazy(|| {
 		let uri = "http://localhost:9933";
diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml
index e2be0f68e2318..651550fffb6d2 100644
--- a/client/authority-discovery/Cargo.toml
+++ b/client/authority-discovery/Cargo.toml
@@ -38,7 +38,7 @@ sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
 sp-api = { version = "2.0.0-rc6", path = "../../primitives/api" }
 
 [dev-dependencies]
-env_logger = "0.7.0"
 quickcheck = "0.9.0"
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 sc-peerset = { version = "2.0.0-rc6", path = "../peerset" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../test-utils/runtime/client"}
diff --git a/client/authority-discovery/src/worker/tests.rs b/client/authority-discovery/src/worker/tests.rs
index 28192283054d1..f7b7dc41fee40 100644
--- a/client/authority-discovery/src/worker/tests.rs
+++ b/client/authority-discovery/src/worker/tests.rs
@@ -283,7 +283,7 @@ fn new_registers_metrics() {
 
 #[test]
 fn triggers_dht_get_query() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let (_dht_event_tx, dht_event_rx) = channel(1000);
 
 	// Generate authority keys
@@ -321,7 +321,7 @@ fn triggers_dht_get_query() {
 
 #[test]
 fn publish_discover_cycle() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 
 	// Node A publishing its address.
 
diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml
index 6bee1afc5a909..933e18180a6f5 100644
--- a/client/cli/Cargo.toml
+++ b/client/cli/Cargo.toml
@@ -13,7 +13,6 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 derive_more = "0.99.2"
-env_logger = "0.7.0"
 log = "0.4.8"
 atty = "0.2.13"
 regex = "1.3.4"
@@ -50,6 +49,9 @@ sc-tracing = { version = "2.0.0-rc6", path = "../tracing" }
 chrono = "0.4.10"
 parity-util-mem = { version = "0.7.0", default-features = false, features = ["primitive-types"] }
 serde = "1.0.111"
+tracing = "0.1.10"
+tracing-log = "0.1.1"
+tracing-subscriber = "0.2.10"
 
 [target.'cfg(not(target_os = "unknown"))'.dependencies]
 rpassword = "4.0.1"
diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs
index 6acb786cc1c23..43b755100244f 100644
--- a/client/cli/src/config.rs
+++ b/client/cli/src/config.rs
@@ -528,7 +528,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		Ok(self.shared_params().log_filters().join(","))
 	}
 
-	/// Initialize substrate. This must be done only once.
+	/// Initialize substrate. This must be done only once per process.
 	///
 	/// This method:
 	///
@@ -537,10 +537,14 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 	/// 3. Raises the FD limit
 	fn init<C: SubstrateCli>(&self) -> Result<()> {
 		let logger_pattern = self.log_filters()?;
+		let tracing_receiver = self.tracing_receiver()?;
+		let tracing_targets = self.tracing_targets()?;
 
 		sp_panic_handler::set(&C::support_url(), &C::impl_version());
 
-		init_logger(&logger_pattern);
+		if let Err(e) = init_logger(&logger_pattern, tracing_receiver, tracing_targets) {
+			log::warn!("💬 Problem initializing global logging framework: {:}", e)
+		}
 
 		if let Some(new_limit) = fdlimit::raise_fd_limit() {
 			if new_limit < RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT {
diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs
index 1de74f087f8ee..f16d02cab51d5 100644
--- a/client/cli/src/lib.rs
+++ b/client/cli/src/lib.rs
@@ -32,10 +32,7 @@ pub use arg_enums::*;
 pub use commands::*;
 pub use config::*;
 pub use error::*;
-use lazy_static::lazy_static;
-use log::info;
 pub use params::*;
-use regex::Regex;
 pub use runner::*;
 use sc_service::{Configuration, TaskExecutor};
 pub use sc_service::{ChainSpec, Role};
@@ -46,6 +43,7 @@ use structopt::{
 	clap::{self, AppSettings},
 	StructOpt,
 };
+use tracing_subscriber::layer::SubscriberExt;
 
 /// Substrate client CLI
 ///
@@ -228,79 +226,76 @@ pub trait SubstrateCli: Sized {
 	fn native_runtime_version(chain_spec: &Box<dyn ChainSpec>) -> &'static RuntimeVersion;
 }
 
-/// Initialize the logger
-pub fn init_logger(pattern: &str) {
-	use ansi_term::Colour;
-
-	let mut builder = env_logger::Builder::new();
-	// Disable info logging by default for some modules:
-	builder.filter(Some("ws"), log::LevelFilter::Off);
-	builder.filter(Some("yamux"), log::LevelFilter::Off);
-	builder.filter(Some("cranelift_codegen"), log::LevelFilter::Off);
-	builder.filter(Some("hyper"), log::LevelFilter::Warn);
-	builder.filter(Some("cranelift_wasm"), log::LevelFilter::Warn);
-	// Always log the special target `sc_tracing`, overrides global level
-	builder.filter(Some("sc_tracing"), log::LevelFilter::Trace);
-	// Enable info for others.
-	builder.filter(None, log::LevelFilter::Info);
+/// Initialize the global logger
+///
+/// This sets various global logging and tracing instances and thus may only be called once.
+pub fn init_logger(
+	pattern: &str,
+	tracing_receiver: sc_tracing::TracingReceiver,
+	tracing_targets: Option<String>,
+) -> std::result::Result<(), String> {
+	if let Err(e) = tracing_log::LogTracer::init() {
+		return Err(format!(
+			"Registering Substrate logger failed: {:}!", e
+		))
+	}
+
+	let mut env_filter = tracing_subscriber::EnvFilter::default()
+		// Disable info logging by default for some modules.
+		.add_directive("ws=off".parse().expect("provided directive is valid"))
+		.add_directive("yamux=off".parse().expect("provided directive is valid"))
+		.add_directive("cranelift_codegen=off".parse().expect("provided directive is valid"))
+		// Set warn logging by default for some modules.
+		.add_directive("cranelife_wasm=warn".parse().expect("provided directive is valid"))
+		.add_directive("hyper=warn".parse().expect("provided directive is valid"))
+		// Always log the special target `sc_tracing`, overrides global level.
+		.add_directive("sc_tracing=trace".parse().expect("provided directive is valid"))
+		// Enable info for others.
+		.add_directive(tracing_subscriber::filter::LevelFilter::INFO.into());
 
 	if let Ok(lvl) = std::env::var("RUST_LOG") {
-		builder.parse_filters(&lvl);
+		if lvl != "" {
+			// We're not sure if log or tracing is available at this moment, so silently ignore the
+			// parse error.
+			if let Ok(directive) = lvl.parse() {
+				env_filter = env_filter.add_directive(directive);
+			}
+		}
+	}
+
+	if pattern != "" {
+		// We're not sure if log or tracing is available at this moment, so silently ignore the
+		// parse error.
+		if let Ok(directive) = pattern.parse() {
+			env_filter = env_filter.add_directive(directive);
+		}
 	}
 
-	builder.parse_filters(pattern);
 	let isatty = atty::is(atty::Stream::Stderr);
 	let enable_color = isatty;
 
-	builder.format(move |buf, record| {
-		let now = time::now();
-		let timestamp =
-			time::strftime("%Y-%m-%d %H:%M:%S", &now).expect("Error formatting log timestamp");
-
-		let mut output = if log::max_level() <= log::LevelFilter::Info {
-			format!(
-				"{} {}",
-				Colour::Black.bold().paint(timestamp),
-				record.args(),
-			)
-		} else {
-			let name = ::std::thread::current()
-				.name()
-				.map_or_else(Default::default, |x| {
-					format!("{}", Colour::Blue.bold().paint(x))
-				});
-			let millis = (now.tm_nsec as f32 / 1000000.0).floor() as usize;
-			let timestamp = format!("{}.{:03}", timestamp, millis);
-			format!(
-				"{} {} {} {}  {}",
-				Colour::Black.bold().paint(timestamp),
-				name,
-				record.level(),
-				record.target(),
-				record.args()
-			)
-		};
-
-		if !isatty && record.level() <= log::Level::Info && atty::is(atty::Stream::Stdout) {
-			// duplicate INFO/WARN output to console
-			println!("{}", output);
+	let subscriber = tracing_subscriber::FmtSubscriber::builder()
+		.with_env_filter(env_filter)
+		.with_target(false)
+		.with_ansi(enable_color)
+		.with_writer(std::io::stderr)
+		.compact()
+		.finish();
+
+	if let Some(tracing_targets) = tracing_targets {
+		let profiling = sc_tracing::ProfilingLayer::new(tracing_receiver, &tracing_targets);
+
+		if let Err(e) = tracing::subscriber::set_global_default(subscriber.with(profiling)) {
+			return Err(format!(
+				"Registering Substrate tracing subscriber failed: {:}!", e
+			))
 		}
-
-		if !enable_color {
-			output = kill_color(output.as_ref());
+	} else {
+		if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
+			return Err(format!(
+				"Registering Substrate tracing subscriber  failed: {:}!", e
+			))
 		}
-
-		writeln!(buf, "{}", output)
-	});
-
-	if builder.try_init().is_err() {
-		info!("💬 Not registering Substrate logger, as there is already a global logger registered!");
-	}
-}
-
-fn kill_color(s: &str) -> String {
-	lazy_static! {
-		static ref RE: Regex = Regex::new("\x1b\\[[^m]+m").expect("Error initializing color regex");
 	}
-	RE.replace_all(s, "").to_string()
+	Ok(())
 }
diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml
index b107499daf48b..9f6f00a621dd4 100644
--- a/client/consensus/aura/Cargo.toml
+++ b/client/consensus/aura/Cargo.toml
@@ -39,10 +39,10 @@ prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../..
 
 [dev-dependencies]
 sp-keyring = { version = "2.0.0-rc6", path = "../../../primitives/keyring" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../../primitives/tracing" }
 sc-executor = { version = "0.8.0-rc6", path = "../../executor" }
 sc-network = { version = "0.8.0-rc6", path = "../../network" }
 sc-network-test = { version = "0.8.0-rc6", path = "../../network/test" }
 sc-service = { version = "0.8.0-rc6", default-features = false, path = "../../service" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../../test-utils/runtime/client" }
-env_logger = "0.7.0"
 tempfile = "3.1.0"
diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs
index 4204028711328..d79caaa1d6ac5 100644
--- a/client/consensus/aura/src/lib.rs
+++ b/client/consensus/aura/src/lib.rs
@@ -991,7 +991,7 @@ mod tests {
 	#[test]
 	#[allow(deprecated)]
 	fn authoring_blocks() {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 		let net = AuraTestNet::new(3);
 
 		let peers = &[
diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml
index 5838567096706..836232dc90a3a 100644
--- a/client/consensus/babe/Cargo.toml
+++ b/client/consensus/babe/Cargo.toml
@@ -53,13 +53,13 @@ retain_mut = "0.1.1"
 
 [dev-dependencies]
 sp-keyring = { version = "2.0.0-rc6", path = "../../../primitives/keyring" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../../primitives/tracing" }
 sc-executor = { version = "0.8.0-rc6", path = "../../executor" }
 sc-network = { version = "0.8.0-rc6", path = "../../network" }
 sc-network-test = { version = "0.8.0-rc6", path = "../../network/test" }
 sc-service = { version = "0.8.0-rc6", default-features = false, path = "../../service" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../../test-utils/runtime/client" }
 sc-block-builder = { version = "0.8.0-rc6", path = "../../block-builder" }
-env_logger = "0.7.0"
 rand_chacha = "0.2.2"
 tempfile = "3.1.0"
 
diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs
index e302a3b3d0a61..87876be8ae456 100644
--- a/client/consensus/babe/src/tests.rs
+++ b/client/consensus/babe/src/tests.rs
@@ -347,7 +347,7 @@ impl TestNetFactory for BabeTestNet {
 #[test]
 #[should_panic]
 fn rejects_empty_block() {
-	env_logger::try_init().unwrap();
+	sp_tracing::try_init_simple();
 	let mut net = BabeTestNet::new(3);
 	let block_builder = |builder: BlockBuilder<_, _, _>| {
 		builder.build().unwrap().block
@@ -360,7 +360,7 @@ fn rejects_empty_block() {
 fn run_one_test(
 	mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static,
 ) {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mutator = Arc::new(mutator) as Mutator;
 
 	MUTATOR.with(|m| *m.borrow_mut() = mutator.clone());
@@ -489,7 +489,7 @@ fn rejects_missing_consensus_digests() {
 
 #[test]
 fn wrong_consensus_engine_id_rejected() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let sig = AuthorityPair::generate().0.sign(b"");
 	let bad_seal: Item = DigestItem::Seal([0; 4], sig.to_vec());
 	assert!(bad_seal.as_babe_pre_digest().is_none());
@@ -498,14 +498,14 @@ fn wrong_consensus_engine_id_rejected() {
 
 #[test]
 fn malformed_pre_digest_rejected() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let bad_seal: Item = DigestItem::Seal(BABE_ENGINE_ID, [0; 64].to_vec());
 	assert!(bad_seal.as_babe_pre_digest().is_none());
 }
 
 #[test]
 fn sig_is_not_pre_digest() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let sig = AuthorityPair::generate().0.sign(b"");
 	let bad_seal: Item = DigestItem::Seal(BABE_ENGINE_ID, sig.to_vec());
 	assert!(bad_seal.as_babe_pre_digest().is_none());
@@ -514,7 +514,7 @@ fn sig_is_not_pre_digest() {
 
 #[test]
 fn can_author_block() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let keystore_path = tempfile::tempdir().expect("Creates keystore path");
 	let keystore = sc_keystore::Store::open(keystore_path.path(), None).expect("Creates keystore");
 	let pair = keystore.write().insert_ephemeral_from_seed::<AuthorityPair>("//Alice")
@@ -821,7 +821,7 @@ fn verify_slots_are_strictly_increasing() {
 
 #[test]
 fn babe_transcript_generation_match() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let keystore_path = tempfile::tempdir().expect("Creates keystore path");
 	let keystore = sc_keystore::Store::open(keystore_path.path(), None).expect("Creates keystore");
 	let pair = keystore.write().insert_ephemeral_from_seed::<AuthorityPair>("//Alice")
diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml
index 8305f856e09a2..33f443bce9d12 100644
--- a/client/consensus/manual-seal/Cargo.toml
+++ b/client/consensus/manual-seal/Cargo.toml
@@ -22,28 +22,27 @@ parking_lot = "0.10.0"
 serde = { version = "1.0", features=["derive"] }
 assert_matches = "1.3.0"
 
-sc-client-api = { path = "../../api", version = "2.0.0-rc5" }
-sc-consensus-babe = { path = "../../consensus/babe", version = "0.8.0-rc5" }
-sc-consensus-epochs = { path = "../../consensus/epochs", version = "0.8.0-rc5" }
-sp-consensus-babe = { path = "../../../primitives/consensus/babe", version = "0.8.0-rc5" }
-sc-keystore = { path = "../../keystore", version = "2.0.0-rc5" }
+sc-client-api = { path = "../../api", version = "2.0.0-rc6" }
+sc-consensus-babe = { path = "../../consensus/babe", version = "0.8.0-rc6" }
+sc-consensus-epochs = { path = "../../consensus/epochs", version = "0.8.0-rc6" }
+sp-consensus-babe = { path = "../../../primitives/consensus/babe", version = "0.8.0-rc6" }
+sc-keystore = { path = "../../keystore", version = "2.0.0-rc6" }
 
-sc-transaction-pool = { path = "../../transaction-pool", version = "2.0.0-rc5" }
-sp-blockchain = { path = "../../../primitives/blockchain", version = "2.0.0-rc5" }
-sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common", version = "0.8.0-rc5" }
-sp-inherents = { path = "../../../primitives/inherents", version = "2.0.0-rc5" }
-sp-runtime = {  path = "../../../primitives/runtime", version = "2.0.0-rc5" }
-sp-core = {  path = "../../../primitives/core", version = "2.0.0-rc5" }
-sp-api = {  path = "../../../primitives/api", version = "2.0.0-rc5" }
-sp-transaction-pool = { path = "../../../primitives/transaction-pool", version = "2.0.0-rc5" }
+sc-transaction-pool = { path = "../../transaction-pool", version = "2.0.0-rc6" }
+sp-blockchain = { path = "../../../primitives/blockchain", version = "2.0.0-rc6" }
+sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common", version = "0.8.0-rc6" }
+sp-inherents = { path = "../../../primitives/inherents", version = "2.0.0-rc6" }
+sp-runtime = {  path = "../../../primitives/runtime", version = "2.0.0-rc6" }
+sp-core = {  path = "../../../primitives/core", version = "2.0.0-rc6" }
+sp-api = {  path = "../../../primitives/api", version = "2.0.0-rc6" }
+sp-transaction-pool = { path = "../../../primitives/transaction-pool", version = "2.0.0-rc6" }
 sp-timestamp = { path = "../../../primitives/timestamp", version = "2.0.0-rc6" }
 
-prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc5" }
+prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc6" }
 
 [dev-dependencies]
 tokio = { version = "0.2", features = ["rt-core", "macros"] }
 sc-basic-authorship = { path = "../../basic-authorship", version = "0.8.0-rc6" }
 substrate-test-runtime-client = { path = "../../../test-utils/runtime/client", version = "2.0.0-rc6" }
 substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool", version = "2.0.0-rc6" }
-env_logger = "0.7.0"
 tempfile = "3.1.0"
diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml
index bbe6f83f4c1d2..79b75582dc1cd 100644
--- a/client/db/Cargo.toml
+++ b/client/db/Cargo.toml
@@ -39,8 +39,8 @@ prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.
 
 [dev-dependencies]
 sp-keyring = { version = "2.0.0-rc6", path = "../../primitives/keyring" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../test-utils/runtime/client" }
-env_logger = "0.7.0"
 quickcheck = "0.9"
 kvdb-rocksdb = "0.9.1"
 tempfile = "3"
diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs
index 927df1c0a7d25..8196a750557a8 100644
--- a/client/db/src/lib.rs
+++ b/client/db/src/lib.rs
@@ -1958,7 +1958,7 @@ pub(crate) mod tests {
 
 	#[test]
 	fn delete_only_when_negative_rc() {
-		let _ = ::env_logger::try_init();
+		sp_tracing::try_init_simple();
 		let key;
 		let backend = Backend::<Block>::new_test(1, 0);
 
diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs
index 434b301ed6240..0b4b6d4f88ef5 100644
--- a/client/db/src/storage_cache.rs
+++ b/client/db/src/storage_cache.rs
@@ -1024,7 +1024,7 @@ mod tests {
 
 	#[test]
 	fn simple_fork() {
-		let _ = ::env_logger::try_init();
+		sp_tracing::try_init_simple();
 
 		let root_parent = H256::random();
 		let key = H256::random()[..].to_vec();
@@ -1245,7 +1245,7 @@ mod tests {
 
 	#[test]
 	fn fix_storage_mismatch_issue() {
-		let _ = ::env_logger::try_init();
+		sp_tracing::try_init_simple();
 		let root_parent = H256::random();
 
 		let key = H256::random()[..].to_vec();
diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml
index 0b9829e6f342b..a60bff877d71e 100644
--- a/client/executor/Cargo.toml
+++ b/client/executor/Cargo.toml
@@ -47,6 +47,7 @@ sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
 sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 sc-tracing = { version = "2.0.0-rc6", path = "../tracing" }
 tracing = "0.1.18"
+tracing-subscriber = "0.2.10"
 
 [features]
 default = [ "std" ]
diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs
index a9ac0d0f30c23..1c744f544b48f 100644
--- a/client/executor/src/integration_tests/mod.rs
+++ b/client/executor/src/integration_tests/mod.rs
@@ -30,6 +30,7 @@ use test_case::test_case;
 use sp_trie::{TrieConfiguration, trie_types::Layout};
 use sp_wasm_interface::HostFunctions as _;
 use sp_runtime::traits::BlakeTwo256;
+use tracing_subscriber::layer::SubscriberExt;
 
 use crate::WasmExecutionMethod;
 
@@ -678,8 +679,11 @@ fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) {
 	let handler = TestTraceHandler(traces.clone());
 
 	// Create subscriber with wasm_tracing disabled
-	let test_subscriber = sc_tracing::ProfilingSubscriber::new_with_handler(
-		Box::new(handler), "integration_test_span_target");
+	let test_subscriber = tracing_subscriber::fmt().finish().with(
+		sc_tracing::ProfilingLayer::new_with_handler(
+			Box::new(handler), "integration_test_span_target"
+		)
+	);
 
 	let _guard = tracing::subscriber::set_default(test_subscriber);
 
diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml
index b73fbbd8d1728..60363544e3e4a 100644
--- a/client/finality-grandpa/Cargo.toml
+++ b/client/finality-grandpa/Cargo.toml
@@ -54,7 +54,7 @@ sp-keyring = { version = "2.0.0-rc6", path = "../../primitives/keyring" }
 substrate-test-runtime-client = { version = "2.0.0-rc6",  path = "../../test-utils/runtime/client" }
 sp-consensus-babe = { version = "0.8.0-rc6", path = "../../primitives/consensus/babe" }
 sp-state-machine = { version = "0.8.0-rc6", path = "../../primitives/state-machine" }
-env_logger = "0.7.0"
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 tokio = { version = "0.2", features = ["rt-core"] }
 tempfile = "3.1.0"
 sp-api = { version = "2.0.0-rc6", path = "../../primitives/api" }
diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs
index 6a1513769aa26..1a773acd6d0fb 100644
--- a/client/finality-grandpa/src/communication/tests.rs
+++ b/client/finality-grandpa/src/communication/tests.rs
@@ -361,7 +361,7 @@ fn good_commit_leads_to_relay() {
 
 #[test]
 fn bad_commit_leads_to_report() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let private = [Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie];
 	let public = make_ids(&private[..]);
 	let voter_set = Arc::new(VoterSet::new(public.iter().cloned()).unwrap());
diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs
index 6e8def57f50ea..7e5282fe3e9f0 100644
--- a/client/finality-grandpa/src/tests.rs
+++ b/client/finality-grandpa/src/tests.rs
@@ -417,7 +417,7 @@ fn add_forced_change(
 
 #[test]
 fn finalize_3_voters_no_observers() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 	let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie];
 	let voters = make_ids(peers);
@@ -522,7 +522,7 @@ fn finalize_3_voters_1_full_observer() {
 
 #[test]
 fn transition_3_voters_twice_1_full_observer() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let peers_a = &[
 		Ed25519Keyring::Alice,
 		Ed25519Keyring::Bob,
@@ -792,7 +792,7 @@ fn sync_justifications_on_change_blocks() {
 
 #[test]
 fn finalizes_multiple_pending_changes_in_order() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 
 	let peers_a = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie];
@@ -852,7 +852,7 @@ fn finalizes_multiple_pending_changes_in_order() {
 
 #[test]
 fn force_change_to_new_set() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 	// two of these guys are offline.
 	let genesis_authorities = &[
@@ -1014,7 +1014,7 @@ fn voter_persists_its_votes() {
 	use futures::future;
 	use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver};
 
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 
 	// we have two authorities but we'll only be running the voter for alice
@@ -1270,7 +1270,7 @@ fn voter_persists_its_votes() {
 
 #[test]
 fn finalize_3_voters_1_light_observer() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 	let authorities = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie];
 	let voters = make_ids(authorities);
@@ -1315,7 +1315,7 @@ fn finalize_3_voters_1_light_observer() {
 
 #[test]
 fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 
 	let peers = &[Ed25519Keyring::Alice];
@@ -1345,7 +1345,7 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ
 	// for debug: to ensure that without forced change light client will sync finality proof
 	const FORCE_CHANGE: bool = true;
 
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 
 	// two of these guys are offline.
@@ -1409,7 +1409,7 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ
 
 #[test]
 fn voter_catches_up_to_latest_round_when_behind() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut runtime = Runtime::new().unwrap();
 
 	let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob];
diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml
index 6e6dc01f91e53..e711384d7f5ba 100644
--- a/client/informant/Cargo.toml
+++ b/client/informant/Cargo.toml
@@ -20,6 +20,6 @@ sc-client-api = { version = "2.0.0-rc6", path = "../api" }
 sc-network = { version = "0.8.0-rc6", path = "../network" }
 sp-blockchain = { version = "2.0.0-rc6", path = "../../primitives/blockchain" }
 sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
-sp-utils = { version = "2.0.0-rc2", path = "../../primitives/utils" }
-sp-transaction-pool = { version = "2.0.0-rc2", path = "../../primitives/transaction-pool" }
+sp-utils = { version = "2.0.0-rc6", path = "../../primitives/utils" }
+sp-transaction-pool = { version = "2.0.0-rc6", path = "../../primitives/transaction-pool" }
 wasm-timer = "0.2"
diff --git a/client/light/Cargo.toml b/client/light/Cargo.toml
index 23b306d178e37..e160526ce8d29 100644
--- a/client/light/Cargo.toml
+++ b/client/light/Cargo.toml
@@ -13,15 +13,15 @@ documentation = "https://docs.rs/sc-light"
 parking_lot = "0.10.0"
 lazy_static = "1.4.0"
 hash-db = "0.15.2"
-sp-runtime = { version = "2.0.0-rc2", path = "../../primitives/runtime" }
-sp-externalities = { version = "0.8.0-rc2", path = "../../primitives/externalities" }
-sp-blockchain = { version = "2.0.0-rc2", path = "../../primitives/blockchain" }
-sp-core = { version = "2.0.0-rc2", path = "../../primitives/core" }
-sp-state-machine = { version = "0.8.0-rc2", path = "../../primitives/state-machine" }
-sc-client-api = { version = "2.0.0-rc2", path = "../api" }
-sp-api = { version = "2.0.0-rc2", path = "../../primitives/api" }
+sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
+sp-externalities = { version = "0.8.0-rc6", path = "../../primitives/externalities" }
+sp-blockchain = { version = "2.0.0-rc6", path = "../../primitives/blockchain" }
+sp-core = { version = "2.0.0-rc6", path = "../../primitives/core" }
+sp-state-machine = { version = "0.8.0-rc6", path = "../../primitives/state-machine" }
+sc-client-api = { version = "2.0.0-rc6", path = "../api" }
+sp-api = { version = "2.0.0-rc6", path = "../../primitives/api" }
 codec = { package = "parity-scale-codec", version = "1.3.4" }
-sc-executor = { version = "0.8.0-rc2", path = "../executor" }
+sc-executor = { version = "0.8.0-rc6", path = "../executor" }
 
 [features]
 default = []
diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml
index 7c06de7ccd1cb..c220728a4e4b4 100644
--- a/client/network/Cargo.toml
+++ b/client/network/Cargo.toml
@@ -69,12 +69,12 @@ features = ["identify", "kad", "mdns-async-std", "mplex", "noise", "ping", "requ
 
 [dev-dependencies]
 assert_matches = "1.3"
-env_logger = "0.7.0"
 libp2p = { version = "0.28.1", default-features = false }
 quickcheck = "0.9.0"
 rand = "0.7.2"
 sp-keyring = { version = "2.0.0-rc6", path = "../../primitives/keyring" }
 sp-test-primitives = { version = "2.0.0-rc6", path = "../../primitives/test-primitives" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 substrate-test-runtime = { version = "2.0.0-rc6", path = "../../test-utils/runtime" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../test-utils/runtime/client" }
 tempfile = "3.1.0"
diff --git a/client/network/src/light_client_handler.rs b/client/network/src/light_client_handler.rs
index 7f5ec54470e5d..c1ff14fc82a22 100644
--- a/client/network/src/light_client_handler.rs
+++ b/client/network/src/light_client_handler.rs
@@ -2004,7 +2004,7 @@ mod tests {
 
 	#[test]
 	fn send_receive_header() {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 		let chan = oneshot::channel();
 		let request = light::RemoteHeaderRequest {
 			cht_root: Default::default(),
diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs
index d025b86b2536f..df336c25339fd 100644
--- a/client/network/src/protocol/sync/extra_requests.rs
+++ b/client/network/src/protocol/sync/extra_requests.rs
@@ -463,7 +463,7 @@ mod tests {
 
 	#[test]
 	fn request_is_rescheduled_when_earlier_block_is_finalized() {
-		let _ = ::env_logger::try_init();
+		sp_tracing::try_init_simple();
 
 		let mut finality_proofs = ExtraRequests::<Block>::new("test");
 
diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml
index fc6c47699fb8f..29b1139877208 100644
--- a/client/network/test/Cargo.toml
+++ b/client/network/test/Cargo.toml
@@ -28,8 +28,8 @@ sp-runtime = { version = "2.0.0-rc6", path = "../../../primitives/runtime" }
 sp-core = { version = "2.0.0-rc6", path = "../../../primitives/core" }
 sc-block-builder = { version = "0.8.0-rc6", path = "../../block-builder" }
 sp-consensus-babe = { version = "0.8.0-rc6", path = "../../../primitives/consensus/babe" }
-env_logger = "0.7.0"
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../../test-utils/runtime/client" }
 substrate-test-runtime = { version = "2.0.0-rc6", path = "../../../test-utils/runtime" }
 tempfile = "3.1.0"
+sp-tracing = { version = "2.0.0-rc6", path = "../../../primitives/tracing" }
 sc-service = { version = "0.8.0-rc6", default-features = false, features = ["test-helpers"],  path = "../../service" }
diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs
index 6b9465a3ea52e..17f94d4a860f1 100644
--- a/client/network/test/src/sync.rs
+++ b/client/network/test/src/sync.rs
@@ -24,7 +24,7 @@ use sp_consensus::block_validation::Validation;
 use substrate_test_runtime::Header;
 
 fn test_ancestor_search_when_common_is(n: usize) {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 
 	net.peer(0).push_blocks(n, false);
@@ -42,7 +42,7 @@ fn test_ancestor_search_when_common_is(n: usize) {
 
 #[test]
 fn sync_peers_works() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 
 	block_on(futures::future::poll_fn::<(), _>(|cx| {
@@ -58,7 +58,7 @@ fn sync_peers_works() {
 
 #[test]
 fn sync_cycle_from_offline_to_syncing_to_offline() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 	for peer in 0..3 {
 		// Offline, and not major syncing.
@@ -113,7 +113,7 @@ fn sync_cycle_from_offline_to_syncing_to_offline() {
 
 #[test]
 fn syncing_node_not_major_syncing_when_disconnected() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 
 	// Generate blocks.
@@ -147,7 +147,7 @@ fn syncing_node_not_major_syncing_when_disconnected() {
 
 #[test]
 fn sync_from_two_peers_works() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 	net.peer(1).push_blocks(100, false);
 	net.peer(2).push_blocks(100, false);
@@ -159,7 +159,7 @@ fn sync_from_two_peers_works() {
 
 #[test]
 fn sync_from_two_peers_with_ancestry_search_works() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 	net.peer(0).push_blocks(10, true);
 	net.peer(1).push_blocks(100, false);
@@ -171,7 +171,7 @@ fn sync_from_two_peers_with_ancestry_search_works() {
 
 #[test]
 fn ancestry_search_works_when_backoff_is_one() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 
 	net.peer(0).push_blocks(1, false);
@@ -185,7 +185,7 @@ fn ancestry_search_works_when_backoff_is_one() {
 
 #[test]
 fn ancestry_search_works_when_ancestor_is_genesis() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 
 	net.peer(0).push_blocks(13, true);
@@ -214,7 +214,7 @@ fn ancestry_search_works_when_common_is_hundred() {
 
 #[test]
 fn sync_long_chain_works() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(2);
 	net.peer(1).push_blocks(500, false);
 	net.block_until_sync();
@@ -224,7 +224,7 @@ fn sync_long_chain_works() {
 
 #[test]
 fn sync_no_common_longer_chain_fails() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 	net.peer(0).push_blocks(20, true);
 	net.peer(1).push_blocks(20, false);
@@ -242,7 +242,7 @@ fn sync_no_common_longer_chain_fails() {
 
 #[test]
 fn sync_justifications() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = JustificationTestNet::new(3);
 	net.peer(0).push_blocks(20, false);
 	net.block_until_sync();
@@ -283,7 +283,7 @@ fn sync_justifications() {
 
 #[test]
 fn sync_justifications_across_forks() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = JustificationTestNet::new(3);
 	// we push 5 blocks
 	net.peer(0).push_blocks(5, false);
@@ -315,7 +315,7 @@ fn sync_justifications_across_forks() {
 
 #[test]
 fn sync_after_fork_works() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 	net.peer(0).push_blocks(30, false);
 	net.peer(1).push_blocks(30, false);
@@ -338,7 +338,7 @@ fn sync_after_fork_works() {
 
 #[test]
 fn syncs_all_forks() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(4);
 	net.peer(0).push_blocks(2, false);
 	net.peer(1).push_blocks(2, false);
@@ -356,7 +356,7 @@ fn syncs_all_forks() {
 
 #[test]
 fn own_blocks_are_announced() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 	net.block_until_sync(); // connect'em
 	net.peer(0).generate_blocks(1, BlockOrigin::Own, |builder| builder.build().unwrap().block);
@@ -372,7 +372,7 @@ fn own_blocks_are_announced() {
 
 #[test]
 fn blocks_are_not_announced_by_light_nodes() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(0);
 
 	// full peer0 is connected to light peer
@@ -401,7 +401,7 @@ fn blocks_are_not_announced_by_light_nodes() {
 
 #[test]
 fn can_sync_small_non_best_forks() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(2);
 	net.peer(0).push_blocks(30, false);
 	net.peer(1).push_blocks(30, false);
@@ -464,7 +464,7 @@ fn can_sync_small_non_best_forks() {
 
 #[test]
 fn can_not_sync_from_light_peer() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 
 	// given the network with 1 full nodes (#0) and 1 light node (#1)
 	let mut net = TestNet::new(1);
@@ -497,7 +497,7 @@ fn can_not_sync_from_light_peer() {
 
 #[test]
 fn light_peer_imports_header_from_announce() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 
 	fn import_with_announce(net: &mut TestNet, hash: H256) {
 		net.peer(0).announce_block(hash, Vec::new());
@@ -530,7 +530,7 @@ fn light_peer_imports_header_from_announce() {
 
 #[test]
 fn can_sync_explicit_forks() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(2);
 	net.peer(0).push_blocks(30, false);
 	net.peer(1).push_blocks(30, false);
@@ -584,7 +584,7 @@ fn can_sync_explicit_forks() {
 
 #[test]
 fn syncs_header_only_forks() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(0);
 	net.add_full_peer_with_config(Default::default());
 	net.add_full_peer_with_config(FullPeerConfig { keep_blocks: Some(3), ..Default::default() });
@@ -602,7 +602,7 @@ fn syncs_header_only_forks() {
 
 #[test]
 fn does_not_sync_announced_old_best_block() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(3);
 
 	let old_hash = net.peer(0).push_blocks(1, false);
@@ -630,7 +630,7 @@ fn does_not_sync_announced_old_best_block() {
 #[test]
 fn full_sync_requires_block_body() {
 	// Check that we don't sync headers-only in full mode.
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(2);
 
 	net.peer(0).push_headers(1);
@@ -649,7 +649,7 @@ fn full_sync_requires_block_body() {
 
 #[test]
 fn imports_stale_once() {
-	let _ = ::env_logger::try_init();
+	sp_tracing::try_init_simple();
 
 	fn import_with_announce(net: &mut TestNet, hash: H256) {
 		// Announce twice
@@ -685,7 +685,7 @@ fn imports_stale_once() {
 
 #[test]
 fn can_sync_to_peers_with_wrong_common_block() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut net = TestNet::new(2);
 
 	net.peer(0).push_blocks(2, true);
@@ -720,7 +720,7 @@ impl BlockAnnounceValidator<Block> for NewBestBlockAnnounceValidator {
 
 #[test]
 fn sync_blocks_when_block_announce_validator_says_it_is_new_best() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	log::trace!(target: "sync", "Test");
 	let mut net = TestNet::with_fork_choice(ForkChoiceStrategy::Custom(false));
 	net.add_full_peer_with_config(Default::default());
diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml
index 9f574ff9ebe46..2c5963e68d1a4 100644
--- a/client/offchain/Cargo.toml
+++ b/client/offchain/Cargo.toml
@@ -36,10 +36,10 @@ hyper = "0.13.2"
 hyper-rustls = "0.21.0"
 
 [dev-dependencies]
-env_logger = "0.7.0"
 sc-client-db = { version = "0.8.0-rc6", default-features = true, path = "../db/" }
 sc-transaction-pool = { version = "2.0.0-rc6", path = "../../client/transaction-pool" }
 sp-transaction-pool = { version = "2.0.0-rc6", path = "../../primitives/transaction-pool" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../test-utils/runtime/client" }
 tokio = "0.2"
 lazy_static = "1.4.0"
diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs
index a7ab07c549665..6fb1da19bf058 100644
--- a/client/offchain/src/api.rs
+++ b/client/offchain/src/api.rs
@@ -328,7 +328,7 @@ mod tests {
 	}
 
 	fn offchain_api() -> (Api<LocalStorage>, AsyncApi) {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 		let db = LocalStorage::new_test();
 		let mock = Arc::new(TestNetwork());
 		let shared_client = SharedClient::new();
diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs
index 89f2b7b8100b2..885294449fb95 100644
--- a/client/offchain/src/lib.rs
+++ b/client/offchain/src/lib.rs
@@ -281,7 +281,7 @@ mod tests {
 
 	#[test]
 	fn should_call_into_runtime_and_produce_extrinsic() {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 
 		let client = Arc::new(substrate_test_runtime_client::new());
 		let spawner = sp_core::testing::TaskExecutor::new();
diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml
index fc4d3298a41d5..370ee679154e3 100644
--- a/client/service/Cargo.toml
+++ b/client/service/Cargo.toml
@@ -68,7 +68,7 @@ sc-rpc-server = { version = "2.0.0-rc6", path = "../rpc-servers" }
 sc-rpc = { version = "2.0.0-rc6", path = "../rpc" }
 sc-block-builder = { version = "0.8.0-rc6", path = "../block-builder" }
 sp-block-builder = { version = "2.0.0-rc6", path = "../../primitives/block-builder" }
-sc-informant = { version = "0.8.0-rc2", path = "../informant" }
+sc-informant = { version = "0.8.0-rc6", path = "../informant" }
 sc-telemetry = { version = "2.0.0-rc6", path = "../telemetry" }
 sc-offchain = { version = "2.0.0-rc6", path = "../offchain" }
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc6"}
diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs
index 49f54365ddf37..25abfdffed845 100644
--- a/client/service/src/builder.rs
+++ b/client/service/src/builder.rs
@@ -36,7 +36,7 @@ use sp_consensus::{
 use futures::{FutureExt, StreamExt, future::ready, channel::oneshot};
 use jsonrpc_pubsub::manager::SubscriptionManager;
 use sc_keystore::Store as Keystore;
-use log::{info, warn, error};
+use log::{info, warn};
 use sc_network::config::{Role, FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder};
 use sc_network::NetworkService;
 use parking_lot::RwLock;
@@ -572,17 +572,6 @@ pub fn spawn_tasks<TBl, TBackend, TExPool, TRpc, TCl>(
 		))
 	});
 
-	// Instrumentation
-	if let Some(tracing_targets) = config.tracing_targets.as_ref() {
-		let subscriber = sc_tracing::ProfilingSubscriber::new(
-			config.tracing_receiver, tracing_targets
-		);
-		match tracing::subscriber::set_global_default(subscriber) {
-			Ok(_) => (),
-			Err(e) => error!(target: "tracing", "Unable to set global default subscriber {}", e),
-		}
-	}
-
 	// Spawn informant task
 	spawn_handle.spawn("informant", sc_informant::build(
 		client.clone(),
diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml
index 03d5e264c85da..83dfa76b8999e 100644
--- a/client/service/test/Cargo.toml
+++ b/client/service/test/Cargo.toml
@@ -17,7 +17,6 @@ tempfile = "3.1.0"
 tokio = "0.1.22"
 futures01 = { package = "futures", version = "0.1.29" }
 log = "0.4.8"
-env_logger = "0.7.0"
 fdlimit = "0.2.0"
 parking_lot = "0.10.0"
 sc-light = { version = "2.0.0-rc6", path = "../../light" }
@@ -42,3 +41,4 @@ sc-block-builder = { version = "0.8.0-rc6", path = "../../block-builder" }
 sc-executor = { version = "0.8.0-rc6", path = "../../executor" }
 sp-panic-handler = { version = "2.0.0-rc6", path = "../../../primitives/panic-handler" }
 parity-scale-codec = "1.3.4"
+sp-tracing = { version = "2.0.0-rc6", path = "../../../primitives/tracing" }
diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs
index ea3eaa7ffbabf..34b063a3e3484 100644
--- a/client/service/test/src/client/mod.rs
+++ b/client/service/test/src/client/mod.rs
@@ -1206,7 +1206,7 @@ fn get_header_by_block_number_doesnt_panic() {
 
 #[test]
 fn state_reverted_on_reorg() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let mut client = substrate_test_runtime_client::new();
 
 	let current_balance = |client: &substrate_test_runtime_client::TestClient|
@@ -1266,7 +1266,7 @@ fn state_reverted_on_reorg() {
 
 #[test]
 fn doesnt_import_blocks_that_revert_finality() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let tmp = tempfile::tempdir().unwrap();
 
 	// we need to run with archive pruning to avoid pruning non-canonical
@@ -1467,7 +1467,7 @@ fn respects_block_rules() {
 
 #[test]
 fn returns_status_for_pruned_blocks() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	let tmp = tempfile::tempdir().unwrap();
 
 	// set to prune after 1 block
@@ -1855,4 +1855,4 @@ fn reorg_triggers_a_notification_even_for_sources_that_should_not_trigger_notifi
 	// We should have a tree route of the re-org
 	let tree_route = notification.tree_route.unwrap();
 	assert_eq!(tree_route.enacted()[0].hash, b1.hash());
-}
+}
\ No newline at end of file
diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs
index 0d589cee7e12d..cfe815f174fac 100644
--- a/client/service/test/src/lib.rs
+++ b/client/service/test/src/lib.rs
@@ -289,7 +289,7 @@ impl<G, E, F, L, U> TestNet<G, E, F, L, U> where
 		)>,
 		base_port: u16
 	) -> TestNet<G, E, F, L, U> {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 		fdlimit::raise_fd_limit();
 		let runtime = Runtime::new().expect("Error creating tokio runtime");
 		let mut net = TestNet {
diff --git a/client/state-db/Cargo.toml b/client/state-db/Cargo.toml
index f78e0ca505a61..7361ef0a8cb21 100644
--- a/client/state-db/Cargo.toml
+++ b/client/state-db/Cargo.toml
@@ -19,6 +19,3 @@ sp-core = { version = "2.0.0-rc6", path = "../../primitives/core" }
 codec = { package = "parity-scale-codec", version = "1.3.4", features = ["derive"] }
 parity-util-mem = { version = "0.7.0", default-features = false, features = ["primitive-types"] }
 parity-util-mem-derive = "0.1.0"
-
-[dev-dependencies]
-env_logger = "0.7.0"
diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml
index 40ab1bd460359..9444a9520f665 100644
--- a/client/tracing/Cargo.toml
+++ b/client/tracing/Cargo.toml
@@ -21,6 +21,5 @@ serde_json = "1.0.41"
 slog = { version = "2.5.2", features = ["nested-values"] }
 tracing = "0.1.18"
 tracing-subscriber = "0.2.10"
-sp-tracing = { version = "2.0.0-rc2", path = "../../primitives/tracing" }
-
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 sc-telemetry = { version = "2.0.0-rc6", path = "../telemetry" }
diff --git a/client/tracing/src/lib.rs b/client/tracing/src/lib.rs
index f642b00720f1a..e509f2218a2ea 100644
--- a/client/tracing/src/lib.rs
+++ b/client/tracing/src/lib.rs
@@ -26,7 +26,6 @@
 
 use rustc_hash::FxHashMap;
 use std::fmt;
-use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Duration, Instant};
 
 use parking_lot::Mutex;
@@ -35,21 +34,18 @@ use tracing::{
 	event::Event,
 	field::{Visit, Field},
 	Level,
-	metadata::Metadata,
 	span::{Attributes, Id, Record},
 	subscriber::Subscriber,
 };
-use tracing_subscriber::CurrentSpan;
+use tracing_subscriber::{CurrentSpan, layer::{Layer, Context}};
 
 use sc_telemetry::{telemetry, SUBSTRATE_INFO};
 use sp_tracing::proxy::{WASM_NAME_KEY, WASM_TARGET_KEY, WASM_TRACE_IDENTIFIER};
 
 const ZERO_DURATION: Duration = Duration::from_nanos(0);
-const PROXY_TARGET: &'static str = "sp_tracing::proxy";
 
 /// Responsible for assigning ids to new spans, which are not re-used.
-pub struct ProfilingSubscriber {
-	next_id: AtomicU64,
+pub struct ProfilingLayer {
 	targets: Vec<(String, Level)>,
 	trace_handler: Box<dyn TraceHandler>,
 	span_data: Mutex<FxHashMap<Id, SpanDatum>>,
@@ -216,12 +212,12 @@ impl slog::Value for Values {
 	}
 }
 
-impl ProfilingSubscriber {
+impl ProfilingLayer {
 	/// Takes a `TracingReceiver` and a comma separated list of targets,
 	/// either with a level: "pallet=trace,frame=debug"
 	/// or without: "pallet,frame" in which case the level defaults to `trace`.
 	/// wasm_tracing indicates whether to enable wasm traces
-	pub fn new(receiver: TracingReceiver, targets: &str) -> ProfilingSubscriber {
+	pub fn new(receiver: TracingReceiver, targets: &str) -> Self {
 		match receiver {
 			TracingReceiver::Log => Self::new_with_handler(Box::new(LogTraceHandler), targets),
 			TracingReceiver::Telemetry => Self::new_with_handler(
@@ -237,11 +233,10 @@ impl ProfilingSubscriber {
 	/// or without: "pallet" in which case the level defaults to `trace`.
 	/// wasm_tracing indicates whether to enable wasm traces
 	pub fn new_with_handler(trace_handler: Box<dyn TraceHandler>, targets: &str)
-		-> ProfilingSubscriber
+		-> Self
 	{
 		let targets: Vec<_> = targets.split(',').map(|s| parse_target(s)).collect();
-		ProfilingSubscriber {
-			next_id: AtomicU64::new(1),
+		Self {
 			targets,
 			trace_handler,
 			span_data: Mutex::new(FxHashMap::default()),
@@ -276,25 +271,14 @@ fn parse_target(s: &str) -> (String, Level) {
 	}
 }
 
-impl Subscriber for ProfilingSubscriber {
-	fn enabled(&self, metadata: &Metadata<'_>) -> bool {
-		if metadata.target() == PROXY_TARGET || self.check_target(metadata.target(), metadata.level()) {
-			log::debug!(target: "tracing", "Enabled target: {}, level: {}", metadata.target(), metadata.level());
-			true
-		} else {
-			log::debug!(target: "tracing", "Disabled target: {}, level: {}", metadata.target(), metadata.level());
-			false
-		}
-	}
-
-	fn new_span(&self, attrs: &Attributes<'_>) -> Id {
-		let id = Id::from_u64(self.next_id.fetch_add(1, Ordering::Relaxed));
+impl<S: Subscriber> Layer<S> for ProfilingLayer {
+	fn new_span(&self, attrs: &Attributes<'_>, id: &Id, _ctx: Context<S>) {
 		let mut values = Values::default();
 		attrs.record(&mut values);
 		// If this is a wasm trace, check if target/level is enabled
 		if let Some(wasm_target) = values.string_values.get(WASM_TARGET_KEY) {
 			if !self.check_target(wasm_target, attrs.metadata().level()) {
-				return id
+				return
 			}
 		}
 		let span_datum = SpanDatum {
@@ -309,19 +293,16 @@ impl Subscriber for ProfilingSubscriber {
 			values,
 		};
 		self.span_data.lock().insert(id.clone(), span_datum);
-		id
 	}
 
-	fn record(&self, span: &Id, values: &Record<'_>) {
+	fn on_record(&self, span: &Id, values: &Record<'_>, _ctx: Context<S>) {
 		let mut span_data = self.span_data.lock();
 		if let Some(s) = span_data.get_mut(span) {
 			values.record(&mut s.values);
 		}
 	}
 
-	fn record_follows_from(&self, _span: &Id, _follows: &Id) {}
-
-	fn event(&self, event: &Event<'_>) {
+	fn on_event(&self, event: &Event<'_>, _ctx: Context<S>) {
 		let mut values = Values::default();
 		event.record(&mut values);
 		let trace_event = TraceEvent {
@@ -334,7 +315,7 @@ impl Subscriber for ProfilingSubscriber {
 		self.trace_handler.handle_event(trace_event);
 	}
 
-	fn enter(&self, span: &Id) {
+	fn on_enter(&self, span: &Id, _ctx: Context<S>) {
 		self.current_span.enter(span.clone());
 		let mut span_data = self.span_data.lock();
 		let start_time = Instant::now();
@@ -343,7 +324,7 @@ impl Subscriber for ProfilingSubscriber {
 		}
 	}
 
-	fn exit(&self, span: &Id) {
+	fn on_exit(&self, span: &Id, _ctx: Context<S>) {
 		self.current_span.exit();
 		let end_time = Instant::now();
 		let mut span_data = self.span_data.lock();
@@ -352,7 +333,7 @@ impl Subscriber for ProfilingSubscriber {
 		}
 	}
 
-	fn try_close(&self, span: Id) -> bool {
+	fn on_close(&self, span: Id, _ctx: Context<S>) {
 		let span_datum = {
 			let mut span_data = self.span_data.lock();
 			span_data.remove(&span)
@@ -373,7 +354,6 @@ impl Subscriber for ProfilingSubscriber {
 				self.trace_handler.handle_span(span_datum);
 			}
 		};
-		true
 	}
 }
 
@@ -458,6 +438,7 @@ impl TraceHandler for TelemetryTraceHandler {
 mod tests {
 	use super::*;
 	use std::sync::Arc;
+	use tracing_subscriber::layer::SubscriberExt;
 
 	struct TestTraceHandler {
 		spans: Arc<Mutex<Vec<SpanDatum>>>,
@@ -474,18 +455,24 @@ mod tests {
 		}
 	}
 
-	fn setup_subscriber() -> (ProfilingSubscriber, Arc<Mutex<Vec<SpanDatum>>>, Arc<Mutex<Vec<TraceEvent>>>) {
+	type TestSubscriber = tracing_subscriber::layer::Layered<
+		ProfilingLayer,
+		tracing_subscriber::fmt::Subscriber
+	>;
+
+	fn setup_subscriber() -> (TestSubscriber, Arc<Mutex<Vec<SpanDatum>>>, Arc<Mutex<Vec<TraceEvent>>>) {
 		let spans = Arc::new(Mutex::new(Vec::new()));
 		let events = Arc::new(Mutex::new(Vec::new()));
 		let handler = TestTraceHandler {
 			spans: spans.clone(),
 			events: events.clone(),
 		};
-		let test_subscriber = ProfilingSubscriber::new_with_handler(
+		let layer = ProfilingLayer::new_with_handler(
 			Box::new(handler),
 			"test_target"
 		);
-		(test_subscriber, spans, events)
+		let subscriber = tracing_subscriber::fmt().finish().with(layer);
+		(subscriber, spans, events)
 	}
 
 	#[test]
diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs
index 93b11e8d95c57..372ae11b148ff 100644
--- a/frame/elections-phragmen/src/lib.rs
+++ b/frame/elections-phragmen/src/lib.rs
@@ -1398,7 +1398,7 @@ mod tests {
 		assert!(
 			Elections::members().iter().chain(
 				Elections::runners_up().iter()
-			).all(|(_, s)| *s != Zero::zero())
+			).all(|(_, s)| *s != u64::zero())
 		);
 	}
 
@@ -1453,15 +1453,15 @@ mod tests {
 			assert_eq!(Elections::term_duration(), 5);
 			assert_eq!(Elections::election_rounds(), 0);
 
-			assert_eq!(Elections::members(), vec![]);
-			assert_eq!(Elections::runners_up(), vec![]);
+			assert!(Elections::members().is_empty());
+			assert!(Elections::runners_up().is_empty());
 
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 			assert_eq!(<Candidates<Test>>::decode_len(), None);
 			assert!(Elections::is_candidate(&1).is_err());
 
-			assert_eq!(all_voters(), vec![]);
-			assert_eq!(votes_of(&1), vec![]);
+			assert!(all_voters().is_empty());
+			assert!(votes_of(&1).is_empty());
 		});
 	}
 
@@ -1536,16 +1536,16 @@ mod tests {
 			assert_eq!(Elections::desired_members(), 2);
 			assert_eq!(Elections::election_rounds(), 0);
 
-			assert_eq!(Elections::members_ids(), vec![]);
-			assert_eq!(Elections::runners_up(), vec![]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::members_ids().is_empty());
+			assert!(Elections::runners_up().is_empty());
+			assert!(Elections::candidates().is_empty());
 
 			System::set_block_number(5);
 			Elections::end_block(System::block_number());
 
-			assert_eq!(Elections::members_ids(), vec![]);
-			assert_eq!(Elections::runners_up(), vec![]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::members_ids().is_empty());
+			assert!(Elections::runners_up().is_empty());
+			assert!(Elections::candidates().is_empty());
 		});
 	}
 
@@ -1588,18 +1588,18 @@ mod tests {
 			assert!(Elections::is_candidate(&2).is_ok());
 			assert_eq!(Elections::candidates(), vec![1, 2]);
 
-			assert_eq!(Elections::members_ids(), vec![]);
-			assert_eq!(Elections::runners_up(), vec![]);
+			assert!(Elections::members_ids().is_empty());
+			assert!(Elections::runners_up().is_empty());
 
 			System::set_block_number(5);
 			Elections::end_block(System::block_number());
 
 			assert!(Elections::is_candidate(&1).is_err());
 			assert!(Elections::is_candidate(&2).is_err());
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 
-			assert_eq!(Elections::members_ids(), vec![]);
-			assert_eq!(Elections::runners_up(), vec![]);
+			assert!(Elections::members_ids().is_empty());
+			assert!(Elections::runners_up().is_empty());
 		});
 	}
 
@@ -1627,8 +1627,8 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members_ids(), vec![5]);
-			assert_eq!(Elections::runners_up(), vec![]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::runners_up().is_empty());
+			assert!(Elections::candidates().is_empty());
 
 			assert_noop!(
 				submit_candidacy(Origin::signed(5)),
@@ -1742,7 +1742,7 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members_ids(), vec![4, 5]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 
 			assert_ok!(vote(Origin::signed(3), vec![4, 5], 10));
 		});
@@ -1765,7 +1765,7 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members_ids(), vec![4, 5]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 
 			assert_ok!(vote(Origin::signed(3), vec![4, 5], 10));
 			assert_eq!(PRIME.with(|p| *p.borrow()), Some(4));
@@ -1791,7 +1791,7 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members_ids(), vec![3, 5]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 
 			assert_eq!(PRIME.with(|p| *p.borrow()), Some(5));
 		});
@@ -1876,7 +1876,7 @@ mod tests {
 			assert_ok!(Elections::remove_voter(Origin::signed(2)));
 
 			assert_eq_uvec!(all_voters(), vec![3]);
-			assert_eq!(votes_of(&2), vec![]);
+			assert!(votes_of(&2).is_empty());
 			assert_eq!(Elections::locked_stake_of(&2), 0);
 
 			assert_eq!(balances(&2), (20, 0));
@@ -1898,7 +1898,7 @@ mod tests {
 			assert_ok!(vote(Origin::signed(2), vec![5], 20));
 
 			assert_ok!(Elections::remove_voter(Origin::signed(2)));
-			assert_eq!(all_voters(), vec![]);
+			assert!(all_voters().is_empty());
 
 			assert_noop!(Elections::remove_voter(Origin::signed(2)), Error::<Test>::MustBeVoter);
 		});
@@ -2008,7 +2008,7 @@ mod tests {
 
 			assert_eq!(Elections::members_ids(), vec![4, 5]);
 			assert_eq!(Elections::runners_up_ids(), vec![6]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 
 			// all of them have a member or runner-up that they voted for.
 			assert_eq!(Elections::is_defunct_voter(&votes_of(&5)), false);
@@ -2044,7 +2044,7 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members_ids(), vec![4, 5]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 
 			assert_eq!(balances(&3), (28, 2));
 			assert_eq!(balances(&5), (45, 5));
@@ -2072,7 +2072,7 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members_ids(), vec![4, 5]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 
 			assert_eq!(balances(&4), (35, 5));
 			assert_eq!(balances(&5), (45, 5));
@@ -2113,9 +2113,9 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members(), vec![(3, 30), (5, 20)]);
-			assert_eq!(Elections::runners_up(), vec![]);
+			assert!(Elections::runners_up().is_empty());
 			assert_eq_uvec!(all_voters(), vec![2, 3, 4]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 			assert_eq!(<Candidates<Test>>::decode_len(), None);
 
 			assert_eq!(Elections::election_rounds(), 1);
@@ -2180,9 +2180,9 @@ mod tests {
 			System::set_block_number(5);
 			Elections::end_block(System::block_number());
 
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 			assert_eq!(Elections::election_rounds(), 1);
-			assert_eq!(Elections::members_ids(), vec![]);
+			assert!(Elections::members_ids().is_empty());
 
 			assert_eq!(
 				System::events().iter().last().unwrap().event,
@@ -2293,7 +2293,7 @@ mod tests {
 
 			System::set_block_number(10);
 			Elections::end_block(System::block_number());
-			assert_eq!(Elections::members_ids(), vec![]);
+			assert!(Elections::members_ids().is_empty());
 
 			assert_eq!(balances(&5), (47, 0));
 		});
@@ -2378,7 +2378,7 @@ mod tests {
 				assert_eq!(Elections::members(), vec![(4, 40), (5, 50)]);
 				assert_eq!(Elections::runners_up(), vec![(2, 20), (3, 30)]);
 				// no new candidates but old members and runners-up are always added.
-				assert_eq!(Elections::candidates(), vec![]);
+				assert!(Elections::candidates().is_empty());
 				assert_eq!(Elections::election_rounds(), b / 5);
 				assert_eq_uvec!(all_voters(), vec![2, 3, 4, 5]);
 			};
@@ -2490,7 +2490,7 @@ mod tests {
 			// meanwhile, no one cares to become a candidate again.
 			System::set_block_number(10);
 			Elections::end_block(System::block_number());
-			assert_eq!(Elections::members_ids(), vec![]);
+			assert!(Elections::members_ids().is_empty());
 			assert_eq!(Elections::election_rounds(), 2);
 		});
 	}
@@ -2658,14 +2658,14 @@ mod tests {
 			Elections::end_block(System::block_number());
 
 			assert_eq!(Elections::members_ids(), vec![4, 5]);
-			assert_eq!(Elections::runners_up_ids(), vec![]);
+			assert!(Elections::runners_up_ids().is_empty());
 
 			assert_ok!(Elections::renounce_candidacy(Origin::signed(4), Renouncing::Member));
 			assert_eq!(balances(&4), (38, 2)); // 2 is voting bond.
 
 			// no replacement
 			assert_eq!(Elections::members_ids(), vec![5]);
-			assert_eq!(Elections::runners_up_ids(), vec![]);
+			assert!(Elections::runners_up_ids().is_empty());
 		})
 	}
 
@@ -2729,7 +2729,7 @@ mod tests {
 
 			assert_ok!(Elections::renounce_candidacy(Origin::signed(5), Renouncing::Candidate(1)));
 			assert_eq!(balances(&5), (50, 0));
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 		})
 	}
 
@@ -2841,7 +2841,7 @@ mod tests {
 
 			assert_eq!(Elections::members_ids(), vec![1, 4]);
 			assert_eq!(Elections::runners_up_ids(), vec![2, 3]);
-			assert_eq!(Elections::candidates(), vec![]);
+			assert!(Elections::candidates().is_empty());
 		})
 	}
 }
diff --git a/frame/elections/src/tests.rs b/frame/elections/src/tests.rs
index 247b6272524b1..92f6e11252b05 100644
--- a/frame/elections/src/tests.rs
+++ b/frame/elections/src/tests.rs
@@ -46,14 +46,14 @@ fn params_should_work() {
 
 		assert_eq!(Elections::voters(0), Vec::<Option<u64>>::new());
 		assert_eq!(Elections::voter_info(1), None);
-		assert_eq!(Elections::all_approvals_of(&1), vec![]);
+		assert!(Elections::all_approvals_of(&1).is_empty());
 	});
 }
 
 #[test]
 fn chunking_bool_to_flag_should_work() {
 	ExtBuilder::default().build().execute_with(|| {
-		assert_eq!(Elections::bool_to_flag(vec![]), vec![]);
+		assert!(Elections::bool_to_flag(vec![]).is_empty());
 		assert_eq!(Elections::bool_to_flag(vec![false]), vec![0]);
 		assert_eq!(Elections::bool_to_flag(vec![true]), vec![1]);
 		assert_eq!(Elections::bool_to_flag(vec![true, true, true, true]), vec![15]);
@@ -274,11 +274,11 @@ fn chunking_approval_storage_should_work() {
 
 		assert_eq!(Elections::all_approvals_of(&2), vec![true]);
 		// NOTE: these two are stored in mem differently though.
-		assert_eq!(Elections::all_approvals_of(&3), vec![]);
-		assert_eq!(Elections::all_approvals_of(&4), vec![]);
+		assert!(Elections::all_approvals_of(&3).is_empty());
+		assert!(Elections::all_approvals_of(&4).is_empty());
 
 		assert_eq!(Elections::approvals_of((3, 0)), vec![0]);
-		assert_eq!(Elections::approvals_of((4, 0)), vec![]);
+		assert!(Elections::approvals_of((4, 0)).is_empty());
 	});
 }
 
@@ -385,7 +385,7 @@ fn voting_locking_stake_and_reserving_bond_works() {
 		assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0));
 
 		assert_eq!(balances(&2), (20, 0));
-		assert_eq!(locks(&2), vec![]);
+		assert!(locks(&2).is_empty());
 		assert_ok!(Elections::set_approvals(Origin::signed(2), vec![], 0, 0, 15));
 		assert_eq!(balances(&2), (18, 2));
 		assert_eq!(locks(&2), vec![15]);
@@ -401,7 +401,7 @@ fn voting_locking_stake_and_reserving_bond_works() {
 		assert_ok!(Elections::retract_voter(Origin::signed(2), 0));
 
 		assert_eq!(balances(&2), (102, 0));
-		assert_eq!(locks(&2), vec![]);
+		assert!(locks(&2).is_empty());
 	});
 }
 
diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs
index e2b00daf31cd7..0585307061bcd 100644
--- a/frame/example/src/lib.rs
+++ b/frame/example/src/lib.rs
@@ -843,7 +843,7 @@ mod tests {
 				WatchDummy::<Test>(PhantomData).validate(&1, &call, &info, 150)
 					.unwrap()
 					.priority,
-				Bounded::max_value(),
+				u64::max_value(),
 			);
 			assert_eq!(
 				WatchDummy::<Test>(PhantomData).validate(&1, &call, &info, 250),
diff --git a/frame/scored-pool/src/tests.rs b/frame/scored-pool/src/tests.rs
index 9c0074ff6e689..44b71bc00ba47 100644
--- a/frame/scored-pool/src/tests.rs
+++ b/frame/scored-pool/src/tests.rs
@@ -153,7 +153,7 @@ fn unscored_entities_must_not_be_used_for_filling_members() {
 
 		// then
 		// the `None` candidates should not have been filled in
-		assert_eq!(ScoredPool::members(), vec![]);
+		assert!(ScoredPool::members().is_empty());
 		assert_eq!(MEMBERS.with(|m| m.borrow().clone()), ScoredPool::members());
 	});
 }
diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml
index 2d1487afb03df..bd64dbcb1d3e0 100644
--- a/frame/staking/Cargo.toml
+++ b/frame/staking/Cargo.toml
@@ -33,6 +33,7 @@ rand_chacha = { version = "0.2", default-features = false, optional = true }
 [dev-dependencies]
 sp-core = { version = "2.0.0-rc6", path = "../../primitives/core" }
 sp-storage = { version = "2.0.0-rc6", path = "../../primitives/storage" }
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 pallet-balances = { version = "2.0.0-rc6", path = "../balances" }
 pallet-timestamp = { version = "2.0.0-rc6", path = "../timestamp" }
 pallet-staking-reward-curve = { version = "2.0.0-rc6",  path = "../staking/reward-curve" }
@@ -40,7 +41,6 @@ substrate-test-utils = { version = "2.0.0-rc6", path = "../../test-utils" }
 frame-benchmarking = { version = "2.0.0-rc6", path = "../benchmarking" }
 rand_chacha = { version = "0.2" }
 parking_lot = "0.10.2"
-env_logger = "0.7.1"
 hex = "0.4"
 
 [features]
diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs
index 31e41e21360a0..805df5d56f38e 100644
--- a/frame/staking/src/mock.rs
+++ b/frame/staking/src/mock.rs
@@ -452,7 +452,7 @@ impl ExtBuilder {
 		MAX_ITERATIONS.with(|v| *v.borrow_mut() = self.max_offchain_iterations);
 	}
 	pub fn build(self) -> sp_io::TestExternalities {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 		self.set_associated_constants();
 		let mut storage = frame_system::GenesisConfig::default()
 			.build_storage::<Test>()
diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs
index 0f5d08a3a8ccc..3feebfbc8ace9 100644
--- a/frame/staking/src/tests.rs
+++ b/frame/staking/src/tests.rs
@@ -1734,7 +1734,7 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election() {
 					.collect::<Vec<_>>(),
 				vec![(31, 1000), (21, 1000), (11, 1000)],
 			);
-			assert_eq!(<Nominators<Test>>::iter().map(|(n, _)| n).collect::<Vec<_>>(), vec![]);
+			assert!(<Nominators<Test>>::iter().map(|(n, _)| n).collect::<Vec<_>>().is_empty());
 
 			// give the man some money
 			let initial_balance = 1000;
@@ -1782,7 +1782,7 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election_elected() {
 					.collect::<Vec<_>>(),
 				vec![(31, 100), (21, 1000), (11, 1000)],
 			);
-			assert_eq!(<Nominators<Test>>::iter().map(|(n, _)| n).collect::<Vec<_>>(), vec![]);
+			assert!(<Nominators<Test>>::iter().map(|(n, _)| n).collect::<Vec<_>>().is_empty());
 
 			// give the man some money
 			let initial_balance = 1000;
diff --git a/frame/sudo/src/tests.rs b/frame/sudo/src/tests.rs
index 79424d2824f90..cba1e1cf60540 100644
--- a/frame/sudo/src/tests.rs
+++ b/frame/sudo/src/tests.rs
@@ -18,9 +18,9 @@
 //! Tests for the module.
 
 use super::*;
-use mock::{ 
+use mock::{
 	Sudo, SudoCall, Origin, Call, Test, new_test_ext, LoggerCall, Logger, System, TestEvent,
-}; 
+};
 use frame_support::{assert_ok, assert_noop};
 
 #[test]
@@ -28,8 +28,8 @@ fn test_setup_works() {
 	// Environment setup, logger storage, and sudo `key` retrieval should work as expected.
 	new_test_ext(1).execute_with(|| {
 		assert_eq!(Sudo::key(),  1u64);
-		assert_eq!(Logger::i32_log(), vec![]);
-		assert_eq!(Logger::account_log(), vec![]);
+		assert!(Logger::i32_log().is_empty());
+		assert!(Logger::account_log().is_empty());
 	});
 }
 
@@ -40,8 +40,8 @@ fn sudo_basics() {
 		// A privileged function should work when `sudo` is passed the root `key` as `origin`.
 		let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log(42, 1_000)));
 		assert_ok!(Sudo::sudo(Origin::signed(1), call));
-		assert_eq!(Logger::i32_log(), vec![42i32]); 
-		
+		assert_eq!(Logger::i32_log(), vec![42i32]);
+
 		// A privileged function should not work when `sudo` is passed a non-root `key` as `origin`.
 		let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log(42, 1_000)));
 		assert_noop!(Sudo::sudo(Origin::signed(2), call), Error::<Test>::RequireSudo);
@@ -58,7 +58,7 @@ fn sudo_emits_events_correctly() {
 		let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log(42, 1)));
 		assert_ok!(Sudo::sudo(Origin::signed(1), call));
 		let expected_event = TestEvent::sudo(RawEvent::Sudid(Ok(())));
-		assert!(System::events().iter().any(|a| a.event == expected_event)); 
+		assert!(System::events().iter().any(|a| a.event == expected_event));
 	})
 }
 
@@ -68,16 +68,16 @@ fn sudo_unchecked_weight_basics() {
 		// A privileged function should work when `sudo` is passed the root `key` as origin.
 		let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log(42, 1_000)));
 		assert_ok!(Sudo::sudo_unchecked_weight(Origin::signed(1), call, 1_000));
-		assert_eq!(Logger::i32_log(), vec![42i32]); 
+		assert_eq!(Logger::i32_log(), vec![42i32]);
 
 		// A privileged function should not work when called with a non-root `key`.
 		let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log(42, 1_000)));
 		assert_noop!(
-			Sudo::sudo_unchecked_weight(Origin::signed(2), call, 1_000), 
+			Sudo::sudo_unchecked_weight(Origin::signed(2), call, 1_000),
 			Error::<Test>::RequireSudo,
 		);
 		// `I32Log` is unchanged after unsuccessful call.
-		assert_eq!(Logger::i32_log(), vec![42i32]); 
+		assert_eq!(Logger::i32_log(), vec![42i32]);
 
 		// Controls the dispatched weight.
 		let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log(42, 1)));
@@ -103,7 +103,7 @@ fn sudo_unchecked_weight_emits_events_correctly() {
 
 #[test]
 fn set_key_basics() {
-	new_test_ext(1).execute_with(|| {	
+	new_test_ext(1).execute_with(|| {
 		// A root `key` can change the root `key`
 		assert_ok!(Sudo::set_key(Origin::signed(1), 2));
 		assert_eq!(Sudo::key(),  2u64);
@@ -117,7 +117,7 @@ fn set_key_basics() {
 
 #[test]
 fn set_key_emits_events_correctly() {
-	new_test_ext(1).execute_with(|| {	
+	new_test_ext(1).execute_with(|| {
 		// Set block number to 1 because events are not emitted on block 0.
 		System::set_block_number(1);
 
@@ -138,8 +138,8 @@ fn sudo_as_basics() {
 		// A privileged function will not work when passed to `sudo_as`.
 		let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log(42, 1_000)));
 		assert_ok!(Sudo::sudo_as(Origin::signed(1), 2, call));
-		assert_eq!(Logger::i32_log(), vec![]); 
-		assert_eq!(Logger::account_log(), vec![]);
+		assert!(Logger::i32_log().is_empty());
+		assert!(Logger::account_log().is_empty());
 
 		// A non-privileged function should not work when called with a non-root `key`.
 		let call = Box::new(Call::Logger(LoggerCall::non_privileged_log(42, 1)));
@@ -156,7 +156,7 @@ fn sudo_as_basics() {
 
 #[test]
 fn sudo_as_emits_events_correctly() {
-		new_test_ext(1).execute_with(|| {	
+		new_test_ext(1).execute_with(|| {
 		// Set block number to 1 because events are not emitted on block 0.
 		System::set_block_number(1);
 
diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs
index 717a9a29ad5f5..5ee144c79c4db 100644
--- a/frame/support/src/storage/mod.rs
+++ b/frame/support/src/storage/mod.rs
@@ -656,7 +656,7 @@ mod test {
 			assert_eq!(MyStorage::final_prefix().to_vec(), k);
 
 			// test iteration
-			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![]);
+			assert!(MyStorage::iter_values().collect::<Vec<_>>().is_empty());
 
 			unhashed::put(&[&k[..], &vec![1][..]].concat(), &1u64);
 			unhashed::put(&[&k[..], &vec![1, 1][..]].concat(), &2u64);
@@ -667,13 +667,13 @@ mod test {
 
 			// test removal
 			MyStorage::remove_all();
-			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![]);
+			assert!(MyStorage::iter_values().collect::<Vec<_>>().is_empty());
 
 			// test migration
 			unhashed::put(&[&k[..], &vec![1][..]].concat(), &1u32);
 			unhashed::put(&[&k[..], &vec![8][..]].concat(), &2u32);
 
-			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![]);
+			assert!(MyStorage::iter_values().collect::<Vec<_>>().is_empty());
 			MyStorage::translate_values(|v: u32| Some(v as u64));
 			assert_eq!(MyStorage::iter_values().collect::<Vec<_>>(), vec![1, 2]);
 			MyStorage::remove_all();
diff --git a/frame/support/test/tests/decl_storage.rs b/frame/support/test/tests/decl_storage.rs
index 9bdc4226263df..800ce459fed35 100644
--- a/frame/support/test/tests/decl_storage.rs
+++ b/frame/support/test/tests/decl_storage.rs
@@ -520,7 +520,7 @@ mod test_append_and_len {
 	fn default_for_option() {
 		TestExternalities::default().execute_with(|| {
 			assert_eq!(OptionVec::get(), None);
-			assert_eq!(JustVec::get(), vec![]);
+			assert!(JustVec::get().is_empty());
 		});
 	}
 
@@ -553,7 +553,7 @@ mod test_append_and_len {
 			let key = JustVec::hashed_key();
 			// Set it to some invalid value.
 			frame_support::storage::unhashed::put_raw(&key, &*b"1");
-			assert_eq!(JustVec::get(), Vec::new());
+			assert!(JustVec::get().is_empty());
 			assert_eq!(frame_support::storage::unhashed::get_raw(&key), Some(b"1".to_vec()));
 
 			JustVec::append(1);
@@ -600,7 +600,7 @@ mod test_append_and_len {
 	fn len_works_ignores_default_assignment() {
 		TestExternalities::default().execute_with(|| {
 			// vec
-			assert_eq!(JustVec::get(), vec![]);
+			assert!(JustVec::get().is_empty());
 			assert_eq!(JustVec::decode_len(), None);
 
 			assert_eq!(JustVecWithDefault::get(), vec![6, 9]);
@@ -610,7 +610,7 @@ mod test_append_and_len {
 			assert_eq!(OptionVec::decode_len(), None);
 
 			// map
-			assert_eq!(MapVec::get(0), vec![]);
+			assert!(MapVec::get(0).is_empty());
 			assert_eq!(MapVec::decode_len(0), None);
 
 			assert_eq!(MapVecWithDefault::get(0), vec![6, 9]);
@@ -620,7 +620,7 @@ mod test_append_and_len {
 			assert_eq!(OptionMapVec::decode_len(0), None);
 
 			// Double map
-			assert_eq!(DoubleMapVec::get(0, 0), vec![]);
+			assert!(DoubleMapVec::get(0, 0).is_empty());
 			assert_eq!(DoubleMapVec::decode_len(0, 1), None);
 
 			assert_eq!(DoubleMapVecWithDefault::get(0, 0), vec![6, 9]);
diff --git a/primitives/consensus/slots/Cargo.toml b/primitives/consensus/slots/Cargo.toml
index ada913b645c7b..fecd3e03d781b 100644
--- a/primitives/consensus/slots/Cargo.toml
+++ b/primitives/consensus/slots/Cargo.toml
@@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
-sp-runtime = { version = "2.0.0-rc2", default-features = false, path = "../../runtime" }
+sp-runtime = { version = "2.0.0-rc6", default-features = false, path = "../../runtime" }
 
 [features]
 default = ["std"]
diff --git a/primitives/tracing/Cargo.toml b/primitives/tracing/Cargo.toml
index 136039475673a..889d116221b5f 100644
--- a/primitives/tracing/Cargo.toml
+++ b/primitives/tracing/Cargo.toml
@@ -15,7 +15,8 @@ targets = ["x86_64-unknown-linux-gnu"]
 tracing = { version = "0.1.18", optional = true }
 rental = { version = "0.5.5", optional = true }
 log = { version = "0.4.8", optional = true }
+tracing-subscriber = { version = "0.2.10", optional = true, features = ["tracing-log"] }
 
 [features]
 default = [ "std" ]
-std = [ "tracing", "rental", "log" ]
+std = [ "tracing", "rental", "log", "tracing-subscriber" ]
diff --git a/primitives/tracing/src/lib.rs b/primitives/tracing/src/lib.rs
index e82d8861cd3f5..ec692b90dfdec 100644
--- a/primitives/tracing/src/lib.rs
+++ b/primitives/tracing/src/lib.rs
@@ -44,6 +44,13 @@ pub mod proxy;
 #[cfg(feature = "std")]
 use std::sync::atomic::{AtomicBool, Ordering};
 
+/// Try to init a simple tracing subscriber with log compatibility layer.
+/// Ignores any error. Useful for testing.
+#[cfg(feature = "std")]
+pub fn try_init_simple() {
+	let _ = tracing_subscriber::fmt().with_writer(std::io::stderr).try_init();
+}
+
 /// Flag to signal whether to run wasm tracing
 #[cfg(feature = "std")]
 static WASM_TRACING_ENABLED: AtomicBool = AtomicBool::new(false);
@@ -114,4 +121,4 @@ pub fn wasm_tracing_enabled() -> bool {
 #[cfg(feature = "std")]
 pub fn set_wasm_tracing(b: bool) {
 	WASM_TRACING_ENABLED.store(b, Ordering::Relaxed)
-}
\ No newline at end of file
+}
diff --git a/utils/frame/rpc/system/Cargo.toml b/utils/frame/rpc/system/Cargo.toml
index 0f1e27efc7037..fc4774738b811 100644
--- a/utils/frame/rpc/system/Cargo.toml
+++ b/utils/frame/rpc/system/Cargo.toml
@@ -31,5 +31,5 @@ sc-rpc-api = { version = "0.8.0-rc6", path = "../../../../client/rpc-api" }
 
 [dev-dependencies]
 substrate-test-runtime-client = { version = "2.0.0-rc6", path = "../../../../test-utils/runtime/client" }
-env_logger = "0.7.0"
+sp-tracing = { version = "2.0.0-rc6", path = "../../../../primitives/tracing" }
 sc-transaction-pool = { version = "2.0.0-rc6", path = "../../../../client/transaction-pool" }
diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs
index 2bb46369fea54..cefe39534a167 100644
--- a/utils/frame/rpc/system/src/lib.rs
+++ b/utils/frame/rpc/system/src/lib.rs
@@ -294,7 +294,7 @@ mod tests {
 
 	#[test]
 	fn should_return_next_nonce_for_some_account() {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 
 		// given
 		let client = Arc::new(substrate_test_runtime_client::new());
@@ -333,7 +333,7 @@ mod tests {
 
 	#[test]
 	fn dry_run_should_deny_unsafe() {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 
 		// given
 		let client = Arc::new(substrate_test_runtime_client::new());
@@ -356,7 +356,7 @@ mod tests {
 
 	#[test]
 	fn dry_run_should_work() {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 
 		// given
 		let client = Arc::new(substrate_test_runtime_client::new());
@@ -388,7 +388,7 @@ mod tests {
 
 	#[test]
 	fn dry_run_should_indicate_error() {
-		let _ = env_logger::try_init();
+		sp_tracing::try_init_simple();
 
 		// given
 		let client = Arc::new(substrate_test_runtime_client::new());

From 86026fcde6ea0d41ea93a80e37eaebe85a2dc1f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bruno=20=C5=A0kvorc?= <bruno@skvorc.me>
Date: Thu, 17 Sep 2020 11:12:14 +0200
Subject: [PATCH 097/122] Typo in error text (#7126)

---
 primitives/runtime/src/transaction_validity.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/primitives/runtime/src/transaction_validity.rs b/primitives/runtime/src/transaction_validity.rs
index 1aad9e75aec34..e9e2f2b3d3c2b 100644
--- a/primitives/runtime/src/transaction_validity.rs
+++ b/primitives/runtime/src/transaction_validity.rs
@@ -98,7 +98,7 @@ impl From<InvalidTransaction> for &'static str {
 			InvalidTransaction::BadProof => "Transaction has a bad signature",
 			InvalidTransaction::AncientBirthBlock => "Transaction has an ancient birth block",
 			InvalidTransaction::ExhaustsResources =>
-				"Transaction would exhausts the block limits",
+				"Transaction would exhaust the block limits",
 			InvalidTransaction::Payment =>
 				"Inability to pay some fees (e.g. account balance too low)",
 			InvalidTransaction::BadMandatory =>

From da02c0b494a4bd1778d0328437e0a149eb5390dd Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Thu, 17 Sep 2020 15:40:51 +0200
Subject: [PATCH 098/122] WeightInfo for ImOnline (#7128)

* Add WeightInfo, not final weights

* benchmark machine weights
---
 bin/node/runtime/src/lib.rs                   |  2 +-
 bin/node/runtime/src/weights/mod.rs           |  1 +
 .../runtime/src/weights/pallet_im_online.rs   | 34 +++++++++++++++++++
 frame/im-online/src/benchmarking.rs           |  2 ++
 frame/im-online/src/default_weight.rs         | 33 ++++++++++++++++++
 frame/im-online/src/lib.rs                    | 28 +++++----------
 6 files changed, 80 insertions(+), 20 deletions(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_im_online.rs
 create mode 100644 frame/im-online/src/default_weight.rs

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index 5737fcfd2e2f1..b543227200750 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -736,7 +736,7 @@ impl pallet_im_online::Trait for Runtime {
 	type SessionDuration = SessionDuration;
 	type ReportUnresponsiveness = Offences;
 	type UnsignedPriority = ImOnlineUnsignedPriority;
-	type WeightInfo = ();
+	type WeightInfo = weights::pallet_im_online::WeightInfo;
 }
 
 parameter_types! {
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index 86cab773b18e3..7fbef9c0953d1 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -19,6 +19,7 @@ pub mod frame_system;
 pub mod pallet_balances;
 pub mod pallet_collective;
 pub mod pallet_democracy;
+pub mod pallet_im_online;
 pub mod pallet_proxy;
 pub mod pallet_timestamp;
 pub mod pallet_utility;
diff --git a/bin/node/runtime/src/weights/pallet_im_online.rs b/bin/node/runtime/src/weights/pallet_im_online.rs
new file mode 100644
index 0000000000000..25daff6a6e18f
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_im_online.rs
@@ -0,0 +1,34 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_im_online::WeightInfo for WeightInfo {
+	fn validate_unsigned_and_then_heartbeat(k: u32, e: u32, ) -> Weight {
+		(139830000 as Weight)
+			.saturating_add((211000 as Weight).saturating_mul(k as Weight))
+			.saturating_add((654000 as Weight).saturating_mul(e as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+}
diff --git a/frame/im-online/src/benchmarking.rs b/frame/im-online/src/benchmarking.rs
index 55f294505602e..b92be023ce480 100644
--- a/frame/im-online/src/benchmarking.rs
+++ b/frame/im-online/src/benchmarking.rs
@@ -65,12 +65,14 @@ pub fn create_heartbeat<T: Trait>(k: u32, e: u32) ->
 benchmarks! {
 	_{ }
 
+	#[extra]
 	heartbeat {
 		let k in 1 .. MAX_KEYS;
 		let e in 1 .. MAX_EXTERNAL_ADDRESSES;
 		let (input_heartbeat, signature) = create_heartbeat::<T>(k, e)?;
 	}: _(RawOrigin::None, input_heartbeat, signature)
 
+	#[extra]
 	validate_unsigned {
 		let k in 1 .. MAX_KEYS;
 		let e in 1 .. MAX_EXTERNAL_ADDRESSES;
diff --git a/frame/im-online/src/default_weight.rs b/frame/im-online/src/default_weight.rs
new file mode 100644
index 0000000000000..e6efb42f2e3d8
--- /dev/null
+++ b/frame/im-online/src/default_weight.rs
@@ -0,0 +1,33 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn validate_unsigned_and_then_heartbeat(k: u32, e: u32, ) -> Weight {
+		(139830000 as Weight)
+			.saturating_add((211000 as Weight).saturating_mul(k as Weight))
+			.saturating_add((654000 as Weight).saturating_mul(e as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+}
diff --git a/frame/im-online/src/lib.rs b/frame/im-online/src/lib.rs
index 7856ecfd5aa46..ef9c6b9182af6 100644
--- a/frame/im-online/src/lib.rs
+++ b/frame/im-online/src/lib.rs
@@ -72,6 +72,7 @@
 mod mock;
 mod tests;
 mod benchmarking;
+mod default_weight;
 
 use sp_application_crypto::RuntimeAppPublic;
 use codec::{Encode, Decode};
@@ -227,17 +228,9 @@ pub struct Heartbeat<BlockNumber>
 }
 
 pub trait WeightInfo {
-	fn heartbeat(k: u32, e: u32, ) -> Weight;
-	fn validate_unsigned(k: u32, e: u32, ) -> Weight;
 	fn validate_unsigned_and_then_heartbeat(k: u32, e: u32, ) -> Weight;
 }
 
-impl WeightInfo for () {
-	fn heartbeat(_k: u32, _e: u32, ) -> Weight { 1_000_000_000 }
-	fn validate_unsigned(_k: u32, _e: u32, ) -> Weight { 1_000_000_000 }
-	fn validate_unsigned_and_then_heartbeat(_k: u32, _e: u32, ) -> Weight { 1_000_000_000 }
-}
-
 pub trait Trait: SendTransactionTypes<Call<Self>> + pallet_session::historical::Trait {
 	/// The identifier type for an authority.
 	type AuthorityId: Member + Parameter + RuntimeAppPublic + Default + Ord;
@@ -333,23 +326,20 @@ decl_module! {
 		fn deposit_event() = default;
 
 		/// # <weight>
-		/// - Complexity: `O(K + E)` where K is length of `Keys` and E is length of
-		///   `Heartbeat.network_state.external_address`
-		///
+		/// - Complexity: `O(K + E)` where K is length of `Keys` (heartbeat.validators_len)
+		///   and E is length of `heartbeat.network_state.external_address`
 		///   - `O(K)`: decoding of length `K`
 		///   - `O(E)`: decoding/encoding of length `E`
 		/// - DbReads: pallet_session `Validators`, pallet_session `CurrentIndex`, `Keys`,
 		///   `ReceivedHeartbeats`
 		/// - DbWrites: `ReceivedHeartbeats`
 		/// # </weight>
-		// NOTE: the weight include cost of validate_unsigned as it is part of the cost to import
-		// block with such an extrinsic.
-		#[weight = (310_000_000 + T::DbWeight::get().reads_writes(4, 1))
-			.saturating_add(750_000.saturating_mul(heartbeat.validators_len as Weight))
-			.saturating_add(
-				1_200_000.saturating_mul(heartbeat.network_state.external_addresses.len() as Weight)
-			)
-		]
+		// NOTE: the weight includes the cost of validate_unsigned as it is part of the cost to
+		// import block with such an extrinsic.
+		#[weight = <T as Trait>::WeightInfo::validate_unsigned_and_then_heartbeat(
+			heartbeat.validators_len as u32,
+			heartbeat.network_state.external_addresses.len() as u32,
+		)]
 		fn heartbeat(
 			origin,
 			heartbeat: Heartbeat<T::BlockNumber>,

From f1380be67fef4ee931ef27e7c125c54c82aad95b Mon Sep 17 00:00:00 2001
From: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Date: Thu, 17 Sep 2020 16:06:00 +0200
Subject: [PATCH 099/122] fix the new staking weight in substrate-node (#7131)

---
 bin/node/runtime/src/lib.rs                   |   2 +-
 bin/node/runtime/src/weights/mod.rs           |   1 +
 .../runtime/src/weights/pallet_staking.rs     | 170 ++++++++++++++++++
 3 files changed, 172 insertions(+), 1 deletion(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_staking.rs

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index b543227200750..104cd2ba96cab 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -458,7 +458,7 @@ impl pallet_staking::Trait for Runtime {
 	type MinSolutionScoreBump = MinSolutionScoreBump;
 	type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator;
 	type UnsignedPriority = StakingUnsignedPriority;
-	type WeightInfo = ();
+	type WeightInfo = weights::pallet_staking::WeightInfo;
 }
 
 parameter_types! {
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index 7fbef9c0953d1..81347ee7ce529 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -24,3 +24,4 @@ pub mod pallet_proxy;
 pub mod pallet_timestamp;
 pub mod pallet_utility;
 pub mod pallet_vesting;
+pub mod pallet_staking;
diff --git a/bin/node/runtime/src/weights/pallet_staking.rs b/bin/node/runtime/src/weights/pallet_staking.rs
new file mode 100644
index 0000000000000..f5a70830b92e7
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_staking.rs
@@ -0,0 +1,170 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Default weights of pallet-staking.
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_staking::WeightInfo for WeightInfo {
+	fn bond() -> Weight {
+		(144278000 as Weight)
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn bond_extra() -> Weight {
+		(110715000 as Weight)
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn unbond() -> Weight {
+		(99840000 as Weight)
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn withdraw_unbonded_update(s: u32, ) -> Weight {
+		(100728000 as Weight)
+			.saturating_add((63000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(5 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn withdraw_unbonded_kill(s: u32, ) -> Weight {
+		(168879000 as Weight)
+			.saturating_add((6666000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(7 as Weight))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn validate() -> Weight {
+		(35539000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn nominate(n: u32, ) -> Weight {
+		(48596000 as Weight)
+			.saturating_add((308000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn chill() -> Weight {
+		(35144000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn set_payee() -> Weight {
+		(24255000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_controller() -> Weight {
+		(52294000 as Weight)
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn set_validator_count() -> Weight {
+		(5185000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_no_eras() -> Weight {
+		(5907000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_new_era() -> Weight {
+		(5917000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_new_era_always() -> Weight {
+		(5952000 as Weight)
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_invulnerables(v: u32, ) -> Weight {
+		(6324000 as Weight)
+			.saturating_add((9000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_unstake(s: u32, ) -> Weight {
+		(119691000 as Weight)
+			.saturating_add((6681000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn cancel_deferred_slash(s: u32, ) -> Weight {
+		(5820201000 as Weight)
+			.saturating_add((34672000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn payout_stakers_dead_controller(n: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((92486000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight)))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(n as Weight)))
+	}
+	fn payout_stakers_alive_staked(n: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((117324000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight)))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(n as Weight)))
+	}
+	fn rebond(l: u32, ) -> Weight {
+		(71316000 as Weight)
+			.saturating_add((142000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn set_history_depth(e: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((51901000 as Weight).saturating_mul(e as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+			.saturating_add(DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight)))
+	}
+	fn reap_stash(s: u32, ) -> Weight {
+		(147166000 as Weight)
+			.saturating_add((6661000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn new_era(v: u32, n: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((1440459000 as Weight).saturating_mul(v as Weight))
+			.saturating_add((182580000 as Weight).saturating_mul(n as Weight))
+			.saturating_add(DbWeight::get().reads(10 as Weight))
+			.saturating_add(DbWeight::get().reads((4 as Weight).saturating_mul(v as Weight)))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight)))
+			.saturating_add(DbWeight::get().writes(8 as Weight))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight)))
+	}
+	fn submit_solution_better(v: u32, n: u32, a: u32, w: u32, ) -> Weight {
+		(0 as Weight)
+			.saturating_add((964000 as Weight).saturating_mul(v as Weight))
+			.saturating_add((432000 as Weight).saturating_mul(n as Weight))
+			.saturating_add((204294000 as Weight).saturating_mul(a as Weight))
+			.saturating_add((9546000 as Weight).saturating_mul(w as Weight))
+			.saturating_add(DbWeight::get().reads(6 as Weight))
+			.saturating_add(DbWeight::get().reads((4 as Weight).saturating_mul(a as Weight)))
+			.saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(w as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+}

From 585629a4379764ecbca95762335ce14c9873ad04 Mon Sep 17 00:00:00 2001
From: Pierre Krieger <pierre.krieger1708@gmail.com>
Date: Thu, 17 Sep 2020 20:31:53 +0200
Subject: [PATCH 100/122] Remove warning about deprecated PeerIds (#7132)

---
 client/network/src/config.rs | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/client/network/src/config.rs b/client/network/src/config.rs
index 4949af031f085..7445ea0534bb7 100644
--- a/client/network/src/config.rs
+++ b/client/network/src/config.rs
@@ -281,21 +281,8 @@ pub fn parse_str_addr(addr_str: &str) -> Result<(PeerId, Multiaddr), ParseErr> {
 /// Splits a Multiaddress into a Multiaddress and PeerId.
 pub fn parse_addr(mut addr: Multiaddr)-> Result<(PeerId, Multiaddr), ParseErr> {
 	let who = match addr.pop() {
-		Some(multiaddr::Protocol::P2p(key)) => {
-			if !matches!(key.algorithm(), multiaddr::multihash::Code::Identity) {
-				// (note: this is the "person bowing" emoji)
-				log::warn!(
-					"🙇 You are using the peer ID {}. This peer ID uses a legacy, deprecated \
-					representation that will no longer be supported in the future. \
-					Please refresh it by performing a RPC query to the appropriate node, \
-					by looking at its logs, or by using `subkey inspect-node-key` on its \
-					private key.",
-					bs58::encode(key.as_bytes()).into_string()
-				);
-			}
-
-			PeerId::from_multihash(key).map_err(|_| ParseErr::InvalidPeerId)?
-		},
+		Some(multiaddr::Protocol::P2p(key)) => PeerId::from_multihash(key)
+			.map_err(|_| ParseErr::InvalidPeerId)?,
 		_ => return Err(ParseErr::PeerIdMissing),
 	};
 

From 9bf2f2f643fec3d72f208d517e8f7f81138380ad Mon Sep 17 00:00:00 2001
From: Arkadiy Paronyan <arkady.paronyan@gmail.com>
Date: Thu, 17 Sep 2020 20:32:26 +0200
Subject: [PATCH 101/122] Fix db initialization for light client (#7130)

* Fix db initialization for light client

* Fix cache distribution
---
 client/db/src/parity_db.rs | 18 ++++++++-----
 client/db/src/utils.rs     | 53 ++++++++++++++++++++++++--------------
 2 files changed, 45 insertions(+), 26 deletions(-)

diff --git a/client/db/src/parity_db.rs b/client/db/src/parity_db.rs
index 7085aa3bf8ccd..313069706f33f 100644
--- a/client/db/src/parity_db.rs
+++ b/client/db/src/parity_db.rs
@@ -18,7 +18,7 @@
 /// A `Database` adapter for parity-db.
 
 use sp_database::{Database, Change, ColumnId, Transaction, error::DatabaseError};
-use crate::utils::NUM_COLUMNS;
+use crate::utils::{DatabaseType, NUM_COLUMNS};
 use crate::columns;
 
 struct DbAdapter(parity_db::Db);
@@ -32,13 +32,17 @@ fn handle_err<T>(result: parity_db::Result<T>) -> T {
 	}
 }
 
-/// Wrap RocksDb database into a trait object that implements `sp_database::Database`
-pub fn open<H: Clone>(path: &std::path::Path) -> parity_db::Result<std::sync::Arc<dyn Database<H>>> {
+/// Wrap parity-db database into a trait object that implements `sp_database::Database`
+pub fn open<H: Clone>(path: &std::path::Path, db_type: DatabaseType)
+	-> parity_db::Result<std::sync::Arc<dyn Database<H>>>
+{
 	let mut config = parity_db::Options::with_columns(path, NUM_COLUMNS as u8);
-	let mut state_col = &mut config.columns[columns::STATE as usize];
-	state_col.ref_counted = true;
-	state_col.preimage = true;
-	state_col.uniform = true;
+	if db_type == DatabaseType::Full {
+		let mut state_col = &mut config.columns[columns::STATE as usize];
+		state_col.ref_counted = true;
+		state_col.preimage = true;
+		state_col.uniform = true;
+	}
 	let db = parity_db::Db::open(&config)?;
 	Ok(std::sync::Arc::new(DbAdapter(db)))
 }
diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs
index 3ad6c421135d0..e999469c18ff0 100644
--- a/client/db/src/utils.rs
+++ b/client/db/src/utils.rs
@@ -227,31 +227,46 @@ pub fn open_database<Block: BlockT>(
 
 			// and now open database assuming that it has the latest version
 			let mut db_config = kvdb_rocksdb::DatabaseConfig::with_columns(NUM_COLUMNS);
-			let state_col_budget = (*cache_size as f64 * 0.9) as usize;
-			let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1);
-			let mut memory_budget = std::collections::HashMap::new();
 			let path = path.to_str()
 				.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
 
-			for i in 0..NUM_COLUMNS {
-				if i == crate::columns::STATE {
-					memory_budget.insert(i, state_col_budget);
-				} else {
-					memory_budget.insert(i, other_col_budget);
+			let mut memory_budget = std::collections::HashMap::new();
+			match db_type {
+				DatabaseType::Full => {
+					let state_col_budget = (*cache_size as f64 * 0.9) as usize;
+					let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1);
+
+					for i in 0..NUM_COLUMNS {
+						if i == crate::columns::STATE {
+							memory_budget.insert(i, state_col_budget);
+						} else {
+							memory_budget.insert(i, other_col_budget);
+						}
+					}
+					log::trace!(
+						target: "db",
+						"Open RocksDB database at {}, state column budget: {} MiB, others({}) column cache: {} MiB",
+						path,
+						state_col_budget,
+						NUM_COLUMNS,
+						other_col_budget,
+					);
+				},
+				DatabaseType::Light => {
+					let col_budget = cache_size / (NUM_COLUMNS as usize);
+					for i in 0..NUM_COLUMNS {
+						memory_budget.insert(i, col_budget);
+					}
+					log::trace!(
+						target: "db",
+						"Open RocksDB light database at {}, column cache: {} MiB",
+						path,
+						col_budget,
+					);
 				}
 			}
-
 			db_config.memory_budget = memory_budget;
 
-			log::trace!(
-				target: "db",
-				"Open RocksDB database at {}, state column budget: {} MiB, others({}) column cache: {} MiB",
-				path,
-				state_col_budget,
-				NUM_COLUMNS,
-				other_col_budget,
-			);
-
 			let db = kvdb_rocksdb::Database::open(&db_config, &path)
 				.map_err(|err| sp_blockchain::Error::Backend(format!("{}", err)))?;
 			sp_database::as_database(db)
@@ -262,7 +277,7 @@ pub fn open_database<Block: BlockT>(
 		},
 		#[cfg(feature = "with-parity-db")]
 		DatabaseSettingsSrc::ParityDb { path } => {
-			crate::parity_db::open(&path)
+			crate::parity_db::open(&path, db_type)
 				.map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))?
 		},
 		#[cfg(not(feature = "with-parity-db"))]

From fa2fbe73ec441d911e439f704d35acf0a7cde173 Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Thu, 17 Sep 2020 23:52:43 +0200
Subject: [PATCH 102/122] WeightInfo for Identity Pallet (#7107)

* update benchmarks

* add automated weights

* Update benchmarking.rs

* use underscores for file out

* update some weights

* more weights

* finish weights

* add basic verification to benchmarks

* patch benchmarks

* Update benchmarking.rs

* final weights

* update for new type

* add weightinfo to node
---
 bin/node/runtime/src/lib.rs                   |   2 +-
 bin/node/runtime/src/weights/mod.rs           |   3 +-
 .../runtime/src/weights/pallet_identity.rs    | 136 +++
 frame/identity/src/benchmarking.rs            | 268 +++---
 frame/identity/src/default_weights.rs         | 135 +++
 frame/identity/src/lib.rs                     | 782 ++----------------
 frame/identity/src/tests.rs                   | 472 +++++++++++
 utils/frame/benchmarking-cli/src/writer.rs    |  19 +-
 8 files changed, 1012 insertions(+), 805 deletions(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_identity.rs
 create mode 100644 frame/identity/src/default_weights.rs
 create mode 100644 frame/identity/src/tests.rs

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index 104cd2ba96cab..16fcc9f70bc16 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -803,7 +803,7 @@ impl pallet_identity::Trait for Runtime {
 	type Slashed = Treasury;
 	type ForceOrigin = EnsureRootOrHalfCouncil;
 	type RegistrarOrigin = EnsureRootOrHalfCouncil;
-	type WeightInfo = ();
+	type WeightInfo = weights::pallet_identity::WeightInfo;
 }
 
 parameter_types! {
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index 81347ee7ce529..fd6d3cab49e28 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -19,9 +19,10 @@ pub mod frame_system;
 pub mod pallet_balances;
 pub mod pallet_collective;
 pub mod pallet_democracy;
+pub mod pallet_identity;
 pub mod pallet_im_online;
 pub mod pallet_proxy;
+pub mod pallet_staking;
 pub mod pallet_timestamp;
 pub mod pallet_utility;
 pub mod pallet_vesting;
-pub mod pallet_staking;
diff --git a/bin/node/runtime/src/weights/pallet_identity.rs b/bin/node/runtime/src/weights/pallet_identity.rs
new file mode 100644
index 0000000000000..2995a7674f8ae
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_identity.rs
@@ -0,0 +1,136 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_identity::WeightInfo for WeightInfo {
+	fn add_registrar(r: u32, ) -> Weight {
+		(39_603_000 as Weight)
+			.saturating_add((418_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_identity(r: u32, x: u32, ) -> Weight {
+		(110_679_000 as Weight)
+			.saturating_add((389_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((2_985_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_subs_new(s: u32, ) -> Weight {
+		(78_697_000 as Weight)
+			.saturating_add((15_225_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight)))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn set_subs_old(p: u32, ) -> Weight {
+		(71_308_000 as Weight)
+			.saturating_add((5_772_000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight)))
+	}
+	fn clear_identity(r: u32, s: u32, x: u32, ) -> Weight {
+		(91_553_000 as Weight)
+			.saturating_add((284_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((5_749_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add((1_621_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn request_judgement(r: u32, x: u32, ) -> Weight {
+		(110_856_000 as Weight)
+			.saturating_add((496_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((3_221_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn cancel_request(r: u32, x: u32, ) -> Weight {
+		(96_857_000 as Weight)
+			.saturating_add((311_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((3_204_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_fee(r: u32, ) -> Weight {
+		(16_276_000 as Weight)
+			.saturating_add((381_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_account_id(r: u32, ) -> Weight {
+		(18_530_000 as Weight)
+			.saturating_add((391_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_fields(r: u32, ) -> Weight {
+		(16_359_000 as Weight)
+			.saturating_add((379_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn provide_judgement(r: u32, x: u32, ) -> Weight {
+		(72_869_000 as Weight)
+			.saturating_add((423_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((3_187_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn kill_identity(r: u32, s: u32, x: u32, ) -> Weight {
+		(123_199_000 as Weight)
+			.saturating_add((71_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((5_730_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add((2_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn add_sub(s: u32, ) -> Weight {
+		(110_070_000 as Weight)
+			.saturating_add((262_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn rename_sub(s: u32, ) -> Weight {
+		(37_130_000 as Weight)
+			.saturating_add((79_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn remove_sub(s: u32, ) -> Weight {
+		(103_295_000 as Weight)
+			.saturating_add((235_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn quit_sub(s: u32, ) -> Weight {
+		(65_716_000 as Weight)
+			.saturating_add((227_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+}
diff --git a/frame/identity/src/benchmarking.rs b/frame/identity/src/benchmarking.rs
index 8b0cb0c27cf0e..d39df27017b71 100644
--- a/frame/identity/src/benchmarking.rs
+++ b/frame/identity/src/benchmarking.rs
@@ -21,30 +21,34 @@
 
 use super::*;
 
-use frame_system::RawOrigin;
-use sp_io::hashing::blake2_256;
-use frame_benchmarking::benchmarks;
+use frame_system::{EventRecord, RawOrigin};
+use frame_benchmarking::{benchmarks, account, whitelisted_caller};
 use sp_runtime::traits::Bounded;
 
 use crate::Module as Identity;
 
-// Support Functions
-fn account<T: Trait>(name: &'static str, index: u32) -> T::AccountId {
-	let entropy = (name, index).using_encoded(blake2_256);
-	T::AccountId::decode(&mut &entropy[..]).unwrap_or_default()
+const SEED: u32 = 0;
+
+fn assert_last_event<T: Trait>(generic_event: <T as Trait>::Event) {
+	let events = frame_system::Module::<T>::events();
+	let system_event: <T as frame_system::Trait>::Event = generic_event.into();
+	// compare to the last event record
+	let EventRecord { event, .. } = &events[events.len() - 1];
+	assert_eq!(event, &system_event);
 }
 
 // Adds `r` registrars to the Identity Pallet. These registrars will have set fees and fields.
 fn add_registrars<T: Trait>(r: u32) -> Result<(), &'static str> {
 	for i in 0..r {
-		let _ = T::Currency::make_free_balance_be(&account::<T>("registrar", i), BalanceOf::<T>::max_value());
-		Identity::<T>::add_registrar(RawOrigin::Root.into(), account::<T>("registrar", i))?;
-		Identity::<T>::set_fee(RawOrigin::Signed(account::<T>("registrar", i)).into(), i.into(), 10.into())?;
+		let registrar: T::AccountId = account("registrar", i, SEED);
+		let _ = T::Currency::make_free_balance_be(&registrar, BalanceOf::<T>::max_value());
+		Identity::<T>::add_registrar(RawOrigin::Root.into(), registrar.clone())?;
+		Identity::<T>::set_fee(RawOrigin::Signed(registrar.clone()).into(), i.into(), 10.into())?;
 		let fields = IdentityFields(
 			IdentityField::Display | IdentityField::Legal | IdentityField::Web | IdentityField::Riot
 			| IdentityField::Email | IdentityField::PgpFingerprint | IdentityField::Image | IdentityField::Twitter
 		);
-		Identity::<T>::set_fields(RawOrigin::Signed(account::<T>("registrar", i)).into(), i.into(), fields)?;
+		Identity::<T>::set_fields(RawOrigin::Signed(registrar.clone()).into(), i.into(), fields)?;
 	}
 
 	assert_eq!(Registrars::<T>::get().len(), r as usize);
@@ -59,7 +63,7 @@ fn create_sub_accounts<T: Trait>(who: &T::AccountId, s: u32) -> Result<Vec<(T::A
 	let data = Data::Raw(vec![0; 32]);
 
 	for i in 0..s {
-		let sub_account = account::<T>("sub", i);
+		let sub_account = account("sub", i, SEED);
 		subs.push((sub_account, data.clone()));
 	}
 
@@ -110,13 +114,13 @@ benchmarks! {
 		let p in 1 .. T::MaxSubAccounts::get() => ();
 		let s in 1 .. T::MaxSubAccounts::get() => {
 			// Give them s many sub accounts
-			let caller = account::<T>("caller", 0);
+			let caller: T::AccountId = whitelisted_caller();
 			let _ = add_sub_accounts::<T>(&caller, s)?;
 		};
 		let x in 1 .. T::MaxAdditionalFields::get() => {
 			// Create their main identity with x additional fields
 			let info = create_identity_info::<T>(x);
-			let caller = account::<T>("caller", 0);
+			let caller: T::AccountId = whitelisted_caller();
 			let caller_origin = <T as frame_system::Trait>::Origin::from(RawOrigin::Signed(caller));
 			Identity::<T>::set_identity(caller_origin, info)?;
 		};
@@ -124,7 +128,11 @@ benchmarks! {
 
 	add_registrar {
 		let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::<T>(r)?;
-	}: _(RawOrigin::Root, account::<T>("registrar", r + 1))
+		ensure!(Registrars::<T>::get().len() as u32 == r, "Registrars not set up correctly.");
+	}: _(RawOrigin::Root, account("registrar", r + 1, SEED))
+	verify {
+		ensure!(Registrars::<T>::get().len() as u32 == r + 1, "Registrars not added.");
+	}
 
 	set_identity {
 		let r in ...;
@@ -133,7 +141,7 @@ benchmarks! {
 		let x in _ .. _ => ();
 		let caller = {
 			// The target user
-			let caller = account::<T>("caller", 0);
+			let caller: T::AccountId = whitelisted_caller();
 			let caller_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(caller.clone());
 			let caller_origin: <T as frame_system::Trait>::Origin = RawOrigin::Signed(caller.clone()).into();
 			let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
@@ -146,7 +154,7 @@ benchmarks! {
 			for i in 0..r {
 				Identity::<T>::request_judgement(caller_origin.clone(), i, 10.into())?;
 				Identity::<T>::provide_judgement(
-					RawOrigin::Signed(account::<T>("registrar", i)).into(),
+					RawOrigin::Signed(account("registrar", i, SEED)).into(),
 					i,
 					caller_lookup.clone(),
 					Judgement::Reasonable
@@ -154,68 +162,44 @@ benchmarks! {
 			}
 			caller
 		};
-	}: _(
-		RawOrigin::Signed(caller),
-		create_identity_info::<T>(x)
-	)
-
-	set_subs {
-		let caller = account::<T>("caller", 0);
+	}: _(RawOrigin::Signed(caller.clone()), create_identity_info::<T>(x))
+	verify {
+		assert_last_event::<T>(Event::<T>::IdentitySet(caller).into());
+	}
 
-		// Give them p many previous sub accounts.
-		let p in 1 .. T::MaxSubAccounts::get() => {
-			let _ = add_sub_accounts::<T>(&caller, p)?;
-		};
+	// We need to split `set_subs` into two benchmarks to accurately isolate the potential
+	// writes caused by new or old sub accounts. The actual weight should simply be
+	// the sum of these two weights.
+	set_subs_new {
+		let caller: T::AccountId = whitelisted_caller();
 		// Create a new subs vec with s sub accounts
 		let s in 1 .. T::MaxSubAccounts::get() => ();
 		let subs = create_sub_accounts::<T>(&caller, s)?;
+		ensure!(SubsOf::<T>::get(&caller).1.len() == 0, "Caller already has subs");
+	}: set_subs(RawOrigin::Signed(caller.clone()), subs)
+	verify {
+		ensure!(SubsOf::<T>::get(&caller).1.len() as u32 == s, "Subs not added");
+	}
 
-	}: _(RawOrigin::Signed(caller), subs)
-
-	add_sub {
-		let caller = account::<T>("caller", 0);
-
+	set_subs_old {
+		let caller: T::AccountId = whitelisted_caller();
 		// Give them p many previous sub accounts.
-		let p in 1 .. T::MaxSubAccounts::get() - 1 => {
+		let p in 1 .. T::MaxSubAccounts::get() => {
 			let _ = add_sub_accounts::<T>(&caller, p)?;
 		};
-		let sub = account::<T>("new_sub", 0);
-		let data = Data::Raw(vec![0; 32]);
-	}: _(RawOrigin::Signed(caller), T::Lookup::unlookup(sub), data)
-
-	rename_sub {
-		let caller = account::<T>("caller", 0);
-
-		let p in 1 .. T::MaxSubAccounts::get();
-
-		// Give them p many previous sub accounts.
-		let (sub, _) = add_sub_accounts::<T>(&caller, p)?.remove(0);
-		let data = Data::Raw(vec![1; 32]);
-
-	}: _(RawOrigin::Signed(caller), T::Lookup::unlookup(sub), data)
-
-	remove_sub {
-		let caller = account::<T>("caller", 0);
-
-		// Give them p many previous sub accounts.
-		let p in 1 .. T::MaxSubAccounts::get();
-		let (sub, _) = add_sub_accounts::<T>(&caller, p)?.remove(0);
-	}: _(RawOrigin::Signed(caller), T::Lookup::unlookup(sub))
-
-	quit_sub {
-		let caller = account::<T>("caller", 0);
-		let sup = account::<T>("super", 0);
-
-		// Give them p many previous sub accounts.
-		let p in 1 .. T::MaxSubAccounts::get() - 1 => {
-			let _ = add_sub_accounts::<T>(&sup, p)?;
-		};
-		let sup_origin = RawOrigin::Signed(sup).into();
-		Identity::<T>::add_sub(sup_origin, T::Lookup::unlookup(caller.clone()), Data::Raw(vec![0; 32]))?;
-	}: _(RawOrigin::Signed(caller))
+		// Remove all subs.
+		let subs = create_sub_accounts::<T>(&caller, 0)?;
+		ensure!(
+			SubsOf::<T>::get(&caller).1.len() as u32 == p,
+			"Caller does have subs",
+		);
+	}: set_subs(RawOrigin::Signed(caller.clone()), subs)
+	verify {
+		ensure!(SubsOf::<T>::get(&caller).1.len() == 0, "Subs not removed");
+	}
 
 	clear_identity {
-		let caller = account::<T>("caller", 0);
+		let caller: T::AccountId = whitelisted_caller();
 		let caller_origin = <T as frame_system::Trait>::Origin::from(RawOrigin::Signed(caller.clone()));
 		let caller_lookup = <T::Lookup as StaticLookup>::unlookup(caller.clone());
 		let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
@@ -228,24 +212,31 @@ benchmarks! {
 		for i in 0..r {
 			Identity::<T>::request_judgement(caller_origin.clone(), i, 10.into())?;
 			Identity::<T>::provide_judgement(
-				RawOrigin::Signed(account::<T>("registrar", i)).into(),
+				RawOrigin::Signed(account("registrar", i, SEED)).into(),
 				i,
 				caller_lookup.clone(),
 				Judgement::Reasonable
 			)?;
 		}
-	}: _(RawOrigin::Signed(caller))
+		ensure!(IdentityOf::<T>::contains_key(&caller), "Identity does not exist.");
+	}: _(RawOrigin::Signed(caller.clone()))
+	verify {
+		ensure!(!IdentityOf::<T>::contains_key(&caller), "Identity not cleared.");
+	}
 
 	request_judgement {
-		let caller = account::<T>("caller", 0);
+		let caller: T::AccountId = whitelisted_caller();
 		let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
 
 		let r in ...;
 		let x in ...;
-	}: _(RawOrigin::Signed(caller), r - 1, 10.into())
+	}: _(RawOrigin::Signed(caller.clone()), r - 1, 10.into())
+	verify {
+		assert_last_event::<T>(Event::<T>::JudgementRequested(caller, r-1).into());
+	}
 
 	cancel_request {
-		let caller = account::<T>("caller", 0);
+		let caller: T::AccountId = whitelisted_caller();
 		let caller_origin = <T as frame_system::Trait>::Origin::from(RawOrigin::Signed(caller.clone()));
 		let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
 
@@ -253,27 +244,42 @@ benchmarks! {
 		let x in ...;
 
 		Identity::<T>::request_judgement(caller_origin, r - 1, 10.into())?;
-	}: _(RawOrigin::Signed(caller), r - 1)
+	}: _(RawOrigin::Signed(caller.clone()), r - 1)
+	verify {
+		assert_last_event::<T>(Event::<T>::JudgementUnrequested(caller, r-1).into());
+	}
 
 	set_fee {
-		let caller = account::<T>("caller", 0);
+		let caller: T::AccountId = whitelisted_caller();
 
 		let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::<T>(r)?;
 
 		Identity::<T>::add_registrar(RawOrigin::Root.into(), caller.clone())?;
-	}: _(RawOrigin::Signed(caller), r, 10.into())
+		let registrars = Registrars::<T>::get();
+		ensure!(registrars[r as usize].as_ref().unwrap().fee == 0.into(), "Fee already set.");
+	}: _(RawOrigin::Signed(caller), r, 100.into())
+	verify {
+		let registrars = Registrars::<T>::get();
+		ensure!(registrars[r as usize].as_ref().unwrap().fee == 100.into(), "Fee not changed.");
+	}
 
 	set_account_id {
-		let caller = account::<T>("caller", 0);
+		let caller: T::AccountId = whitelisted_caller();
 		let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
 
 		let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::<T>(r)?;
 
 		Identity::<T>::add_registrar(RawOrigin::Root.into(), caller.clone())?;
-	}: _(RawOrigin::Signed(caller), r, account::<T>("new", 0))
+		let registrars = Registrars::<T>::get();
+		ensure!(registrars[r as usize].as_ref().unwrap().account == caller.clone(), "id not set.");
+	}: _(RawOrigin::Signed(caller), r, account("new", 0, SEED))
+	verify {
+		let registrars = Registrars::<T>::get();
+		ensure!(registrars[r as usize].as_ref().unwrap().account == account("new", 0, SEED), "id not changed.");
+	}
 
 	set_fields {
-		let caller = account::<T>("caller", 0);
+		let caller: T::AccountId = whitelisted_caller();
 		let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
 
 		let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::<T>(r)?;
@@ -283,16 +289,22 @@ benchmarks! {
 			IdentityField::Display | IdentityField::Legal | IdentityField::Web | IdentityField::Riot
 			| IdentityField::Email | IdentityField::PgpFingerprint | IdentityField::Image | IdentityField::Twitter
 		);
+		let registrars = Registrars::<T>::get();
+		ensure!(registrars[r as usize].as_ref().unwrap().fields == Default::default(), "fields already set.");
 	}: _(RawOrigin::Signed(caller), r, fields)
+	verify {
+		let registrars = Registrars::<T>::get();
+		ensure!(registrars[r as usize].as_ref().unwrap().fields != Default::default(), "fields not set.");
+	}
 
 	provide_judgement {
 		// The user
-		let user = account::<T>("user", r);
+		let user: T::AccountId = account("user", r, SEED);
 		let user_origin = <T as frame_system::Trait>::Origin::from(RawOrigin::Signed(user.clone()));
 		let user_lookup = <T::Lookup as StaticLookup>::unlookup(user.clone());
 		let _ = T::Currency::make_free_balance_be(&user, BalanceOf::<T>::max_value());
 
-		let caller = account::<T>("caller", 0);
+		let caller: T::AccountId = whitelisted_caller();
 		let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
 
 		let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::<T>(r)?;
@@ -305,28 +317,91 @@ benchmarks! {
 		Identity::<T>::add_registrar(RawOrigin::Root.into(), caller.clone())?;
 		Identity::<T>::request_judgement(user_origin.clone(), r, 10.into())?;
 	}: _(RawOrigin::Signed(caller), r, user_lookup, Judgement::Reasonable)
+	verify {
+		assert_last_event::<T>(Event::<T>::JudgementGiven(user, r).into())
+	}
 
 	kill_identity {
-		let caller = account::<T>("caller", 0);
-		let caller_origin: <T as frame_system::Trait>::Origin = RawOrigin::Signed(caller.clone()).into();
-		let caller_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(caller.clone());
-		let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
-
 		let r in ...;
-		let s in ...;
-		let x in ...;
+		// Setting up our own account below.
+		let s in _ .. _ => {};
+		let x in _ .. _ => {};
+
+		let target: T::AccountId = account("target", 0, SEED);
+		let target_origin: <T as frame_system::Trait>::Origin = RawOrigin::Signed(target.clone()).into();
+		let target_lookup: <T::Lookup as StaticLookup>::Source = T::Lookup::unlookup(target.clone());
+		let _ = T::Currency::make_free_balance_be(&target, BalanceOf::<T>::max_value());
+
+		let info = create_identity_info::<T>(x);
+		Identity::<T>::set_identity(target_origin.clone(), info)?;
+		let _ = add_sub_accounts::<T>(&target, s)?;
 
 		// User requests judgement from all the registrars, and they approve
 		for i in 0..r {
-			Identity::<T>::request_judgement(caller_origin.clone(), i, 10.into())?;
+			Identity::<T>::request_judgement(target_origin.clone(), i, 10.into())?;
 			Identity::<T>::provide_judgement(
-				RawOrigin::Signed(account::<T>("registrar", i)).into(),
+				RawOrigin::Signed(account("registrar", i, SEED)).into(),
 				i,
-				caller_lookup.clone(),
+				target_lookup.clone(),
 				Judgement::Reasonable
 			)?;
 		}
-	}: _(RawOrigin::Root, caller_lookup)
+		ensure!(IdentityOf::<T>::contains_key(&target), "Identity not set");
+	}: _(RawOrigin::Root, target_lookup)
+	verify {
+		ensure!(!IdentityOf::<T>::contains_key(&target), "Identity not removed");
+	}
+
+	add_sub {
+		let s in 1 .. T::MaxSubAccounts::get() - 1;
+
+		let caller: T::AccountId = whitelisted_caller();
+		let _ = add_sub_accounts::<T>(&caller, s)?;
+		let sub = account("new_sub", 0, SEED);
+		let data = Data::Raw(vec![0; 32]);
+		ensure!(SubsOf::<T>::get(&caller).1.len() as u32 == s, "Subs not set.");
+	}: _(RawOrigin::Signed(caller.clone()), T::Lookup::unlookup(sub), data)
+	verify {
+		ensure!(SubsOf::<T>::get(&caller).1.len() as u32 == s + 1, "Subs not added.");
+	}
+
+	rename_sub {
+		let s in 1 .. T::MaxSubAccounts::get();
+
+		let caller: T::AccountId = whitelisted_caller();
+		let (sub, _) = add_sub_accounts::<T>(&caller, s)?.remove(0);
+		let data = Data::Raw(vec![1; 32]);
+		ensure!(SuperOf::<T>::get(&sub).unwrap().1 != data, "data already set");
+	}: _(RawOrigin::Signed(caller), T::Lookup::unlookup(sub.clone()), data.clone())
+	verify {
+		ensure!(SuperOf::<T>::get(&sub).unwrap().1 == data, "data not set");
+	}
+
+	remove_sub {
+		let s in 1 .. T::MaxSubAccounts::get();
+
+		let caller: T::AccountId = whitelisted_caller();
+		let (sub, _) = add_sub_accounts::<T>(&caller, s)?.remove(0);
+		ensure!(SuperOf::<T>::contains_key(&sub), "Sub doesn't exists");
+	}: _(RawOrigin::Signed(caller), T::Lookup::unlookup(sub.clone()))
+	verify {
+		ensure!(!SuperOf::<T>::contains_key(&sub), "Sub not removed");
+	}
+
+	quit_sub {
+		let s in 1 .. T::MaxSubAccounts::get() - 1;
+
+		let caller: T::AccountId = whitelisted_caller();
+		let sup = account("super", 0, SEED);
+		let _ = add_sub_accounts::<T>(&sup, s)?;
+		let sup_origin = RawOrigin::Signed(sup).into();
+		Identity::<T>::add_sub(sup_origin, T::Lookup::unlookup(caller.clone()), Data::Raw(vec![0; 32]))?;
+		ensure!(SuperOf::<T>::contains_key(&caller), "Sub doesn't exists");
+	}: _(RawOrigin::Signed(caller.clone()))
+	verify {
+		ensure!(!SuperOf::<T>::contains_key(&caller), "Sub not removed");
+	}
+
 }
 
 #[cfg(test)]
@@ -340,7 +415,8 @@ mod tests {
 		new_test_ext().execute_with(|| {
 			assert_ok!(test_benchmark_add_registrar::<Test>());
 			assert_ok!(test_benchmark_set_identity::<Test>());
-			assert_ok!(test_benchmark_set_subs::<Test>());
+			assert_ok!(test_benchmark_set_subs_new::<Test>());
+			assert_ok!(test_benchmark_set_subs_old::<Test>());
 			assert_ok!(test_benchmark_clear_identity::<Test>());
 			assert_ok!(test_benchmark_request_judgement::<Test>());
 			assert_ok!(test_benchmark_cancel_request::<Test>());
@@ -349,6 +425,10 @@ mod tests {
 			assert_ok!(test_benchmark_set_fields::<Test>());
 			assert_ok!(test_benchmark_provide_judgement::<Test>());
 			assert_ok!(test_benchmark_kill_identity::<Test>());
+			assert_ok!(test_benchmark_add_sub::<Test>());
+			assert_ok!(test_benchmark_rename_sub::<Test>());
+			assert_ok!(test_benchmark_remove_sub::<Test>());
+			assert_ok!(test_benchmark_quit_sub::<Test>());
 		});
 	}
 }
diff --git a/frame/identity/src/default_weights.rs b/frame/identity/src/default_weights.rs
new file mode 100644
index 0000000000000..93b1c89ab93dd
--- /dev/null
+++ b/frame/identity/src/default_weights.rs
@@ -0,0 +1,135 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn add_registrar(r: u32, ) -> Weight {
+		(39_603_000 as Weight)
+			.saturating_add((418_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_identity(r: u32, x: u32, ) -> Weight {
+		(110_679_000 as Weight)
+			.saturating_add((389_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((2_985_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_subs_new(s: u32, ) -> Weight {
+		(78_697_000 as Weight)
+			.saturating_add((15_225_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight)))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn set_subs_old(p: u32, ) -> Weight {
+		(71_308_000 as Weight)
+			.saturating_add((5_772_000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight)))
+	}
+	fn clear_identity(r: u32, s: u32, x: u32, ) -> Weight {
+		(91_553_000 as Weight)
+			.saturating_add((284_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((5_749_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add((1_621_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn request_judgement(r: u32, x: u32, ) -> Weight {
+		(110_856_000 as Weight)
+			.saturating_add((496_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((3_221_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn cancel_request(r: u32, x: u32, ) -> Weight {
+		(96_857_000 as Weight)
+			.saturating_add((311_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((3_204_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_fee(r: u32, ) -> Weight {
+		(16_276_000 as Weight)
+			.saturating_add((381_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_account_id(r: u32, ) -> Weight {
+		(18_530_000 as Weight)
+			.saturating_add((391_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn set_fields(r: u32, ) -> Weight {
+		(16_359_000 as Weight)
+			.saturating_add((379_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn provide_judgement(r: u32, x: u32, ) -> Weight {
+		(72_869_000 as Weight)
+			.saturating_add((423_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((3_187_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn kill_identity(r: u32, s: u32, x: u32, ) -> Weight {
+		(123_199_000 as Weight)
+			.saturating_add((71_000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((5_730_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add((2_000 as Weight).saturating_mul(x as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+			.saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight)))
+	}
+	fn add_sub(s: u32, ) -> Weight {
+		(110_070_000 as Weight)
+			.saturating_add((262_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn rename_sub(s: u32, ) -> Weight {
+		(37_130_000 as Weight)
+			.saturating_add((79_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn remove_sub(s: u32, ) -> Weight {
+		(103_295_000 as Weight)
+			.saturating_add((235_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn quit_sub(s: u32, ) -> Weight {
+		(65_716_000 as Weight)
+			.saturating_add((227_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+}
diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs
index e69255ab1980b..1ff69af9a9036 100644
--- a/frame/identity/src/lib.rs
+++ b/frame/identity/src/lib.rs
@@ -86,7 +86,10 @@ use frame_support::{
 };
 use frame_system::ensure_signed;
 
+#[cfg(test)]
+mod tests;
 mod benchmarking;
+mod default_weights;
 
 type BalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
 type NegativeImbalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::NegativeImbalance;
@@ -94,7 +97,12 @@ type NegativeImbalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_sy
 pub trait WeightInfo {
 	fn add_registrar(r: u32, ) -> Weight;
 	fn set_identity(r: u32, x: u32, ) -> Weight;
-	fn set_subs(p: u32, s: u32, ) -> Weight;
+	fn set_subs_new(s: u32, ) -> Weight;
+	fn set_subs_old(p: u32, ) -> Weight;
+	fn add_sub(p: u32, ) -> Weight;
+	fn rename_sub(p: u32, ) -> Weight;
+	fn remove_sub(p: u32, ) -> Weight;
+	fn quit_sub(p: u32, ) -> Weight;
 	fn clear_identity(r: u32, s: u32, x: u32, ) -> Weight;
 	fn request_judgement(r: u32, x: u32, ) -> Weight;
 	fn cancel_request(r: u32, x: u32, ) -> Weight;
@@ -103,28 +111,6 @@ pub trait WeightInfo {
 	fn set_fields(r: u32, ) -> Weight;
 	fn provide_judgement(r: u32, x: u32, ) -> Weight;
 	fn kill_identity(r: u32, s: u32, x: u32, ) -> Weight;
-	fn add_sub(p: u32, ) -> Weight;
-	fn rename_sub() -> Weight;
-	fn remove_sub(p: u32, ) -> Weight;
-	fn quit_sub(p: u32, ) -> Weight;
-}
-
-impl WeightInfo for () {
-	fn add_registrar(_r: u32, ) -> Weight { 1_000_000_000 }
-	fn set_identity(_r: u32, _x: u32, ) -> Weight { 1_000_000_000 }
-	fn set_subs(_p: u32, _s: u32, ) -> Weight { 1_000_000_000 }
-	fn clear_identity(_r: u32, _s: u32, _x: u32, ) -> Weight { 1_000_000_000 }
-	fn request_judgement(_r: u32, _x: u32, ) -> Weight { 1_000_000_000 }
-	fn cancel_request(_r: u32, _x: u32, ) -> Weight { 1_000_000_000 }
-	fn set_fee(_r: u32, ) -> Weight { 1_000_000_000 }
-	fn set_account_id(_r: u32, ) -> Weight { 1_000_000_000 }
-	fn set_fields(_r: u32, ) -> Weight { 1_000_000_000 }
-	fn provide_judgement(_r: u32, _x: u32, ) -> Weight { 1_000_000_000 }
-	fn kill_identity(_r: u32, _s: u32, _x: u32, ) -> Weight { 1_000_000_000 }
-	fn add_sub(_p: u32, ) -> Weight { 1_000_000_000 }
-	fn rename_sub() -> Weight { 1_000_000_000 }
-	fn remove_sub(_p: u32, ) -> Weight { 1_000_000_000 }
-	fn quit_sub(_p: u32, ) -> Weight { 1_000_000_000 }
 }
 
 pub trait Trait: frame_system::Trait {
@@ -525,161 +511,6 @@ decl_error! {
 	}
 }
 
-/// Functions for calcuating the weight of dispatchables.
-mod weight_for {
-	use frame_support::{traits::Get, weights::Weight};
-	use super::Trait;
-
-	/// Weight calculation for `add_registrar`.
-	///
-	/// Based on benchmark:
-	/// 22.24 + R * 0.371 µs (min squares analysis)
-	pub(crate) fn add_registrar<T: Trait>(
-		registrars: Weight
-	) -> Weight {
-		T::DbWeight::get().reads_writes(1, 1)
-			+ 23_000_000 // constant
-			+ 380_000 * registrars // R
-	}
-
-	/// Weight calculation for `set_identity`.
-	///
-	/// Based on benchmark:
-	/// 50.64 + R * 0.215 + X * 1.424 µs (min squares analysis)
-	pub(crate) fn set_identity<T: Trait>(
-		judgements: Weight,
-		extra_fields: Weight
-	) -> Weight {
-		T::DbWeight::get().reads_writes(1, 1)
-			+ 51_000_000 // constant
-			+ 220_000 * judgements // R
-			+ 1_500_000 * extra_fields // X
-	}
-
-	/// Weight calculation for `set_subs`.
-	///
-	/// Based on benchmark:
-	/// 36.21 + P * 2.481 + S * 3.633 µs (min squares analysis)
-	pub(crate) fn set_subs<T: Trait>(
-		old_subs: Weight,
-		subs: Weight
-	) -> Weight {
-		let db = T::DbWeight::get();
-		db.reads(1) // storage-exists (`IdentityOf::contains_key`)
-			.saturating_add(db.reads_writes(1, old_subs)) // `SubsOf::get` read + P old DB deletions
-			.saturating_add(db.writes(subs + 1)) // S + 1 new DB writes
-			.saturating_add(37_000_000) // constant
-			.saturating_add(2_500_000 * old_subs) // P
-			.saturating_add(subs.saturating_mul(3_700_000)) // S
-	}
-
-	/// Weight calculation for `clear_identity`.
-	///
-	/// Based on benchmark:
-	/// 43.19 + R * 0.099 + S * 2.547 + X * 0.875 µs (min squares analysis)
-	pub(crate) fn clear_identity<T: Trait>(
-		judgements: Weight,
-		subs: Weight,
-		extra_fields: Weight
-	) -> Weight {
-		T::DbWeight::get().reads_writes(2, subs + 2) // S + 2 deletions
-			+ 44_000_000 // constant
-			+ 100_000 * judgements // R
-			+ 2_600_000 * subs // S
-			+ 900_000 * extra_fields // X
-	}
-
-	/// Weight calculation for `request_judgement`.
-	///
-	/// Based on benchmark:
-	/// 51.51 + R * 0.32 + X * 1.85 µs (min squares analysis)
-	pub(crate) fn request_judgement<T: Trait>(
-		judgements: Weight,
-		extra_fields: Weight
-	) -> Weight {
-		T::DbWeight::get().reads_writes(2, 1)
-			+ 52_000_000 // constant
-			+ 400_000 * judgements // R
-			+ 1_900_000 * extra_fields // X
-	}
-
-	/// Weight calculation for `cancel_request`.
-	///
-	/// Based on benchmark:
-	/// 40.95 + R * 0.219 + X * 1.655 µs (min squares analysis)
-	pub(crate) fn cancel_request<T: Trait>(
-		judgements: Weight,
-		extra_fields: Weight
-	) -> Weight {
-		T::DbWeight::get().reads_writes(1, 1)
-			+ 41_000_000 // constant
-			+ 300_000 * judgements // R
-			+ 1_700_000 * extra_fields // X
-	}
-
-	/// Weight calculation for `provide_judgement`.
-	///
-	/// Based on benchmark:
-	/// 40.77 + R * 0.282 + X * 1.66 µs (min squares analysis)
-	pub(crate) fn provide_judgement<T: Trait>(
-		judgements: Weight,
-		extra_fields: Weight
-	) -> Weight {
-		T::DbWeight::get().reads_writes(2, 1)
-			+ 41_000_000 // constant
-			+ 300_000 * judgements // R
-			+ 1_700_000 * extra_fields// X
-	}
-
-	/// Weight calculation for `kill_identity`.
-	///
-	/// Based on benchmark:
-	/// 83.96 + R * 0.122 + S * 2.533 + X * 0.867 µs (min squares analysis)
-	pub(crate) fn kill_identity<T: Trait>(
-		judgements: Weight,
-		subs: Weight,
-		extra_fields: Weight
-	) -> Weight {
-		let db = T::DbWeight::get();
-		db.reads_writes(2, subs + 2) // 2 `take`s + S deletions
-			+ db.reads_writes(1, 1) // balance ops
-			+ 84_000_000 // constant
-			+ 130_000 * judgements // R
-			+ 2_600_000 * subs // S
-			+ 900_000 * extra_fields // X
-	}
-
-	/// Weight calculation for `add_sub`.
-	pub(crate) fn add_sub<T: Trait>(
-		subs: Weight,
-	) -> Weight {
-		let db = T::DbWeight::get();
-		db.reads_writes(4, 3) + 124_000_000 + 156_000 * subs
-	}
-
-	/// Weight calculation for `rename_sub`.
-	pub(crate) fn rename_sub<T: Trait>() -> Weight {
-		let db = T::DbWeight::get();
-		db.reads_writes(2, 1) + 30_000_000
-	}
-
-	/// Weight calculation for `remove_sub`.
-	pub(crate) fn remove_sub<T: Trait>(
-		subs: Weight,
-	) -> Weight {
-		let db = T::DbWeight::get();
-		db.reads_writes(4, 3) + 86_000_000 + 50_000 * subs
-	}
-
-	/// Weight calculation for `quit_sub`.
-	pub(crate) fn quit_sub<T: Trait>(
-		subs: Weight,
-	) -> Weight {
-		let db = T::DbWeight::get();
-		db.reads_writes(3, 2) + 63_000_000 + 230_000 * subs
-	}
-}
-
 decl_module! {
 	/// Identity module declaration.
 	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
@@ -722,7 +553,7 @@ decl_module! {
 		/// - One storage mutation (codec `O(R)`).
 		/// - One event.
 		/// # </weight>
-		#[weight = weight_for::add_registrar::<T>(T::MaxRegistrars::get().into()) ]
+		#[weight = T::WeightInfo::add_registrar(T::MaxRegistrars::get()) ]
 		fn add_registrar(origin, account: T::AccountId) -> DispatchResultWithPostInfo {
 			T::RegistrarOrigin::ensure_origin(origin)?;
 
@@ -738,7 +569,7 @@ decl_module! {
 
 			Self::deposit_event(RawEvent::RegistrarAdded(i));
 
-			Ok(Some(weight_for::add_registrar::<T>(registrar_count as Weight)).into())
+			Ok(Some(T::WeightInfo::add_registrar(registrar_count as u32)).into())
 		}
 
 		/// Set an account's identity information and reserve the appropriate deposit.
@@ -760,7 +591,7 @@ decl_module! {
 		/// - One storage mutation (codec-read `O(X' + R)`, codec-write `O(X + R)`).
 		/// - One event.
 		/// # </weight>
-		#[weight =  weight_for::set_identity::<T>(
+		#[weight =  T::WeightInfo::set_identity(
 			T::MaxRegistrars::get().into(), // R
 			T::MaxAdditionalFields::get().into(), // X
 		)]
@@ -789,13 +620,13 @@ decl_module! {
 				let _ = T::Currency::unreserve(&sender, old_deposit - id.deposit);
 			}
 
-			let judgements = id.judgements.len() as Weight;
+			let judgements = id.judgements.len();
 			<IdentityOf<T>>::insert(&sender, id);
 			Self::deposit_event(RawEvent::IdentitySet(sender));
 
-			Ok(Some(weight_for::set_identity::<T>(
-				judgements, // R
-				extra_fields as Weight // X
+			Ok(Some(T::WeightInfo::set_identity(
+				judgements as u32, // R
+				extra_fields // X
 			)).into())
 		}
 
@@ -820,10 +651,15 @@ decl_module! {
 		///   - One storage write (codec complexity `O(S)`).
 		///   - One storage-exists (`IdentityOf::contains_key`).
 		/// # </weight>
-		#[weight = weight_for::set_subs::<T>(
-			T::MaxSubAccounts::get().into(), // P
-			subs.len() as Weight // S
-		)]
+		// TODO: This whole extrinsic screams "not optimized". For example we could
+		// filter any overlap between new and old subs, and avoid reading/writing
+		// to those values... We could also ideally avoid needing to write to
+		// N storage items for N sub accounts. Right now the weight on this function
+		// is a large overestimate due to the fact that it could potentially write
+		// to 2 x T::MaxSubAccounts::get().
+		#[weight = T::WeightInfo::set_subs_old(T::MaxSubAccounts::get()) // P: Assume max sub accounts removed.
+			.saturating_add(T::WeightInfo::set_subs_new(subs.len() as u32)) // S: Assume all subs are new.
+		]
 		fn set_subs(origin, subs: Vec<(T::AccountId, Data)>) -> DispatchResultWithPostInfo {
 			let sender = ensure_signed(origin)?;
 			ensure!(<IdentityOf<T>>::contains_key(&sender), Error::<T>::NotFound);
@@ -837,11 +673,10 @@ decl_module! {
 
 			if old_deposit < new_deposit {
 				T::Currency::reserve(&sender, new_deposit - old_deposit)?;
-			}
-			// do nothing if they're equal.
-			if old_deposit > new_deposit {
+			} else if old_deposit > new_deposit {
 				let _ = T::Currency::unreserve(&sender, old_deposit - new_deposit);
 			}
+			// do nothing if they're equal.
 
 			for s in old_ids.iter() {
 				<SuperOf<T>>::remove(s);
@@ -850,7 +685,7 @@ decl_module! {
 				<SuperOf<T>>::insert(&id, (sender.clone(), name));
 				id
 			}).collect::<Vec<_>>();
-			let new_subs = ids.len() as Weight;
+			let new_subs = ids.len();
 
 			if ids.is_empty() {
 				<SubsOf<T>>::remove(&sender);
@@ -858,10 +693,10 @@ decl_module! {
 				<SubsOf<T>>::insert(&sender, (new_deposit, ids));
 			}
 
-			Ok(Some(weight_for::set_subs::<T>(
-				old_ids.len() as Weight, // P
-				new_subs // S
-			)).into())
+			Ok(Some(
+				T::WeightInfo::set_subs_old(old_ids.len() as u32) // P: Real number of old accounts removed.
+				.saturating_add(T::WeightInfo::set_subs_new(new_subs as u32)) // S: New subs added.
+			).into())
 		}
 
 		/// Clear an account's identity info and all sub-accounts and return all deposits.
@@ -882,7 +717,7 @@ decl_module! {
 		/// - `2` storage reads and `S + 2` storage deletions.
 		/// - One event.
 		/// # </weight>
-		#[weight = weight_for::clear_identity::<T>(
+		#[weight = T::WeightInfo::clear_identity(
 			T::MaxRegistrars::get().into(), // R
 			T::MaxSubAccounts::get().into(), // S
 			T::MaxAdditionalFields::get().into(), // X
@@ -901,10 +736,10 @@ decl_module! {
 
 			Self::deposit_event(RawEvent::IdentityCleared(sender, deposit));
 
-			Ok(Some(weight_for::clear_identity::<T>(
-				id.judgements.len() as Weight, // R
-				sub_ids.len() as Weight, // S
-				id.info.additional.len() as Weight // X
+			Ok(Some(T::WeightInfo::clear_identity(
+				id.judgements.len() as u32, // R
+				sub_ids.len() as u32, // S
+				id.info.additional.len() as u32 // X
 			)).into())
 		}
 
@@ -931,7 +766,7 @@ decl_module! {
 		/// - Storage: 1 read `O(R)`, 1 mutate `O(X + R)`.
 		/// - One event.
 		/// # </weight>
-		#[weight = weight_for::request_judgement::<T>(
+		#[weight = T::WeightInfo::request_judgement(
 			T::MaxRegistrars::get().into(), // R
 			T::MaxAdditionalFields::get().into(), // X
 		)]
@@ -958,13 +793,16 @@ decl_module! {
 
 			T::Currency::reserve(&sender, registrar.fee)?;
 
-			let judgements = id.judgements.len() as Weight;
-			let extra_fields = id.info.additional.len() as Weight;
+			let judgements = id.judgements.len();
+			let extra_fields = id.info.additional.len();
 			<IdentityOf<T>>::insert(&sender, id);
 
 			Self::deposit_event(RawEvent::JudgementRequested(sender, reg_index));
 
-			Ok(Some(weight_for::request_judgement::<T>(judgements, extra_fields)).into())
+			Ok(Some(T::WeightInfo::request_judgement(
+				judgements as u32,
+				extra_fields as u32,
+			)).into())
 		}
 
 		/// Cancel a previous request.
@@ -984,7 +822,7 @@ decl_module! {
 		/// - One storage mutation `O(R + X)`.
 		/// - One event
 		/// # </weight>
-		#[weight = weight_for::cancel_request::<T>(
+		#[weight = T::WeightInfo::cancel_request(
 			T::MaxRegistrars::get().into(), // R
 			T::MaxAdditionalFields::get().into(), // X
 		)]
@@ -1001,13 +839,16 @@ decl_module! {
 			};
 
 			let _ = T::Currency::unreserve(&sender, fee);
-			let judgements = id.judgements.len() as Weight;
-			let extra_fields = id.info.additional.len() as Weight;
+			let judgements = id.judgements.len();
+			let extra_fields = id.info.additional.len();
 			<IdentityOf<T>>::insert(&sender, id);
 
 			Self::deposit_event(RawEvent::JudgementUnrequested(sender, reg_index));
 
-			Ok(Some(weight_for::request_judgement::<T>(judgements, extra_fields)).into())
+			Ok(Some(T::WeightInfo::cancel_request(
+				judgements as u32,
+				extra_fields as u32
+			)).into())
 		}
 
 		/// Set the fee required for a judgement to be requested from a registrar.
@@ -1023,10 +864,7 @@ decl_module! {
 		/// - One storage mutation `O(R)`.
 		/// - Benchmark: 7.315 + R * 0.329 µs (min squares analysis)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 1)
-			+ 7_400_000 // constant
-			+ 330_000 * T::MaxRegistrars::get() as Weight // R
-		]
+		#[weight = T::WeightInfo::set_fee(T::MaxRegistrars::get())] // R
 		fn set_fee(origin,
 			#[compact] index: RegistrarIndex,
 			#[compact] fee: BalanceOf<T>,
@@ -1040,9 +878,7 @@ decl_module! {
 					.ok_or_else(|| DispatchError::from(Error::<T>::InvalidIndex))?;
 				Ok(rs.len())
 			})?;
-			Ok(Some(T::DbWeight::get().reads_writes(1, 1)
-				+ 7_400_000 + 330_000 * registrars as Weight // R
-			).into())
+			Ok(Some(T::WeightInfo::set_fee(registrars as u32)).into()) // R
 		}
 
 		/// Change the account associated with a registrar.
@@ -1058,10 +894,7 @@ decl_module! {
 		/// - One storage mutation `O(R)`.
 		/// - Benchmark: 8.823 + R * 0.32 µs (min squares analysis)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 1)
-			+ 8_900_000 // constant
-			+ 320_000 * T::MaxRegistrars::get() as Weight // R
-		]
+		#[weight = T::WeightInfo::set_account_id(T::MaxRegistrars::get())] // R
 		fn set_account_id(origin,
 			#[compact] index: RegistrarIndex,
 			new: T::AccountId,
@@ -1075,9 +908,7 @@ decl_module! {
 					.ok_or_else(|| DispatchError::from(Error::<T>::InvalidIndex))?;
 				Ok(rs.len())
 			})?;
-			Ok(Some(T::DbWeight::get().reads_writes(1, 1)
-				+ 8_900_000 + 320_000 * registrars as Weight // R
-			).into())
+			Ok(Some(T::WeightInfo::set_account_id(registrars as u32)).into()) // R
 		}
 
 		/// Set the field information for a registrar.
@@ -1093,10 +924,7 @@ decl_module! {
 		/// - One storage mutation `O(R)`.
 		/// - Benchmark: 7.464 + R * 0.325 µs (min squares analysis)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 1)
-			+ 7_500_000 // constant
-			+ 330_000 * T::MaxRegistrars::get() as Weight // R
-		]
+		#[weight = T::WeightInfo::set_fields(T::MaxRegistrars::get())] // R
 		fn set_fields(origin,
 			#[compact] index: RegistrarIndex,
 			fields: IdentityFields,
@@ -1110,9 +938,9 @@ decl_module! {
 					.ok_or_else(|| DispatchError::from(Error::<T>::InvalidIndex))?;
 				Ok(rs.len())
 			})?;
-			Ok(Some(T::DbWeight::get().reads_writes(1, 1)
-				+ 7_500_000 + 330_000 * registrars as Weight // R
-			).into())
+			Ok(Some(T::WeightInfo::set_fields(
+				registrars as u32 // R
+			)).into())
 		}
 
 		/// Provide a judgement for an account's identity.
@@ -1134,7 +962,7 @@ decl_module! {
 		/// - Storage: 1 read `O(R)`, 1 mutate `O(R + X)`.
 		/// - One event.
 		/// # </weight>
-		#[weight = weight_for::provide_judgement::<T>(
+		#[weight = T::WeightInfo::provide_judgement(
 			T::MaxRegistrars::get().into(), // R
 			T::MaxAdditionalFields::get().into(), // X
 		)]
@@ -1164,12 +992,15 @@ decl_module! {
 				Err(position) => id.judgements.insert(position, item),
 			}
 
-			let judgements = id.judgements.len() as Weight;
-			let extra_fields = id.info.additional.len() as Weight;
+			let judgements = id.judgements.len();
+			let extra_fields = id.info.additional.len();
 			<IdentityOf<T>>::insert(&target, id);
 			Self::deposit_event(RawEvent::JudgementGiven(target, reg_index));
 
-			Ok(Some(weight_for::provide_judgement::<T>(judgements, extra_fields)).into())
+			Ok(Some(T::WeightInfo::provide_judgement(
+				judgements as u32,
+				extra_fields as u32,
+			)).into())
 		}
 
 		/// Remove an account's identity and sub-account information and slash the deposits.
@@ -1191,7 +1022,7 @@ decl_module! {
 		/// - `S + 2` storage mutations.
 		/// - One event.
 		/// # </weight>
-		#[weight = weight_for::kill_identity::<T>(
+		#[weight = T::WeightInfo::kill_identity(
 			T::MaxRegistrars::get().into(), // R
 			T::MaxSubAccounts::get().into(), // S
 			T::MaxAdditionalFields::get().into(), // X
@@ -1213,10 +1044,10 @@ decl_module! {
 
 			Self::deposit_event(RawEvent::IdentityKilled(target, deposit));
 
-			Ok(Some(weight_for::kill_identity::<T>(
-				id.judgements.len() as Weight, // R
-				sub_ids.len() as Weight, // S
-				id.info.additional.len() as Weight // X
+			Ok(Some(T::WeightInfo::kill_identity(
+				id.judgements.len() as u32, // R
+				sub_ids.len() as u32, // S
+				id.info.additional.len() as u32 // X
 			)).into())
 		}
 
@@ -1227,9 +1058,7 @@ decl_module! {
 		///
 		/// The dispatch origin for this call must be _Signed_ and the sender must have a registered
 		/// sub identity of `sub`.
-		#[weight = weight_for::add_sub::<T>(
-			T::MaxSubAccounts::get().into(), // S
-		)]
+		#[weight = T::WeightInfo::add_sub(T::MaxSubAccounts::get())]
 		fn add_sub(origin, sub: <T::Lookup as StaticLookup>::Source, data: Data) -> DispatchResult {
 			let sender = ensure_signed(origin)?;
 			let sub = T::Lookup::lookup(sub)?;
@@ -1257,7 +1086,7 @@ decl_module! {
 		///
 		/// The dispatch origin for this call must be _Signed_ and the sender must have a registered
 		/// sub identity of `sub`.
-		#[weight = weight_for::rename_sub::<T>()]
+		#[weight = T::WeightInfo::rename_sub(T::MaxSubAccounts::get())]
 		fn rename_sub(origin, sub: <T::Lookup as StaticLookup>::Source, data: Data) {
 			let sender = ensure_signed(origin)?;
 			let sub = T::Lookup::lookup(sub)?;
@@ -1273,9 +1102,7 @@ decl_module! {
 		///
 		/// The dispatch origin for this call must be _Signed_ and the sender must have a registered
 		/// sub identity of `sub`.
-		#[weight = weight_for::remove_sub::<T>(
-			T::MaxSubAccounts::get().into(), // S
-		)]
+		#[weight = T::WeightInfo::remove_sub(T::MaxSubAccounts::get())]
 		fn remove_sub(origin, sub: <T::Lookup as StaticLookup>::Source) {
 			let sender = ensure_signed(origin)?;
 			ensure!(IdentityOf::<T>::contains_key(&sender), Error::<T>::NoIdentity);
@@ -1302,9 +1129,7 @@ decl_module! {
 		///
 		/// NOTE: This should not normally be used, but is provided in the case that the non-
 		/// controller of an account is maliciously registered as a sub-account.
-		#[weight = weight_for::quit_sub::<T>(
-			T::MaxSubAccounts::get().into(), // S
-		)]
+		#[weight = T::WeightInfo::quit_sub(T::MaxSubAccounts::get())]
 		fn quit_sub(origin) {
 			let sender = ensure_signed(origin)?;
 			let (sup, _) = SuperOf::<T>::take(&sender).ok_or(Error::<T>::NotSub)?;
@@ -1328,460 +1153,3 @@ impl<T: Trait> Module<T> {
 			.collect()
 	}
 }
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-
-	use sp_runtime::traits::BadOrigin;
-	use frame_support::{
-		assert_ok, assert_noop, impl_outer_origin, parameter_types, weights::Weight,
-		ord_parameter_types,
-	};
-	use sp_core::H256;
-	use frame_system::{EnsureSignedBy, EnsureOneOf, EnsureRoot};
-	use sp_runtime::{
-		Perbill, testing::Header, traits::{BlakeTwo256, IdentityLookup},
-	};
-
-	impl_outer_origin! {
-		pub enum Origin for Test where system = frame_system {}
-	}
-
-	#[derive(Clone, Eq, PartialEq)]
-	pub struct Test;
-	parameter_types! {
-		pub const BlockHashCount: u64 = 250;
-		pub const MaximumBlockWeight: Weight = 1024;
-		pub const MaximumBlockLength: u32 = 2 * 1024;
-		pub const AvailableBlockRatio: Perbill = Perbill::one();
-	}
-	impl frame_system::Trait for Test {
-		type BaseCallFilter = ();
-		type Origin = Origin;
-		type Index = u64;
-		type BlockNumber = u64;
-		type Hash = H256;
-		type Call = ();
-		type Hashing = BlakeTwo256;
-		type AccountId = u64;
-		type Lookup = IdentityLookup<Self::AccountId>;
-		type Header = Header;
-		type Event = ();
-		type BlockHashCount = BlockHashCount;
-		type MaximumBlockWeight = MaximumBlockWeight;
-		type DbWeight = ();
-		type BlockExecutionWeight = ();
-		type ExtrinsicBaseWeight = ();
-		type MaximumExtrinsicWeight = MaximumBlockWeight;
-		type MaximumBlockLength = MaximumBlockLength;
-		type AvailableBlockRatio = AvailableBlockRatio;
-		type Version = ();
-		type ModuleToIndex = ();
-		type AccountData = pallet_balances::AccountData<u64>;
-		type OnNewAccount = ();
-		type OnKilledAccount = ();
-		type SystemWeightInfo = ();
-	}
-	parameter_types! {
-		pub const ExistentialDeposit: u64 = 1;
-	}
-	impl pallet_balances::Trait for Test {
-		type MaxLocks = ();
-		type Balance = u64;
-		type Event = ();
-		type DustRemoval = ();
-		type ExistentialDeposit = ExistentialDeposit;
-		type AccountStore = System;
-		type WeightInfo = ();
-	}
-	parameter_types! {
-		pub const BasicDeposit: u64 = 10;
-		pub const FieldDeposit: u64 = 10;
-		pub const SubAccountDeposit: u64 = 10;
-		pub const MaxSubAccounts: u32 = 2;
-		pub const MaxAdditionalFields: u32 = 2;
-		pub const MaxRegistrars: u32 = 20;
-	}
-	ord_parameter_types! {
-		pub const One: u64 = 1;
-		pub const Two: u64 = 2;
-	}
-	type EnsureOneOrRoot = EnsureOneOf<
-		u64,
-		EnsureRoot<u64>,
-		EnsureSignedBy<One, u64>
-	>;
-	type EnsureTwoOrRoot = EnsureOneOf<
-		u64,
-		EnsureRoot<u64>,
-		EnsureSignedBy<Two, u64>
-	>;
-	impl Trait for Test {
-		type Event = ();
-		type Currency = Balances;
-		type Slashed = ();
-		type BasicDeposit = BasicDeposit;
-		type FieldDeposit = FieldDeposit;
-		type SubAccountDeposit = SubAccountDeposit;
-		type MaxSubAccounts = MaxSubAccounts;
-		type MaxAdditionalFields = MaxAdditionalFields;
-		type MaxRegistrars = MaxRegistrars;
-		type RegistrarOrigin = EnsureOneOrRoot;
-		type ForceOrigin = EnsureTwoOrRoot;
-		type WeightInfo = ();
-	}
-	type System = frame_system::Module<Test>;
-	type Balances = pallet_balances::Module<Test>;
-	type Identity = Module<Test>;
-
-	pub fn new_test_ext() -> sp_io::TestExternalities {
-		let mut t = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();
-		pallet_balances::GenesisConfig::<Test> {
-			balances: vec![
-				(1, 10),
-				(2, 10),
-				(3, 10),
-				(10, 100),
-				(20, 100),
-				(30, 100),
-			],
-		}.assimilate_storage(&mut t).unwrap();
-		t.into()
-	}
-
-	fn ten() -> IdentityInfo {
-		IdentityInfo {
-			display: Data::Raw(b"ten".to_vec()),
-			legal: Data::Raw(b"The Right Ordinal Ten, Esq.".to_vec()),
-			.. Default::default()
-		}
-	}
-
-	fn twenty() -> IdentityInfo {
-		IdentityInfo {
-			display: Data::Raw(b"twenty".to_vec()),
-			legal: Data::Raw(b"The Right Ordinal Twenty, Esq.".to_vec()),
-			.. Default::default()
-		}
-	}
-
-	#[test]
-	fn editing_subaccounts_should_work() {
-		new_test_ext().execute_with(|| {
-			let data = |x| Data::Raw(vec![x; 1]);
-
-			assert_noop!(Identity::add_sub(Origin::signed(10), 20, data(1)), Error::<Test>::NoIdentity);
-
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-
-			// first sub account
-			assert_ok!(Identity::add_sub(Origin::signed(10), 1, data(1)));
-			assert_eq!(SuperOf::<Test>::get(1), Some((10, data(1))));
-			assert_eq!(Balances::free_balance(10), 80);
-
-			// second sub account
-			assert_ok!(Identity::add_sub(Origin::signed(10), 2, data(2)));
-			assert_eq!(SuperOf::<Test>::get(1), Some((10, data(1))));
-			assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
-			assert_eq!(Balances::free_balance(10), 70);
-
-			// third sub account is too many
-			assert_noop!(Identity::add_sub(Origin::signed(10), 3, data(3)), Error::<Test>::TooManySubAccounts);
-
-			// rename first sub account
-			assert_ok!(Identity::rename_sub(Origin::signed(10), 1, data(11)));
-			assert_eq!(SuperOf::<Test>::get(1), Some((10, data(11))));
-			assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
-			assert_eq!(Balances::free_balance(10), 70);
-
-			// remove first sub account
-			assert_ok!(Identity::remove_sub(Origin::signed(10), 1));
-			assert_eq!(SuperOf::<Test>::get(1), None);
-			assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
-			assert_eq!(Balances::free_balance(10), 80);
-
-			// add third sub account
-			assert_ok!(Identity::add_sub(Origin::signed(10), 3, data(3)));
-			assert_eq!(SuperOf::<Test>::get(1), None);
-			assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
-			assert_eq!(SuperOf::<Test>::get(3), Some((10, data(3))));
-			assert_eq!(Balances::free_balance(10), 70);
-		});
-	}
-
-	#[test]
-	fn resolving_subaccount_ownership_works() {
-		new_test_ext().execute_with(|| {
-			let data = |x| Data::Raw(vec![x; 1]);
-
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_ok!(Identity::set_identity(Origin::signed(20), twenty()));
-
-			// 10 claims 1 as a subaccount
-			assert_ok!(Identity::add_sub(Origin::signed(10), 1, data(1)));
-			assert_eq!(Balances::free_balance(1), 10);
-			assert_eq!(Balances::free_balance(10), 80);
-			assert_eq!(Balances::reserved_balance(10), 20);
-			// 20 cannot claim 1 now
-			assert_noop!(Identity::add_sub(Origin::signed(20), 1, data(1)), Error::<Test>::AlreadyClaimed);
-			// 1 wants to be with 20 so it quits from 10
-			assert_ok!(Identity::quit_sub(Origin::signed(1)));
-			// 1 gets the 10 that 10 paid.
-			assert_eq!(Balances::free_balance(1), 20);
-			assert_eq!(Balances::free_balance(10), 80);
-			assert_eq!(Balances::reserved_balance(10), 10);
-			// 20 can claim 1 now
-			assert_ok!(Identity::add_sub(Origin::signed(20), 1, data(1)));
-		});
-	}
-
-	#[test]
-	fn trailing_zeros_decodes_into_default_data() {
-		let encoded = Data::Raw(b"Hello".to_vec()).encode();
-		assert!(<(Data, Data)>::decode(&mut &encoded[..]).is_err());
-		let input = &mut &encoded[..];
-		let (a, b) = <(Data, Data)>::decode(&mut AppendZerosInput::new(input)).unwrap();
-		assert_eq!(a, Data::Raw(b"Hello".to_vec()));
-		assert_eq!(b, Data::None);
-	}
-
-	#[test]
-	fn adding_registrar_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
-			let fields = IdentityFields(IdentityField::Display | IdentityField::Legal);
-			assert_ok!(Identity::set_fields(Origin::signed(3), 0, fields));
-			assert_eq!(Identity::registrars(), vec![
-				Some(RegistrarInfo { account: 3, fee: 10, fields })
-			]);
-		});
-	}
-
-	#[test]
-	fn amount_of_registrars_is_limited() {
-		new_test_ext().execute_with(|| {
-			for i in 1..MaxRegistrars::get() + 1 {
-				assert_ok!(Identity::add_registrar(Origin::signed(1), i as u64));
-			}
-			let last_registrar = MaxRegistrars::get() as u64 + 1;
-			assert_noop!(
-				Identity::add_registrar(Origin::signed(1), last_registrar),
-				Error::<Test>::TooManyRegistrars
-			);
-		});
-	}
-
-	#[test]
-	fn registration_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
-			let mut three_fields = ten();
-			three_fields.additional.push(Default::default());
-			three_fields.additional.push(Default::default());
-			three_fields.additional.push(Default::default());
-			assert_noop!(
-				Identity::set_identity(Origin::signed(10), three_fields),
-				Error::<Test>::TooManyFields
-			);
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_eq!(Identity::identity(10).unwrap().info, ten());
-			assert_eq!(Balances::free_balance(10), 90);
-			assert_ok!(Identity::clear_identity(Origin::signed(10)));
-			assert_eq!(Balances::free_balance(10), 100);
-			assert_noop!(Identity::clear_identity(Origin::signed(10)), Error::<Test>::NotNamed);
-		});
-	}
-
-	#[test]
-	fn uninvited_judgement_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_noop!(
-				Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable),
-				Error::<Test>::InvalidIndex
-			);
-
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			assert_noop!(
-				Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable),
-				Error::<Test>::InvalidTarget
-			);
-
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_noop!(
-				Identity::provide_judgement(Origin::signed(10), 0, 10, Judgement::Reasonable),
-				Error::<Test>::InvalidIndex
-			);
-			assert_noop!(
-				Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::FeePaid(1)),
-				Error::<Test>::InvalidJudgement
-			);
-
-			assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable));
-			assert_eq!(Identity::identity(10).unwrap().judgements, vec![(0, Judgement::Reasonable)]);
-		});
-	}
-
-	#[test]
-	fn clearing_judgement_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable));
-			assert_ok!(Identity::clear_identity(Origin::signed(10)));
-			assert_eq!(Identity::identity(10), None);
-		});
-	}
-
-	#[test]
-	fn killing_slashing_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_noop!(Identity::kill_identity(Origin::signed(1), 10), BadOrigin);
-			assert_ok!(Identity::kill_identity(Origin::signed(2), 10));
-			assert_eq!(Identity::identity(10), None);
-			assert_eq!(Balances::free_balance(10), 90);
-			assert_noop!(Identity::kill_identity(Origin::signed(2), 10), Error::<Test>::NotNamed);
-		});
-	}
-
-	#[test]
-	fn setting_subaccounts_should_work() {
-		new_test_ext().execute_with(|| {
-			let mut subs = vec![(20, Data::Raw(vec![40; 1]))];
-			assert_noop!(Identity::set_subs(Origin::signed(10), subs.clone()), Error::<Test>::NotFound);
-
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_ok!(Identity::set_subs(Origin::signed(10), subs.clone()));
-			assert_eq!(Balances::free_balance(10), 80);
-			assert_eq!(Identity::subs_of(10), (10, vec![20]));
-			assert_eq!(Identity::super_of(20), Some((10, Data::Raw(vec![40; 1]))));
-
-			// push another item and re-set it.
-			subs.push((30, Data::Raw(vec![50; 1])));
-			assert_ok!(Identity::set_subs(Origin::signed(10), subs.clone()));
-			assert_eq!(Balances::free_balance(10), 70);
-			assert_eq!(Identity::subs_of(10), (20, vec![20, 30]));
-			assert_eq!(Identity::super_of(20), Some((10, Data::Raw(vec![40; 1]))));
-			assert_eq!(Identity::super_of(30), Some((10, Data::Raw(vec![50; 1]))));
-
-			// switch out one of the items and re-set.
-			subs[0] = (40, Data::Raw(vec![60; 1]));
-			assert_ok!(Identity::set_subs(Origin::signed(10), subs.clone()));
-			assert_eq!(Balances::free_balance(10), 70); // no change in the balance
-			assert_eq!(Identity::subs_of(10), (20, vec![40, 30]));
-			assert_eq!(Identity::super_of(20), None);
-			assert_eq!(Identity::super_of(30), Some((10, Data::Raw(vec![50; 1]))));
-			assert_eq!(Identity::super_of(40), Some((10, Data::Raw(vec![60; 1]))));
-
-			// clear
-			assert_ok!(Identity::set_subs(Origin::signed(10), vec![]));
-			assert_eq!(Balances::free_balance(10), 90);
-			assert_eq!(Identity::subs_of(10), (0, vec![]));
-			assert_eq!(Identity::super_of(30), None);
-			assert_eq!(Identity::super_of(40), None);
-
-			subs.push((20, Data::Raw(vec![40; 1])));
-			assert_noop!(Identity::set_subs(Origin::signed(10), subs.clone()), Error::<Test>::TooManySubAccounts);
-		});
-	}
-
-	#[test]
-	fn clearing_account_should_remove_subaccounts_and_refund() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_ok!(Identity::set_subs(Origin::signed(10), vec![(20, Data::Raw(vec![40; 1]))]));
-			assert_ok!(Identity::clear_identity(Origin::signed(10)));
-			assert_eq!(Balances::free_balance(10), 100);
-			assert!(Identity::super_of(20).is_none());
-		});
-	}
-
-	#[test]
-	fn killing_account_should_remove_subaccounts_and_not_refund() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_ok!(Identity::set_subs(Origin::signed(10), vec![(20, Data::Raw(vec![40; 1]))]));
-			assert_ok!(Identity::kill_identity(Origin::signed(2), 10));
-			assert_eq!(Balances::free_balance(10), 80);
-			assert!(Identity::super_of(20).is_none());
-		});
-	}
-
-	#[test]
-	fn cancelling_requested_judgement_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
-			assert_noop!(Identity::cancel_request(Origin::signed(10), 0), Error::<Test>::NoIdentity);
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_ok!(Identity::request_judgement(Origin::signed(10), 0, 10));
-			assert_ok!(Identity::cancel_request(Origin::signed(10), 0));
-			assert_eq!(Balances::free_balance(10), 90);
-			assert_noop!(Identity::cancel_request(Origin::signed(10), 0), Error::<Test>::NotFound);
-
-			assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable));
-			assert_noop!(Identity::cancel_request(Origin::signed(10), 0), Error::<Test>::JudgementGiven);
-		});
-	}
-
-	#[test]
-	fn requesting_judgement_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
-			assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
-			assert_noop!(Identity::request_judgement(Origin::signed(10), 0, 9), Error::<Test>::FeeChanged);
-			assert_ok!(Identity::request_judgement(Origin::signed(10), 0, 10));
-			// 10 for the judgement request, 10 for the identity.
-			assert_eq!(Balances::free_balance(10), 80);
-
-			// Re-requesting won't work as we already paid.
-			assert_noop!(Identity::request_judgement(Origin::signed(10), 0, 10), Error::<Test>::StickyJudgement);
-			assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Erroneous));
-			// Registrar got their payment now.
-			assert_eq!(Balances::free_balance(3), 20);
-
-			// Re-requesting still won't work as it's erroneous.
-			assert_noop!(Identity::request_judgement(Origin::signed(10), 0, 10), Error::<Test>::StickyJudgement);
-
-			// Requesting from a second registrar still works.
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 4));
-			assert_ok!(Identity::request_judgement(Origin::signed(10), 1, 10));
-
-			// Re-requesting after the judgement has been reduced works.
-			assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::OutOfDate));
-			assert_ok!(Identity::request_judgement(Origin::signed(10), 0, 10));
-		});
-	}
-
-	#[test]
-	fn field_deposit_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
-			assert_ok!(Identity::set_identity(Origin::signed(10), IdentityInfo {
-				additional: vec![
-					(Data::Raw(b"number".to_vec()), Data::Raw(10u32.encode())),
-					(Data::Raw(b"text".to_vec()), Data::Raw(b"10".to_vec())),
-				], .. Default::default()
-			}));
-			assert_eq!(Balances::free_balance(10), 70);
-		});
-	}
-
-	#[test]
-	fn setting_account_id_should_work() {
-		new_test_ext().execute_with(|| {
-			assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
-			// account 4 cannot change the first registrar's identity since it's owned by 3.
-			assert_noop!(Identity::set_account_id(Origin::signed(4), 0, 3), Error::<Test>::InvalidIndex);
-			// account 3 can, because that's the registrar's current account.
-			assert_ok!(Identity::set_account_id(Origin::signed(3), 0, 4));
-			// account 4 can now, because that's their new ID.
-			assert_ok!(Identity::set_account_id(Origin::signed(4), 0, 3));
-		});
-	}
-}
diff --git a/frame/identity/src/tests.rs b/frame/identity/src/tests.rs
new file mode 100644
index 0000000000000..109bab89cfc95
--- /dev/null
+++ b/frame/identity/src/tests.rs
@@ -0,0 +1,472 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Tests for Identity Pallet
+
+use super::*;
+
+use sp_runtime::traits::BadOrigin;
+use frame_support::{
+	assert_ok, assert_noop, impl_outer_origin, parameter_types, weights::Weight,
+	ord_parameter_types,
+};
+use sp_core::H256;
+use frame_system::{EnsureSignedBy, EnsureOneOf, EnsureRoot};
+use sp_runtime::{
+	Perbill, testing::Header, traits::{BlakeTwo256, IdentityLookup},
+};
+
+impl_outer_origin! {
+	pub enum Origin for Test where system = frame_system {}
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct Test;
+parameter_types! {
+	pub const BlockHashCount: u64 = 250;
+	pub const MaximumBlockWeight: Weight = 1024;
+	pub const MaximumBlockLength: u32 = 2 * 1024;
+	pub const AvailableBlockRatio: Perbill = Perbill::one();
+}
+impl frame_system::Trait for Test {
+	type BaseCallFilter = ();
+	type Origin = Origin;
+	type Index = u64;
+	type BlockNumber = u64;
+	type Hash = H256;
+	type Call = ();
+	type Hashing = BlakeTwo256;
+	type AccountId = u64;
+	type Lookup = IdentityLookup<Self::AccountId>;
+	type Header = Header;
+	type Event = ();
+	type BlockHashCount = BlockHashCount;
+	type MaximumBlockWeight = MaximumBlockWeight;
+	type DbWeight = ();
+	type BlockExecutionWeight = ();
+	type ExtrinsicBaseWeight = ();
+	type MaximumExtrinsicWeight = MaximumBlockWeight;
+	type MaximumBlockLength = MaximumBlockLength;
+	type AvailableBlockRatio = AvailableBlockRatio;
+	type Version = ();
+	type ModuleToIndex = ();
+	type AccountData = pallet_balances::AccountData<u64>;
+	type OnNewAccount = ();
+	type OnKilledAccount = ();
+	type SystemWeightInfo = ();
+}
+parameter_types! {
+	pub const ExistentialDeposit: u64 = 1;
+}
+impl pallet_balances::Trait for Test {
+	type Balance = u64;
+	type Event = ();
+	type DustRemoval = ();
+	type ExistentialDeposit = ExistentialDeposit;
+	type AccountStore = System;
+	type MaxLocks = ();
+	type WeightInfo = ();
+}
+parameter_types! {
+	pub const BasicDeposit: u64 = 10;
+	pub const FieldDeposit: u64 = 10;
+	pub const SubAccountDeposit: u64 = 10;
+	pub const MaxSubAccounts: u32 = 2;
+	pub const MaxAdditionalFields: u32 = 2;
+	pub const MaxRegistrars: u32 = 20;
+}
+ord_parameter_types! {
+	pub const One: u64 = 1;
+	pub const Two: u64 = 2;
+}
+type EnsureOneOrRoot = EnsureOneOf<
+	u64,
+	EnsureRoot<u64>,
+	EnsureSignedBy<One, u64>
+>;
+type EnsureTwoOrRoot = EnsureOneOf<
+	u64,
+	EnsureRoot<u64>,
+	EnsureSignedBy<Two, u64>
+>;
+impl Trait for Test {
+	type Event = ();
+	type Currency = Balances;
+	type Slashed = ();
+	type BasicDeposit = BasicDeposit;
+	type FieldDeposit = FieldDeposit;
+	type SubAccountDeposit = SubAccountDeposit;
+	type MaxSubAccounts = MaxSubAccounts;
+	type MaxAdditionalFields = MaxAdditionalFields;
+	type MaxRegistrars = MaxRegistrars;
+	type RegistrarOrigin = EnsureOneOrRoot;
+	type ForceOrigin = EnsureTwoOrRoot;
+	type WeightInfo = ();
+}
+type System = frame_system::Module<Test>;
+type Balances = pallet_balances::Module<Test>;
+type Identity = Module<Test>;
+
+pub fn new_test_ext() -> sp_io::TestExternalities {
+	let mut t = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();
+	pallet_balances::GenesisConfig::<Test> {
+		balances: vec![
+			(1, 10),
+			(2, 10),
+			(3, 10),
+			(10, 100),
+			(20, 100),
+			(30, 100),
+		],
+	}.assimilate_storage(&mut t).unwrap();
+	t.into()
+}
+
+fn ten() -> IdentityInfo {
+	IdentityInfo {
+		display: Data::Raw(b"ten".to_vec()),
+		legal: Data::Raw(b"The Right Ordinal Ten, Esq.".to_vec()),
+		.. Default::default()
+	}
+}
+
+fn twenty() -> IdentityInfo {
+	IdentityInfo {
+		display: Data::Raw(b"twenty".to_vec()),
+		legal: Data::Raw(b"The Right Ordinal Twenty, Esq.".to_vec()),
+		.. Default::default()
+	}
+}
+
+#[test]
+fn editing_subaccounts_should_work() {
+	new_test_ext().execute_with(|| {
+		let data = |x| Data::Raw(vec![x; 1]);
+
+		assert_noop!(Identity::add_sub(Origin::signed(10), 20, data(1)), Error::<Test>::NoIdentity);
+
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+
+		// first sub account
+		assert_ok!(Identity::add_sub(Origin::signed(10), 1, data(1)));
+		assert_eq!(SuperOf::<Test>::get(1), Some((10, data(1))));
+		assert_eq!(Balances::free_balance(10), 80);
+
+		// second sub account
+		assert_ok!(Identity::add_sub(Origin::signed(10), 2, data(2)));
+		assert_eq!(SuperOf::<Test>::get(1), Some((10, data(1))));
+		assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
+		assert_eq!(Balances::free_balance(10), 70);
+
+		// third sub account is too many
+		assert_noop!(Identity::add_sub(Origin::signed(10), 3, data(3)), Error::<Test>::TooManySubAccounts);
+
+		// rename first sub account
+		assert_ok!(Identity::rename_sub(Origin::signed(10), 1, data(11)));
+		assert_eq!(SuperOf::<Test>::get(1), Some((10, data(11))));
+		assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
+		assert_eq!(Balances::free_balance(10), 70);
+
+		// remove first sub account
+		assert_ok!(Identity::remove_sub(Origin::signed(10), 1));
+		assert_eq!(SuperOf::<Test>::get(1), None);
+		assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
+		assert_eq!(Balances::free_balance(10), 80);
+
+		// add third sub account
+		assert_ok!(Identity::add_sub(Origin::signed(10), 3, data(3)));
+		assert_eq!(SuperOf::<Test>::get(1), None);
+		assert_eq!(SuperOf::<Test>::get(2), Some((10, data(2))));
+		assert_eq!(SuperOf::<Test>::get(3), Some((10, data(3))));
+		assert_eq!(Balances::free_balance(10), 70);
+	});
+}
+
+#[test]
+fn resolving_subaccount_ownership_works() {
+	new_test_ext().execute_with(|| {
+		let data = |x| Data::Raw(vec![x; 1]);
+
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_ok!(Identity::set_identity(Origin::signed(20), twenty()));
+
+		// 10 claims 1 as a subaccount
+		assert_ok!(Identity::add_sub(Origin::signed(10), 1, data(1)));
+		assert_eq!(Balances::free_balance(1), 10);
+		assert_eq!(Balances::free_balance(10), 80);
+		assert_eq!(Balances::reserved_balance(10), 20);
+		// 20 cannot claim 1 now
+		assert_noop!(Identity::add_sub(Origin::signed(20), 1, data(1)), Error::<Test>::AlreadyClaimed);
+		// 1 wants to be with 20 so it quits from 10
+		assert_ok!(Identity::quit_sub(Origin::signed(1)));
+		// 1 gets the 10 that 10 paid.
+		assert_eq!(Balances::free_balance(1), 20);
+		assert_eq!(Balances::free_balance(10), 80);
+		assert_eq!(Balances::reserved_balance(10), 10);
+		// 20 can claim 1 now
+		assert_ok!(Identity::add_sub(Origin::signed(20), 1, data(1)));
+	});
+}
+
+#[test]
+fn trailing_zeros_decodes_into_default_data() {
+	let encoded = Data::Raw(b"Hello".to_vec()).encode();
+	assert!(<(Data, Data)>::decode(&mut &encoded[..]).is_err());
+	let input = &mut &encoded[..];
+	let (a, b) = <(Data, Data)>::decode(&mut AppendZerosInput::new(input)).unwrap();
+	assert_eq!(a, Data::Raw(b"Hello".to_vec()));
+	assert_eq!(b, Data::None);
+}
+
+#[test]
+fn adding_registrar_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
+		let fields = IdentityFields(IdentityField::Display | IdentityField::Legal);
+		assert_ok!(Identity::set_fields(Origin::signed(3), 0, fields));
+		assert_eq!(Identity::registrars(), vec![
+			Some(RegistrarInfo { account: 3, fee: 10, fields })
+		]);
+	});
+}
+
+#[test]
+fn amount_of_registrars_is_limited() {
+	new_test_ext().execute_with(|| {
+		for i in 1..MaxRegistrars::get() + 1 {
+			assert_ok!(Identity::add_registrar(Origin::signed(1), i as u64));
+		}
+		let last_registrar = MaxRegistrars::get() as u64 + 1;
+		assert_noop!(
+			Identity::add_registrar(Origin::signed(1), last_registrar),
+			Error::<Test>::TooManyRegistrars
+		);
+	});
+}
+
+#[test]
+fn registration_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
+		let mut three_fields = ten();
+		three_fields.additional.push(Default::default());
+		three_fields.additional.push(Default::default());
+		three_fields.additional.push(Default::default());
+		assert_noop!(
+			Identity::set_identity(Origin::signed(10), three_fields),
+			Error::<Test>::TooManyFields
+		);
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_eq!(Identity::identity(10).unwrap().info, ten());
+		assert_eq!(Balances::free_balance(10), 90);
+		assert_ok!(Identity::clear_identity(Origin::signed(10)));
+		assert_eq!(Balances::free_balance(10), 100);
+		assert_noop!(Identity::clear_identity(Origin::signed(10)), Error::<Test>::NotNamed);
+	});
+}
+
+#[test]
+fn uninvited_judgement_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_noop!(
+			Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable),
+			Error::<Test>::InvalidIndex
+		);
+
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		assert_noop!(
+			Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable),
+			Error::<Test>::InvalidTarget
+		);
+
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_noop!(
+			Identity::provide_judgement(Origin::signed(10), 0, 10, Judgement::Reasonable),
+			Error::<Test>::InvalidIndex
+		);
+		assert_noop!(
+			Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::FeePaid(1)),
+			Error::<Test>::InvalidJudgement
+		);
+
+		assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable));
+		assert_eq!(Identity::identity(10).unwrap().judgements, vec![(0, Judgement::Reasonable)]);
+	});
+}
+
+#[test]
+fn clearing_judgement_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable));
+		assert_ok!(Identity::clear_identity(Origin::signed(10)));
+		assert_eq!(Identity::identity(10), None);
+	});
+}
+
+#[test]
+fn killing_slashing_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_noop!(Identity::kill_identity(Origin::signed(1), 10), BadOrigin);
+		assert_ok!(Identity::kill_identity(Origin::signed(2), 10));
+		assert_eq!(Identity::identity(10), None);
+		assert_eq!(Balances::free_balance(10), 90);
+		assert_noop!(Identity::kill_identity(Origin::signed(2), 10), Error::<Test>::NotNamed);
+	});
+}
+
+#[test]
+fn setting_subaccounts_should_work() {
+	new_test_ext().execute_with(|| {
+		let mut subs = vec![(20, Data::Raw(vec![40; 1]))];
+		assert_noop!(Identity::set_subs(Origin::signed(10), subs.clone()), Error::<Test>::NotFound);
+
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_ok!(Identity::set_subs(Origin::signed(10), subs.clone()));
+		assert_eq!(Balances::free_balance(10), 80);
+		assert_eq!(Identity::subs_of(10), (10, vec![20]));
+		assert_eq!(Identity::super_of(20), Some((10, Data::Raw(vec![40; 1]))));
+
+		// push another item and re-set it.
+		subs.push((30, Data::Raw(vec![50; 1])));
+		assert_ok!(Identity::set_subs(Origin::signed(10), subs.clone()));
+		assert_eq!(Balances::free_balance(10), 70);
+		assert_eq!(Identity::subs_of(10), (20, vec![20, 30]));
+		assert_eq!(Identity::super_of(20), Some((10, Data::Raw(vec![40; 1]))));
+		assert_eq!(Identity::super_of(30), Some((10, Data::Raw(vec![50; 1]))));
+
+		// switch out one of the items and re-set.
+		subs[0] = (40, Data::Raw(vec![60; 1]));
+		assert_ok!(Identity::set_subs(Origin::signed(10), subs.clone()));
+		assert_eq!(Balances::free_balance(10), 70); // no change in the balance
+		assert_eq!(Identity::subs_of(10), (20, vec![40, 30]));
+		assert_eq!(Identity::super_of(20), None);
+		assert_eq!(Identity::super_of(30), Some((10, Data::Raw(vec![50; 1]))));
+		assert_eq!(Identity::super_of(40), Some((10, Data::Raw(vec![60; 1]))));
+
+		// clear
+		assert_ok!(Identity::set_subs(Origin::signed(10), vec![]));
+		assert_eq!(Balances::free_balance(10), 90);
+		assert_eq!(Identity::subs_of(10), (0, vec![]));
+		assert_eq!(Identity::super_of(30), None);
+		assert_eq!(Identity::super_of(40), None);
+
+		subs.push((20, Data::Raw(vec![40; 1])));
+		assert_noop!(Identity::set_subs(Origin::signed(10), subs.clone()), Error::<Test>::TooManySubAccounts);
+	});
+}
+
+#[test]
+fn clearing_account_should_remove_subaccounts_and_refund() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_ok!(Identity::set_subs(Origin::signed(10), vec![(20, Data::Raw(vec![40; 1]))]));
+		assert_ok!(Identity::clear_identity(Origin::signed(10)));
+		assert_eq!(Balances::free_balance(10), 100);
+		assert!(Identity::super_of(20).is_none());
+	});
+}
+
+#[test]
+fn killing_account_should_remove_subaccounts_and_not_refund() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_ok!(Identity::set_subs(Origin::signed(10), vec![(20, Data::Raw(vec![40; 1]))]));
+		assert_ok!(Identity::kill_identity(Origin::signed(2), 10));
+		assert_eq!(Balances::free_balance(10), 80);
+		assert!(Identity::super_of(20).is_none());
+	});
+}
+
+#[test]
+fn cancelling_requested_judgement_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
+		assert_noop!(Identity::cancel_request(Origin::signed(10), 0), Error::<Test>::NoIdentity);
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_ok!(Identity::request_judgement(Origin::signed(10), 0, 10));
+		assert_ok!(Identity::cancel_request(Origin::signed(10), 0));
+		assert_eq!(Balances::free_balance(10), 90);
+		assert_noop!(Identity::cancel_request(Origin::signed(10), 0), Error::<Test>::NotFound);
+
+		assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Reasonable));
+		assert_noop!(Identity::cancel_request(Origin::signed(10), 0), Error::<Test>::JudgementGiven);
+	});
+}
+
+#[test]
+fn requesting_judgement_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
+		assert_ok!(Identity::set_identity(Origin::signed(10), ten()));
+		assert_noop!(Identity::request_judgement(Origin::signed(10), 0, 9), Error::<Test>::FeeChanged);
+		assert_ok!(Identity::request_judgement(Origin::signed(10), 0, 10));
+		// 10 for the judgement request, 10 for the identity.
+		assert_eq!(Balances::free_balance(10), 80);
+
+		// Re-requesting won't work as we already paid.
+		assert_noop!(Identity::request_judgement(Origin::signed(10), 0, 10), Error::<Test>::StickyJudgement);
+		assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::Erroneous));
+		// Registrar got their payment now.
+		assert_eq!(Balances::free_balance(3), 20);
+
+		// Re-requesting still won't work as it's erroneous.
+		assert_noop!(Identity::request_judgement(Origin::signed(10), 0, 10), Error::<Test>::StickyJudgement);
+
+		// Requesting from a second registrar still works.
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 4));
+		assert_ok!(Identity::request_judgement(Origin::signed(10), 1, 10));
+
+		// Re-requesting after the judgement has been reduced works.
+		assert_ok!(Identity::provide_judgement(Origin::signed(3), 0, 10, Judgement::OutOfDate));
+		assert_ok!(Identity::request_judgement(Origin::signed(10), 0, 10));
+	});
+}
+
+#[test]
+fn field_deposit_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		assert_ok!(Identity::set_fee(Origin::signed(3), 0, 10));
+		assert_ok!(Identity::set_identity(Origin::signed(10), IdentityInfo {
+			additional: vec![
+				(Data::Raw(b"number".to_vec()), Data::Raw(10u32.encode())),
+				(Data::Raw(b"text".to_vec()), Data::Raw(b"10".to_vec())),
+			], .. Default::default()
+		}));
+		assert_eq!(Balances::free_balance(10), 70);
+	});
+}
+
+#[test]
+fn setting_account_id_should_work() {
+	new_test_ext().execute_with(|| {
+		assert_ok!(Identity::add_registrar(Origin::signed(1), 3));
+		// account 4 cannot change the first registrar's identity since it's owned by 3.
+		assert_noop!(Identity::set_account_id(Origin::signed(4), 0, 3), Error::<Test>::InvalidIndex);
+		// account 3 can, because that's the registrar's current account.
+		assert_ok!(Identity::set_account_id(Origin::signed(3), 0, 4));
+		// account 4 can now, because that's their new ID.
+		assert_ok!(Identity::set_account_id(Origin::signed(4), 0, 3));
+	});
+}
diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs
index 964c1bf5fc112..2f5fd92ff01f0 100644
--- a/utils/frame/benchmarking-cli/src/writer.rs
+++ b/utils/frame/benchmarking-cli/src/writer.rs
@@ -32,6 +32,21 @@ pub fn open_file(path: &str) -> Result<File, std::io::Error> {
 		.open(path)
 }
 
+fn underscore<Number>(i: Number) -> String
+	where Number: std::string::ToString
+{
+    let mut s = String::new();
+    let i_str = i.to_string();
+    let a = i_str.chars().rev().enumerate();
+    for (idx, val) in a {
+        if idx != 0 && idx % 3 == 0 {
+            s.insert(0, '_');
+        }
+        s.insert(0, val);
+    }
+    s
+}
+
 pub fn write_trait(file: &mut File, batches: Vec<BenchmarkBatch>) -> Result<(), std::io::Error> {
 	let mut current_pallet = Vec::<u8>::new();
 
@@ -176,10 +191,10 @@ pub fn write_results(batches: &[BenchmarkBatch]) -> Result<(), std::io::Error> {
 		// return value
 		write!(file, ") -> Weight {{\n")?;
 
-		write!(file, "\t\t({} as Weight)\n", extrinsic_time.base.saturating_mul(1000))?;
+		write!(file, "\t\t({} as Weight)\n", underscore(extrinsic_time.base.saturating_mul(1000)))?;
 		used_extrinsic_time.iter().try_for_each(|(slope, name)| -> Result<(), std::io::Error> {
 			write!(file, "\t\t\t.saturating_add(({} as Weight).saturating_mul({} as Weight))\n",
-				slope.saturating_mul(1000),
+				underscore(slope.saturating_mul(1000)),
 				name,
 			)
 		})?;

From fbf617e930b00cde22d924b7a6bd6f45fbc36d6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Fri, 18 Sep 2020 00:07:25 +0200
Subject: [PATCH 103/122] Make sure we update the `Cargo.lock` in the polkadot
 companion (#7135)

---
 .maintain/gitlab/check_polkadot_companion_build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh
index 2ee1e824aed5d..219af5001b053 100755
--- a/.maintain/gitlab/check_polkadot_companion_build.sh
+++ b/.maintain/gitlab/check_polkadot_companion_build.sh
@@ -90,4 +90,5 @@ $CARGO_HOME/bin/diener --substrate --branch $CI_COMMIT_REF_NAME --git https://gi
 cd polkadot
 
 # Test Polkadot pr or master branch with this Substrate commit.
+cargo update -p sp-io
 time cargo test --all --release --verbose

From 5a2c400c7d5a3aea831f15b6c4417f6132781fb9 Mon Sep 17 00:00:00 2001
From: Benjamin Kampmann <ben@gnunicorn.org>
Date: Fri, 18 Sep 2020 08:10:59 +0200
Subject: [PATCH 104/122] Tracing for wasm with bridging to native (#6916)

* implement events handling, implement parent_id for spans & events

* add events to sp_io::storage

* update test

* add tests

* adjust limit

* let tracing crate handle parent_ids

* re-enable current-id tracking

* add test for threads with CurrentSpan

* fix log level

* remove redundant check for non wasm traces

* remove duplicate definition in test

* Adding conditional events API

* prefer explicit parent_id over current,

enhance test

* limit changes to client::tracing event implementation

* remove From impl due to fallback required on parent_id

* make tracing codecable

* replace with global tracing

* new tracing interface

* impl TracingSubscriber in client

* implement access to global TracingSubscriber from primitives

* span for wasm

* increment towards Wasm Tracing Subscriber implementation

* increment, remove sp-tracing from runtime-interface

* increment, it compiles

* attained original functionality with new mechanism

* implement remaining TracingSubscriber functions

* remove spans from decl_module

* add handling for encoded values

* Revert "replace with global tracing"

This reverts commit 8824a60deea54d9b437407a21c8ceaf6a1902ee5.

* Wasm Side Tracing

* tracing on wasm

* enable tracing wasm on node-runtime

* export all the macros in std

* tracing subscriber on wasm-side only

* pass spans and events over and record them

* reactivate previous code and  cleanup

* further cleaning up

* extend the span macros, activate through executive

* tracking the actual extrinsic, too

* style

* fixing tests

* spaces -> tabs

* attempting to reactivate params

* activate our tests in CI

* some passing

* tests passing

* with core lazy

* global tracer for wasm side with pass over

* fixing metadata referencing

* remove const_fn feature requirement

* reenable dispatch traces

* reset client tracing

* further cleaning up

* fixing runtime-test

* move tracing-build setup into runtime-test

* Merge DebugWriter from tracing and frame-support, move to sp-std

* remove dangling fixme

* Docs for tracing primitives

* cleaning up a bit more

* Wasm interface docs

* optimise docs.rs setup

* adding tracing flags to uncomment

* remove brace

* fixing imports

* fixing broken syntax

* add required modules

* nicer formatting

* better target management

* adding low level storage tracing events into frame

* add custom Debug impl for WasmMetadata

* cloning profiler

* adding info about cloning profiler

* using in-scope for within calls

* proper time tracing, cleaning up println

* allow to disable tracing on runtime_interface-macro

* disable tracing for wasm-tracing-interface

* simplify wasm-tracing-api

* update client to new interface

* fixing docs and tests for sp-tracing

* update integration tests

* re-activating enter_span

* dropping FIXME, it's documented

* fix formatting

* fix formatting

* fix imports

* more debug info

* inform wasm about it being disabled by returning 1

* only one tracer, but enabled multi-all support

* make trait pub again for tests

* Apply suggestions from code review

Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

* fixing wasm doc tests for proper usage

* remove unnecessary import

* fixing formatting

* minor style fixes

* downgrading wabt

* update error message for UI

* Fix interface test

* next attempt to fix macros

* geee

* revert tracing on hashed for future PR

* remove local macros, use originals

* we are able to convert to static items

* implement more WasmValue types

* adding support to convert str, debug and encoded values

* more minor fixes

* revert unsafe 'static making

* fix indentation

* remove commented lines

* bump all them tracing versions

* cleaning up docs and info

* document new flag

* the new layered system handles span cloning better

* Apply suggestions from code review

Co-authored-by: David <dvdplm@gmail.com>

Co-authored-by: Matt Rutherford <mattrutherford@users.noreply.github.com>
Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>
Co-authored-by: David <dvdplm@gmail.com>
---
 .gitlab-ci.yml                                |   3 +
 Cargo.lock                                    |  42 +-
 bin/node/cli/Cargo.toml                       |   2 +-
 bin/node/runtime/Cargo.toml                   |   1 +
 client/executor/Cargo.toml                    |   2 +-
 client/executor/runtime-test/Cargo.toml       |   2 +
 client/executor/runtime-test/build.rs         |  13 +-
 client/executor/runtime-test/src/lib.rs       |   4 +-
 client/executor/src/integration_tests/mod.rs  |  49 +-
 client/service/Cargo.toml                     |   3 +-
 client/tracing/Cargo.toml                     |   3 +-
 client/tracing/src/lib.rs                     |  22 +-
 client/transaction-pool/src/api.rs            |  39 +-
 frame/executive/Cargo.toml                    |   5 +
 frame/executive/src/lib.rs                    |  62 +-
 frame/support/src/debug.rs                    |  28 +-
 frame/support/src/dispatch.rs                 |  14 +-
 frame/support/test/Cargo.toml                 |   2 +-
 primitives/arithmetic/src/biguint.rs          |   2 +-
 primitives/io/Cargo.toml                      |   9 +
 primitives/io/src/lib.rs                      | 185 ++++--
 primitives/npos-elections/src/reduce.rs       |   1 +
 .../runtime-interface/proc-macro/src/lib.rs   |  46 +-
 .../bare_function_interface.rs                |  15 +-
 .../host_function_interface.rs                |   1 -
 .../proc-macro/src/runtime_interface/mod.rs   |  10 +-
 primitives/runtime-interface/src/lib.rs       |   8 +
 primitives/runtime-interface/test/Cargo.toml  |   3 +-
 primitives/runtime-interface/test/src/lib.rs  |  25 +-
 primitives/std/src/lib.rs                     |  25 +-
 primitives/tracing/Cargo.toml                 |  27 +-
 primitives/tracing/src/lib.rs                 | 210 ++++--
 primitives/tracing/src/proxy.rs               | 165 -----
 primitives/tracing/src/types.rs               | 623 ++++++++++++++++++
 34 files changed, 1211 insertions(+), 440 deletions(-)
 delete mode 100644 primitives/tracing/src/proxy.rs
 create mode 100644 primitives/tracing/src/types.rs

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 56ac4c7f9487d..ec22c993031bf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -336,6 +336,9 @@ check-web-wasm:
     - time cargo build --target=wasm32-unknown-unknown -p sc-telemetry
     # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
     - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown -Z features=itarget
+    # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
+    - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
+    - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
     - sccache -s
 
 test-full-crypto-feature:
diff --git a/Cargo.lock b/Cargo.lock
index e80ba143d3de8..fc293472baecd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6024,27 +6024,6 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
-[[package]]
-name = "rental"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8545debe98b2b139fb04cad8618b530e9b07c152d99a5de83c860b877d67847f"
-dependencies = [
- "rental-impl",
- "stable_deref_trait",
-]
-
-[[package]]
-name = "rental-impl"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "475e68978dc5b743f2f40d8e0a8fdc83f1c5e78cbf4b8fa5e74e73beebc340de"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "retain_mut"
 version = "0.1.1"
@@ -7208,6 +7187,7 @@ dependencies = [
  "sp-runtime",
  "sp-session",
  "sp-state-machine",
+ "sp-tracing",
  "sp-transaction-pool",
  "sp-trie",
  "sp-utils",
@@ -7303,6 +7283,7 @@ dependencies = [
  "slog",
  "sp-tracing",
  "tracing",
+ "tracing-core",
  "tracing-subscriber",
 ]
 
@@ -8153,6 +8134,8 @@ dependencies = [
  "sp-tracing",
  "sp-trie",
  "sp-wasm-interface",
+ "tracing",
+ "tracing-core",
 ]
 
 [[package]]
@@ -8296,6 +8279,7 @@ dependencies = [
  "sp-runtime-interface-test-wasm-deprecated",
  "sp-state-machine",
  "tracing",
+ "tracing-core",
 ]
 
 [[package]]
@@ -8432,8 +8416,10 @@ name = "sp-tracing"
 version = "2.0.0-rc6"
 dependencies = [
  "log",
- "rental",
+ "parity-scale-codec",
+ "sp-std",
  "tracing",
+ "tracing-core",
  "tracing-subscriber",
 ]
 
@@ -9388,9 +9374,9 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
 
 [[package]]
 name = "tracing"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0aae59226cf195d8e74d4b34beae1859257efb4e5fed3f147d2dc2c7d372178"
+checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c"
 dependencies = [
  "cfg-if",
  "log",
@@ -9411,9 +9397,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-core"
-version = "0.1.12"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2734b5a028fa697686f16c6d18c2c6a3c7e41513f9a213abb6754c4acb3c8d7"
+checksum = "5bcf46c1f1f06aeea2d6b81f3c863d0930a596c86ad1920d4e5bad6dd1d7119a"
 dependencies = [
  "lazy_static",
 ]
@@ -9521,9 +9507,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
 [[package]]
 name = "trybuild"
-version = "1.0.30"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe777c4e2060f44d83892be1189f96200be8ed3d99569d5c2d5ee26e62c0ea9"
+checksum = "b7d30fe369fd650072b352b1a9cb9587669de6b89be3b8225544012c1c45292d"
 dependencies = [
  "dissimilar",
  "glob",
diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml
index fdc63f09555bb..26894a9115c89 100644
--- a/bin/node/cli/Cargo.toml
+++ b/bin/node/cli/Cargo.toml
@@ -41,7 +41,7 @@ hex-literal = "0.3.1"
 log = "0.4.8"
 rand = "0.7.2"
 structopt = { version = "0.3.8", optional = true }
-tracing = "0.1.18"
+tracing = "0.1.19"
 parking_lot = "0.10.0"
 
 # primitives
diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml
index 1195456b0fada..ed8ae0c1d4b0b 100644
--- a/bin/node/runtime/Cargo.toml
+++ b/bin/node/runtime/Cargo.toml
@@ -88,6 +88,7 @@ sp-io = { version = "2.0.0-rc6", path = "../../../primitives/io" }
 
 [features]
 default = ["std"]
+with-tracing = [ "frame-executive/with-tracing" ]
 std = [
 	"sp-authority-discovery/std",
 	"pallet-authority-discovery/std",
diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml
index a60bff877d71e..b0f420de4f068 100644
--- a/client/executor/Cargo.toml
+++ b/client/executor/Cargo.toml
@@ -46,7 +46,7 @@ test-case = "0.3.3"
 sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
 sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 sc-tracing = { version = "2.0.0-rc6", path = "../tracing" }
-tracing = "0.1.18"
+tracing = "0.1.19"
 tracing-subscriber = "0.2.10"
 
 [features]
diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml
index 037359ac9eef6..dddbb780ca808 100644
--- a/client/executor/runtime-test/Cargo.toml
+++ b/client/executor/runtime-test/Cargo.toml
@@ -29,5 +29,7 @@ std = [
 	"sp-io/std",
 	"sp-sandbox/std",
 	"sp-std/std",
+	"sp-core/std",
+	"sp-runtime/std",
 	"sp-allocator/std",
 ]
diff --git a/client/executor/runtime-test/build.rs b/client/executor/runtime-test/build.rs
index 1ed5aa44bc5c4..cf4fca01acd9f 100644
--- a/client/executor/runtime-test/build.rs
+++ b/client/executor/runtime-test/build.rs
@@ -17,10 +17,21 @@
 use wasm_builder_runner::WasmBuilder;
 
 fn main() {
+	// regular build
 	WasmBuilder::new()
 		.with_current_project()
 		.with_wasm_builder_from_crates_or_path("2.0.0", "../../../utils/wasm-builder")
 		.export_heap_base()
 		.import_memory()
-		.build()
+		.build();
+
+	// and building with tracing activated
+	WasmBuilder::new()
+		.with_current_project()
+		.with_wasm_builder_from_crates_or_path("2.0.0", "../../../utils/wasm-builder")
+		.export_heap_base()
+		.import_memory()
+		.set_file_name("wasm_binary_with_tracing.rs")
+		.append_to_rust_flags("--cfg feature=\\\"with-tracing\\\"")
+		.build();
 }
diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs
index a80ee1d6ba40f..04397afd776d7 100644
--- a/client/executor/runtime-test/src/lib.rs
+++ b/client/executor/runtime-test/src/lib.rs
@@ -254,11 +254,11 @@ sp_core::wasm_export_functions! {
 	}
 
 	fn test_enter_span() -> u64 {
-		wasm_tracing::enter_span("integration_test_span_target", "integration_test_span_name")
+		wasm_tracing::enter_span(Default::default())
 	}
 
 	fn test_exit_span(span_id: u64) {
-		wasm_tracing::exit_span(span_id)
+		wasm_tracing::exit(span_id)
 	}
 
 	fn returns_mutable_static() -> u64 {
diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs
index 1c744f544b48f..3ff676fdbe61f 100644
--- a/client/executor/src/integration_tests/mod.rs
+++ b/client/executor/src/integration_tests/mod.rs
@@ -681,7 +681,7 @@ fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) {
 	// Create subscriber with wasm_tracing disabled
 	let test_subscriber = tracing_subscriber::fmt().finish().with(
 		sc_tracing::ProfilingLayer::new_with_handler(
-			Box::new(handler), "integration_test_span_target"
+			Box::new(handler), "default"
 		)
 	);
 
@@ -690,49 +690,9 @@ fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 	let mut ext = ext.ext();
 
-	// Test tracing disabled
-	assert!(!sp_tracing::wasm_tracing_enabled());
-
-	let span_id = call_in_wasm(
-		"test_enter_span",
-		&[],
-		wasm_method,
-		&mut ext,
-	).unwrap();
-
-	assert_eq!(
-		0u64.encode(),
-		span_id
-	);
-	// Repeat to check span id always 0 when deactivated
-	let span_id = call_in_wasm(
-		"test_enter_span",
-		&[],
-		wasm_method,
-		&mut ext,
-	).unwrap();
-
-	assert_eq!(
-		0u64.encode(),
-		span_id
-	);
-
-	call_in_wasm(
-		"test_exit_span",
-		&span_id.encode(),
-		wasm_method,
-		&mut ext,
-	).unwrap();
-	// Check span has not been recorded
-	let len = traces.lock().unwrap().len();
-	assert_eq!(len, 0);
-
-	// Test tracing enabled
-	sp_tracing::set_wasm_tracing(true);
-
 	let span_id = call_in_wasm(
 		"test_enter_span",
-		&[],
+		Default::default(),
 		wasm_method,
 		&mut ext,
 	).unwrap();
@@ -756,8 +716,7 @@ fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) {
 
 	let span_datum = traces.lock().unwrap().pop().unwrap();
 	let values = span_datum.values;
-	assert_eq!(span_datum.target, "integration_test_span_target");
-	assert_eq!(span_datum.name, "integration_test_span_name");
+	assert_eq!(span_datum.target, "default");
+	assert_eq!(span_datum.name, "");
 	assert_eq!(values.bool_values.get("wasm").unwrap(), &true);
-	assert_eq!(values.bool_values.get("is_valid_trace").unwrap(), &true);
 }
diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml
index 370ee679154e3..fb9d489adf480 100644
--- a/client/service/Cargo.toml
+++ b/client/service/Cargo.toml
@@ -73,7 +73,8 @@ sc-telemetry = { version = "2.0.0-rc6", path = "../telemetry" }
 sc-offchain = { version = "2.0.0-rc6", path = "../offchain" }
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc6"}
 sc-tracing = { version = "2.0.0-rc6", path = "../tracing" }
-tracing = "0.1.18"
+sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
+tracing = "0.1.19"
 parity-util-mem = { version = "0.7.0", default-features = false, features = ["primitive-types"] }
 
 [target.'cfg(not(target_os = "unknown"))'.dependencies]
diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml
index 9444a9520f665..42d6fc5b34667 100644
--- a/client/tracing/Cargo.toml
+++ b/client/tracing/Cargo.toml
@@ -19,7 +19,8 @@ rustc-hash = "1.1.0"
 serde = "1.0.101"
 serde_json = "1.0.41"
 slog = { version = "2.5.2", features = ["nested-values"] }
-tracing = "0.1.18"
+tracing = "0.1.19"
+tracing-core = "0.1.13"
 tracing-subscriber = "0.2.10"
 sp-tracing = { version = "2.0.0-rc6", path = "../../primitives/tracing" }
 sc-telemetry = { version = "2.0.0-rc6", path = "../telemetry" }
diff --git a/client/tracing/src/lib.rs b/client/tracing/src/lib.rs
index e509f2218a2ea..6690f283464ea 100644
--- a/client/tracing/src/lib.rs
+++ b/client/tracing/src/lib.rs
@@ -40,8 +40,7 @@ use tracing::{
 use tracing_subscriber::{CurrentSpan, layer::{Layer, Context}};
 
 use sc_telemetry::{telemetry, SUBSTRATE_INFO};
-use sp_tracing::proxy::{WASM_NAME_KEY, WASM_TARGET_KEY, WASM_TRACE_IDENTIFIER};
-
+use sp_tracing::{WASM_NAME_KEY, WASM_TARGET_KEY, WASM_TRACE_IDENTIFIER};
 const ZERO_DURATION: Duration = Duration::from_nanos(0);
 
 /// Responsible for assigning ids to new spans, which are not re-used.
@@ -275,12 +274,6 @@ impl<S: Subscriber> Layer<S> for ProfilingLayer {
 	fn new_span(&self, attrs: &Attributes<'_>, id: &Id, _ctx: Context<S>) {
 		let mut values = Values::default();
 		attrs.record(&mut values);
-		// If this is a wasm trace, check if target/level is enabled
-		if let Some(wasm_target) = values.string_values.get(WASM_TARGET_KEY) {
-			if !self.check_target(wasm_target, attrs.metadata().level()) {
-				return
-			}
-		}
 		let span_datum = SpanDatum {
 			id: id.clone(),
 			parent_id: attrs.parent().cloned().or_else(|| self.current_span.id()),
@@ -327,18 +320,13 @@ impl<S: Subscriber> Layer<S> for ProfilingLayer {
 	fn on_exit(&self, span: &Id, _ctx: Context<S>) {
 		self.current_span.exit();
 		let end_time = Instant::now();
-		let mut span_data = self.span_data.lock();
-		if let Some(mut s) = span_data.get_mut(&span) {
-			s.overall_time = end_time - s.start_time + s.overall_time;
-		}
-	}
-
-	fn on_close(&self, span: Id, _ctx: Context<S>) {
 		let span_datum = {
 			let mut span_data = self.span_data.lock();
 			span_data.remove(&span)
 		};
+
 		if let Some(mut span_datum) = span_datum {
+			span_datum.overall_time += end_time - span_datum.start_time;
 			if span_datum.name == WASM_TRACE_IDENTIFIER {
 				span_datum.values.bool_values.insert("wasm".to_owned(), true);
 				if let Some(n) = span_datum.values.string_values.remove(WASM_NAME_KEY) {
@@ -355,6 +343,10 @@ impl<S: Subscriber> Layer<S> for ProfilingLayer {
 			}
 		};
 	}
+
+	fn on_close(&self, span: Id, ctx: Context<S>) {
+		self.on_exit(&span, ctx)
+	}
 }
 
 /// TraceHandler for sending span data to the logger
diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs
index c6671fd5bd7f0..853b66f6e74bb 100644
--- a/client/transaction-pool/src/api.rs
+++ b/client/transaction-pool/src/api.rs
@@ -168,23 +168,28 @@ where
 	Client::Api: TaggedTransactionQueue<Block>,
 	sp_api::ApiErrorFor<Client, Block>: Send + std::fmt::Display,
 {
-	sp_tracing::enter_span!("validate_transaction");
-	let runtime_api = client.runtime_api();
-	let has_v2 = sp_tracing::tracing_span! { "check_version";
-		runtime_api
-			.has_api_with::<dyn TaggedTransactionQueue<Block, Error=()>, _>(&at, |v| v >= 2)
-			.unwrap_or_default()
-	};
-
-	sp_tracing::enter_span!("runtime::validate_transaction");
-	let res = if has_v2 {
-		runtime_api.validate_transaction(&at, source, uxt)
-	} else {
-		#[allow(deprecated)] // old validate_transaction
-		runtime_api.validate_transaction_before_version_2(&at, uxt)
-	};
-
-	res.map_err(|e| Error::RuntimeApi(e.to_string()))
+	sp_tracing::within_span!(sp_tracing::Level::TRACE, "validate_transaction";
+	{
+		let runtime_api = client.runtime_api();
+		let has_v2 = sp_tracing::within_span! { sp_tracing::Level::TRACE, "check_version";
+			runtime_api
+				.has_api_with::<dyn TaggedTransactionQueue<Block, Error=()>, _>(&at, |v| v >= 2)
+				.unwrap_or_default()
+		};
+
+		let res = sp_tracing::within_span!(
+			sp_tracing::Level::TRACE, "runtime::validate_transaction";
+		{
+			if has_v2 {
+				runtime_api.validate_transaction(&at, source, uxt)
+			} else {
+				#[allow(deprecated)] // old validate_transaction
+				runtime_api.validate_transaction_before_version_2(&at, uxt)
+			}
+		});
+
+		res.map_err(|e| Error::RuntimeApi(e.to_string()))
+	})
 }
 
 impl<Client, Block> FullChainApi<Client, Block>
diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml
index 8114f74b8fe67..76a9251e546de 100644
--- a/frame/executive/Cargo.toml
+++ b/frame/executive/Cargo.toml
@@ -20,6 +20,7 @@ sp-runtime = { version = "2.0.0-rc6", default-features = false, path = "../../pr
 sp-tracing = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/tracing" }
 sp-std = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/std" }
 sp-io = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/io" }
+sp-core = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/core" }
 
 [dev-dependencies]
 hex-literal = "0.3.1"
@@ -32,11 +33,15 @@ sp-version = { version = "2.0.0-rc6", path = "../../primitives/version" }
 
 [features]
 default = ["std"]
+with-tracing = [
+	"sp-tracing/with-tracing"
+]
 std = [
 	"codec/std",
 	"frame-support/std",
 	"frame-system/std",
 	"serde",
+	"sp-core/std",
 	"sp-runtime/std",
 	"sp-tracing/std",
 	"sp-std/std",
diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs
index cd9642fb82c9d..1d1220d652fbf 100644
--- a/frame/executive/src/lib.rs
+++ b/frame/executive/src/lib.rs
@@ -207,6 +207,8 @@ where
 {
 	/// Start the execution of a particular block.
 	pub fn initialize_block(header: &System::Header) {
+		sp_io::init_tracing();
+		sp_tracing::enter_span!(sp_tracing::Level::TRACE, "init_block");
 		let digests = Self::extract_pre_digest(&header);
 		Self::initialize_block_impl(
 			header.number(),
@@ -270,6 +272,7 @@ where
 	}
 
 	fn initial_checks(block: &Block) {
+		sp_tracing::enter_span!(sp_tracing::Level::TRACE, "initial_checks");
 		let header = block.header();
 
 		// Check that `parent_hash` is correct.
@@ -288,23 +291,28 @@ where
 
 	/// Actually execute all transitions for `block`.
 	pub fn execute_block(block: Block) {
-		Self::initialize_block(block.header());
+		sp_io::init_tracing();
+		sp_tracing::within_span! {
+			sp_tracing::info_span!( "execute_block", ?block);
+		{
+			Self::initialize_block(block.header());
 
-		// any initial checks
-		Self::initial_checks(&block);
+			// any initial checks
+			Self::initial_checks(&block);
 
-		let signature_batching = sp_runtime::SignatureBatching::start();
+			let signature_batching = sp_runtime::SignatureBatching::start();
 
-		// execute extrinsics
-		let (header, extrinsics) = block.deconstruct();
-		Self::execute_extrinsics_with_book_keeping(extrinsics, *header.number());
+			// execute extrinsics
+			let (header, extrinsics) = block.deconstruct();
+			Self::execute_extrinsics_with_book_keeping(extrinsics, *header.number());
 
-		if !signature_batching.verify() {
-			panic!("Signature verification failed.");
-		}
+			if !signature_batching.verify() {
+				panic!("Signature verification failed.");
+			}
 
-		// any final checks
-		Self::final_checks(&header);
+			// any final checks
+			Self::final_checks(&header);
+		} };
 	}
 
 	/// Execute given extrinsics and take care of post-extrinsics book-keeping.
@@ -320,6 +328,8 @@ where
 	/// Finalize the block - it is up the caller to ensure that all header fields are valid
 	/// except state-root.
 	pub fn finalize_block() -> System::Header {
+		sp_io::init_tracing();
+		sp_tracing::enter_span!( sp_tracing::Level::TRACE, "finalize_block" );
 		<frame_system::Module<System>>::note_finished_extrinsics();
 		let block_number = <frame_system::Module<System>>::block_number();
 		<frame_system::Module<System> as OnFinalize<System::BlockNumber>>::on_finalize(block_number);
@@ -335,6 +345,7 @@ where
 	/// This doesn't attempt to validate anything regarding the block, but it builds a list of uxt
 	/// hashes.
 	pub fn apply_extrinsic(uxt: Block::Extrinsic) -> ApplyExtrinsicResult {
+		sp_io::init_tracing();
 		let encoded = uxt.encode();
 		let encoded_len = encoded.len();
 		Self::apply_extrinsic_with_len(uxt, encoded_len, Some(encoded))
@@ -355,6 +366,10 @@ where
 		encoded_len: usize,
 		to_note: Option<Vec<u8>>,
 	) -> ApplyExtrinsicResult {
+		sp_tracing::enter_span!(
+			sp_tracing::info_span!("apply_extrinsic",
+				ext=?sp_core::hexdisplay::HexDisplay::from(&uxt.encode()))
+		);
 		// Verify that the signature is good.
 		let xt = uxt.check(&Default::default())?;
 
@@ -377,6 +392,7 @@ where
 	}
 
 	fn final_checks(header: &System::Header) {
+		sp_tracing::enter_span!(sp_tracing::Level::TRACE, "final_checks");
 		// remove temporaries
 		let new_header = <frame_system::Module<System>>::finalize();
 
@@ -406,24 +422,32 @@ where
 		source: TransactionSource,
 		uxt: Block::Extrinsic,
 	) -> TransactionValidity {
-		use sp_tracing::tracing_span;
+		sp_io::init_tracing();
+		use sp_tracing::{enter_span, within_span};
 
-		sp_tracing::enter_span!("validate_transaction");
+		enter_span!{ sp_tracing::Level::TRACE, "validate_transaction" };
 
-		let encoded_len = tracing_span!{ "using_encoded"; uxt.using_encoded(|d| d.len()) };
+		let encoded_len = within_span!{ sp_tracing::Level::TRACE, "using_encoded";
+			uxt.using_encoded(|d| d.len())
+		};
 
-		let xt = tracing_span!{ "check"; uxt.check(&Default::default())? };
+		let xt = within_span!{ sp_tracing::Level::TRACE, "check";
+			uxt.check(&Default::default())
+		}?;
 
-		let dispatch_info = tracing_span!{ "dispatch_info"; xt.get_dispatch_info() };
+		let dispatch_info = within_span!{ sp_tracing::Level::TRACE, "dispatch_info";
+			xt.get_dispatch_info()
+		};
 
-		tracing_span! {
-			"validate";
+		within_span! {
+			sp_tracing::Level::TRACE, "validate";
 			xt.validate::<UnsignedValidator>(source, &dispatch_info, encoded_len)
 		}
 	}
 
 	/// Start an offchain worker and generate extrinsics.
 	pub fn offchain_worker(header: &System::Header) {
+		sp_io::init_tracing();
 		// We need to keep events available for offchain workers,
 		// hence we initialize the block manually.
 		// OffchainWorker RuntimeApi should skip initialization.
diff --git a/frame/support/src/debug.rs b/frame/support/src/debug.rs
index e4a48068460c6..86b40f1664dcf 100644
--- a/frame/support/src/debug.rs
+++ b/frame/support/src/debug.rs
@@ -87,11 +87,11 @@
 //!	native::print!("My struct: {:?}", x);
 //! ```
 
-use sp_std::vec::Vec;
 use sp_std::fmt::{self, Debug};
 
 pub use log::{info, debug, error, trace, warn};
 pub use crate::runtime_print as print;
+pub use sp_std::Writer;
 
 /// Native-only logging.
 ///
@@ -132,9 +132,9 @@ macro_rules! runtime_print {
 	($($arg:tt)+) => {
 		{
 			use core::fmt::Write;
-			let mut w = $crate::debug::Writer::default();
+			let mut w = $crate::sp_std::Writer::default();
 			let _ = core::write!(&mut w, $($arg)+);
-			w.print();
+			sp_io::misc::print_utf8(&w.inner())
 		}
 	}
 }
@@ -144,24 +144,6 @@ pub fn debug(data: &impl Debug) {
 	runtime_print!("{:?}", data);
 }
 
-/// A target for `core::write!` macro - constructs a string in memory.
-#[derive(Default)]
-pub struct Writer(Vec<u8>);
-
-impl fmt::Write for Writer {
-	fn write_str(&mut self, s: &str) -> fmt::Result {
-		self.0.extend(s.as_bytes());
-		Ok(())
-	}
-}
-
-impl Writer {
-	/// Print the content of this `Writer` out.
-	pub fn print(&self) {
-		sp_io::misc::print_utf8(&self.0)
-	}
-}
-
 /// Runtime logger implementation - `log` crate backend.
 ///
 /// The logger should be initialized if you want to display
@@ -204,13 +186,13 @@ impl log::Log for RuntimeLogger {
 
 	fn log(&self, record: &log::Record) {
 		use fmt::Write;
-		let mut w = Writer::default();
+		let mut w = sp_std::Writer::default();
 		let _ = core::write!(&mut w, "{}", record.args());
 
 		sp_io::logging::log(
 			record.level().into(),
 			record.target(),
-			&w.0,
+			w.inner(),
 		);
 	}
 
diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs
index 5446b4a59bd66..ca0a78d730b7c 100644
--- a/frame/support/src/dispatch.rs
+++ b/frame/support/src/dispatch.rs
@@ -1275,7 +1275,7 @@ macro_rules! decl_module {
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn on_initialize(_block_number_not_used: <$trait_instance as $system::Trait>::BlockNumber) -> $return {
-				$crate::sp_tracing::enter_span!("on_initialize");
+				$crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!("on_initialize"));
 				{ $( $impl )* }
 			}
 		}
@@ -1292,7 +1292,7 @@ macro_rules! decl_module {
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn on_initialize($param: $param_ty) -> $return {
-				$crate::sp_tracing::enter_span!("on_initialize");
+				$crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!("on_initialize"));
 				{ $( $impl )* }
 			}
 		}
@@ -1319,7 +1319,7 @@ macro_rules! decl_module {
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn on_runtime_upgrade() -> $return {
-				$crate::sp_tracing::enter_span!("on_runtime_upgrade");
+				$crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!("on_runtime_upgrade"));
 				{ $( $impl )* }
 			}
 		}
@@ -1375,7 +1375,7 @@ macro_rules! decl_module {
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn on_finalize(_block_number_not_used: <$trait_instance as $system::Trait>::BlockNumber) {
-				$crate::sp_tracing::enter_span!("on_finalize");
+				$crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!("on_finalize"));
 				{ $( $impl )* }
 			}
 		}
@@ -1392,7 +1392,7 @@ macro_rules! decl_module {
 			for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn on_finalize($param: $param_ty) {
-				$crate::sp_tracing::enter_span!("on_finalize");
+				$crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!("on_finalize"));
 				{ $( $impl )* }
 			}
 		}
@@ -1465,7 +1465,7 @@ macro_rules! decl_module {
 		$vis fn $name(
 			$origin: $origin_ty $(, $param: $param_ty )*
 		) -> $crate::dispatch::DispatchResult {
-			$crate::sp_tracing::enter_span!(stringify!($name));
+			$crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!(stringify!($name)));
 			{ $( $impl )* }
 			Ok(())
 		}
@@ -1484,7 +1484,7 @@ macro_rules! decl_module {
 	) => {
 		$(#[$fn_attr])*
 		$vis fn $name($origin: $origin_ty $(, $param: $param_ty )* ) -> $result {
-			$crate::sp_tracing::enter_span!(stringify!($name));
+			$crate::sp_tracing::enter_span!($crate::sp_tracing::trace_span!(stringify!($name)));
 			$( $impl )*
 		}
 	};
diff --git a/frame/support/test/Cargo.toml b/frame/support/test/Cargo.toml
index f2f70fb95279e..7c839e4462ad1 100644
--- a/frame/support/test/Cargo.toml
+++ b/frame/support/test/Cargo.toml
@@ -21,7 +21,7 @@ sp-inherents = { version = "2.0.0-rc6", default-features = false, path = "../../
 sp-runtime = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/runtime" }
 sp-core = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/core" }
 sp-std = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/std" }
-trybuild = "1.0.17"
+trybuild = "1.0.33"
 pretty_assertions = "0.6.1"
 rustversion = "1.0.0"
 
diff --git a/primitives/arithmetic/src/biguint.rs b/primitives/arithmetic/src/biguint.rs
index 41e2c759a5967..1fed54f598eec 100644
--- a/primitives/arithmetic/src/biguint.rs
+++ b/primitives/arithmetic/src/biguint.rs
@@ -18,7 +18,7 @@
 //! Infinite precision unsigned integer for substrate runtime.
 
 use num_traits::Zero;
-use sp_std::{cmp::Ordering, ops, prelude::*, cell::RefCell, convert::TryFrom};
+use sp_std::{cmp::Ordering, ops, prelude::*, vec, cell::RefCell, convert::TryFrom};
 
 // A sensible value for this would be half of the dword size of the host machine. Since the
 // runtime is compiled to 32bit webassembly, using 32 and 64 for single and double respectively
diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml
index a08451db24389..06672190e7fe3 100644
--- a/primitives/io/Cargo.toml
+++ b/primitives/io/Cargo.toml
@@ -28,6 +28,8 @@ sp-tracing = { version = "2.0.0-rc6", default-features = false, path = "../traci
 log = { version = "0.4.8", optional = true }
 futures = { version = "0.3.1", features = ["thread-pool"], optional = true }
 parking_lot = { version = "0.10.0", optional = true }
+tracing = { version = "0.1.19", default-features = false }
+tracing-core = { version = "0.1.15", default-features = false}
 
 [features]
 default = ["std"]
@@ -42,11 +44,18 @@ std = [
 	"sp-runtime-interface/std",
 	"sp-externalities",
 	"sp-wasm-interface/std",
+	"sp-tracing/std",
+	"tracing/std",
+	"tracing-core/std",
 	"log",
 	"futures",
 	"parking_lot",
 ]
 
+with-tracing = [
+	"sp-tracing/with-tracing"
+]
+
 # These two features are used for `no_std` builds for the environments which already provides
 # `#[panic_handler]`, `#[alloc_error_handler]` and `#[global_allocator]`.
 #
diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs
index 9c4a0c59b51c2..2e7cb3e7efad9 100644
--- a/primitives/io/src/lib.rs
+++ b/primitives/io/src/lib.rs
@@ -32,6 +32,9 @@ use sp_std::vec::Vec;
 #[cfg(feature = "std")]
 use sp_std::ops::Deref;
 
+#[cfg(feature = "std")]
+use tracing;
+
 #[cfg(feature = "std")]
 use sp_core::{
 	crypto::Pair,
@@ -52,6 +55,7 @@ use sp_core::{
 use sp_trie::{TrieConfiguration, trie_types::Layout};
 
 use sp_runtime_interface::{runtime_interface, Pointer};
+use sp_runtime_interface::pass_by::PassBy;
 
 use codec::{Encode, Decode};
 
@@ -1004,55 +1008,156 @@ pub trait Logging {
 	}
 }
 
-#[cfg(feature = "std")]
-sp_externalities::decl_extension! {
-	/// Extension to allow running traces in wasm via Proxy
-	pub struct TracingProxyExt(sp_tracing::proxy::TracingProxy);
+#[derive(Encode, Decode)]
+/// Crossing is a helper wrapping any Encode-Decodeable type
+/// for transferring over the wasm barrier.
+pub struct Crossing<T: Encode + Decode>(T);
+
+impl<T: Encode + Decode> PassBy for Crossing<T> {
+	type PassBy = sp_runtime_interface::pass_by::Codec<Self>;
 }
 
-/// Interface that provides functions for profiling the runtime.
-#[runtime_interface]
+impl<T: Encode + Decode> Crossing<T> {
+
+	/// Convert into the inner type
+	pub fn into_inner(self) -> T {
+		self.0
+	}
+}
+
+// useful for testing
+impl<T> core::default::Default for Crossing<T>
+	where T: core::default::Default + Encode + Decode
+{
+	fn default() -> Self {
+		Self(Default::default())
+	}
+
+}
+
+/// Interface to provide tracing facilities for wasm. Modelled after tokios `tracing`-crate
+/// interfaces. See `sp-tracing` for more information.
+#[runtime_interface(wasm_only, no_tracing)]
 pub trait WasmTracing {
-	/// To create and enter a `tracing` span, using `sp_tracing::proxy`
-	/// Returns 0 value to indicate that no further traces should be attempted
-	fn enter_span(&mut self, target: &str, name: &str) -> u64 {
-		if sp_tracing::wasm_tracing_enabled() {
-			match self.extension::<TracingProxyExt>() {
-				Some(proxy) => return proxy.enter_span(target, name),
-				None => {
-					if self.register_extension(TracingProxyExt(sp_tracing::proxy::TracingProxy::new())).is_ok() {
-						if let Some(proxy) = self.extension::<TracingProxyExt>() {
-							return proxy.enter_span(target, name);
-						}
-					} else {
-						log::warn!(
-							target: "tracing",
-							"Unable to register extension: TracingProxyExt"
-						);
-					}
-				}
+	/// Whether the span described in `WasmMetadata` should be traced wasm-side
+	/// On the host converts into a static Metadata and checks against the global `tracing` dispatcher.
+	///
+	/// When returning false the calling code should skip any tracing-related execution. In general
+	/// within the same block execution this is not expected to change and it doesn't have to be
+	/// checked more than once per metadata. This exists for optimisation purposes but is still not
+	/// cheap as it will jump the wasm-native-barrier every time it is called. So an implementation might
+	/// chose to cache the result for the execution of the entire block.
+	fn enabled(&mut self, metadata: Crossing<sp_tracing::WasmMetadata>) -> bool {
+		let metadata: &tracing_core::metadata::Metadata<'static> = (&metadata.into_inner()).into();
+		tracing::dispatcher::get_default(|d| {
+			d.enabled(metadata)
+		})
+	}
+
+	/// Open a new span with the given attributes. Return the u64 Id of the span.
+	///
+	/// On the native side this goes through the default `tracing` dispatcher to register the span
+	/// and then calls `clone_span` with the ID to signal that we are keeping it around on the wasm-
+	/// side even after the local span is dropped. The resulting ID is then handed over to the wasm-
+	/// side.
+	fn enter_span(&mut self, span: Crossing<sp_tracing::WasmEntryAttributes>) -> u64 {
+		let span: tracing::Span = span.into_inner().into();
+		match span.id() {
+			Some(id) => tracing::dispatcher::get_default(|d| {
+				// inform dispatch that we'll keep the ID around
+				// then enter it immediately
+				let final_id = d.clone_span(&id);
+				d.enter(&final_id);
+				final_id.into_u64()
+			}),
+			_ => {
+				0
 			}
 		}
-		log::debug!(
-			target: "tracing",
-			"Notify to runtime that tracing is disabled."
-		);
-		0
-	}
-
-	/// Exit a `tracing` span, using `sp_tracing::proxy`
-	fn exit_span(&mut self, id: u64) {
-		if let Some(proxy) = self.extension::<TracingProxyExt>() {
-			proxy.exit_span(id)
-		} else {
-			log::warn!(
-				target: "tracing",
-				"Unable to load extension: TracingProxyExt"
-			);
+	}
+
+	/// Emit the given event to the global tracer on the native side
+	fn event(&mut self, event: Crossing<sp_tracing::WasmEntryAttributes>) {
+		event.into_inner().emit();
+	}
+
+	/// Signal that a given span-id has been exited. On native, this directly
+	/// proxies the span to the global dispatcher.
+	fn exit(&mut self, span: u64) {
+		tracing::dispatcher::get_default(|d| {
+			let id = tracing_core::span::Id::from_u64(span);
+			d.exit(&id);
+		});
+	}
+}
+
+#[cfg(all(not(feature="std"), feature="with-tracing"))]
+mod tracing_setup {
+	use core::sync::atomic::{AtomicBool, Ordering};
+	use tracing_core::{
+		dispatcher::{Dispatch, set_global_default},
+		span::{Id, Record, Attributes},
+		Metadata, Event,
+	};
+	use super::{wasm_tracing, Crossing};
+
+	const TRACING_SET : AtomicBool = AtomicBool::new(false);
+
+
+	/// The PassingTracingSubscriber implements `tracing_core::Subscriber`
+	/// and pushes the information across the runtime interface to the host
+	struct PassingTracingSubsciber;
+
+	impl tracing_core::Subscriber for PassingTracingSubsciber {
+		fn enabled(&self, metadata: &Metadata<'_>) -> bool {
+			wasm_tracing::enabled(Crossing(metadata.into()))
+		}
+		fn new_span(&self, attrs: &Attributes<'_>) -> Id {
+			Id::from_u64(wasm_tracing::enter_span(Crossing(attrs.into())))
+		}
+		fn enter(&self, span: &Id) {
+			// Do nothing, we already entered the span previously
+		}
+		/// Not implemented! We do not support recording values later
+		/// Will panic when used.
+		fn record(&self, span: &Id, values: &Record<'_>) {
+			unimplemented!{} // this usage is not supported
+		}
+		/// Not implemented! We do not support recording values later
+		/// Will panic when used.
+		fn record_follows_from(&self, span: &Id, follows: &Id) {
+			unimplemented!{ } // this usage is not supported
+		}
+		fn event(&self, event: &Event<'_>) {
+			wasm_tracing::event(Crossing(event.into()))
+		}
+		fn exit(&self, span: &Id) {
+			wasm_tracing::exit(span.into_u64())
+		}
+	}
+
+
+	/// Initialize tracing of sp_tracing on wasm with `with-tracing` enabled.
+	/// Can be called multiple times from within the same process and will only
+	/// set the global bridging subscriber once.
+	pub fn init_tracing() {
+		if TRACING_SET.load(Ordering::Relaxed) == false {
+			set_global_default(Dispatch::new(PassingTracingSubsciber {}))
+				.expect("We only ever call this once");
+			TRACING_SET.store(true, Ordering::Relaxed);
 		}
 	}
 }
 
+#[cfg(not(all(not(feature="std"), feature="with-tracing")))]
+mod tracing_setup {
+	/// Initialize tracing of sp_tracing not necessary – noop. To enable build
+	/// without std and with the `with-tracing`-feature.
+	pub fn init_tracing() { }
+}
+
+pub use tracing_setup::init_tracing;
+
 /// Wasm-only interface that provides functions for interacting with the sandbox.
 #[runtime_interface(wasm_only)]
 pub trait Sandbox {
diff --git a/primitives/npos-elections/src/reduce.rs b/primitives/npos-elections/src/reduce.rs
index 6d458a5fffb38..17d7dd1290f7d 100644
--- a/primitives/npos-elections/src/reduce.rs
+++ b/primitives/npos-elections/src/reduce.rs
@@ -52,6 +52,7 @@ use crate::{ExtendedBalance, IdentifierT, StakedAssignment};
 use sp_arithmetic::traits::{Bounded, Zero};
 use sp_std::{
 	collections::btree_map::{BTreeMap, Entry::*},
+	vec,
 	prelude::*,
 };
 
diff --git a/primitives/runtime-interface/proc-macro/src/lib.rs b/primitives/runtime-interface/proc-macro/src/lib.rs
index 2f5b9de1c14e7..df43551398a12 100644
--- a/primitives/runtime-interface/proc-macro/src/lib.rs
+++ b/primitives/runtime-interface/proc-macro/src/lib.rs
@@ -26,21 +26,59 @@
 //! 3. The [`PassByEnum`](derive.PassByInner.html) derive macro for implementing `PassBy` with `Enum`.
 //! 4. The [`PassByInner`](derive.PassByInner.html) derive macro for implementing `PassBy` with `Inner`.
 
-use syn::{parse_macro_input, ItemTrait, DeriveInput};
+use syn::{parse_macro_input, ItemTrait, DeriveInput, Result, Token};
+use syn::parse::{Parse, ParseStream};
 
 mod pass_by;
 mod runtime_interface;
 mod utils;
 
+struct Options {
+	wasm_only: bool,
+	tracing: bool
+}
+
+impl Options {
+	fn unpack(self) -> (bool, bool) {
+		(self.wasm_only, self.tracing)
+	}
+}
+impl Default for Options {
+	fn default() -> Self {
+		Options { wasm_only: false, tracing: true }
+	}
+}
+
+impl Parse for Options {
+	fn parse(input: ParseStream) -> Result<Self> {
+		let mut res = Self::default();
+		while !input.is_empty() {
+			let lookahead = input.lookahead1();
+			if lookahead.peek(runtime_interface::keywords::wasm_only) {
+				let _ = input.parse::<runtime_interface::keywords::wasm_only>();
+				res.wasm_only = true;
+			} else if lookahead.peek(runtime_interface::keywords::no_tracing) {
+				let _ = input.parse::<runtime_interface::keywords::no_tracing>();
+				res.tracing = false;
+			} else if lookahead.peek(Token![,]) {
+				let _ = input.parse::<Token![,]>();
+			} else {
+				return Err(lookahead.error())
+			}
+		}
+		Ok(res)
+	}
+}
+
 #[proc_macro_attribute]
 pub fn runtime_interface(
 	attrs: proc_macro::TokenStream,
 	input: proc_macro::TokenStream,
 ) -> proc_macro::TokenStream {
 	let trait_def = parse_macro_input!(input as ItemTrait);
-	let wasm_only = parse_macro_input!(attrs as Option<runtime_interface::keywords::wasm_only>);
+	let (wasm_only, tracing) = parse_macro_input!(attrs as Options).unpack();
 
-	runtime_interface::runtime_interface_impl(trait_def, wasm_only.is_some())
+	runtime_interface::runtime_interface_impl(trait_def, wasm_only, tracing)
 		.unwrap_or_else(|e| e.to_compile_error())
 		.into()
 }
@@ -61,4 +99,4 @@ pub fn pass_by_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream
 pub fn pass_by_enum(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
 	let input = parse_macro_input!(input as DeriveInput);
 	pass_by::enum_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into()
-}
+}
\ No newline at end of file
diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs
index 6760e9656113a..2725bd2c89ce5 100644
--- a/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs
+++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs
@@ -46,7 +46,7 @@ use std::iter;
 
 /// Generate one bare function per trait method. The name of the bare function is equal to the name
 /// of the trait method.
-pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
+pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool, tracing: bool) -> Result<TokenStream> {
 	let trait_name = &trait_def.ident;
 	let runtime_interface = get_runtime_interface(trait_def)?;
 
@@ -63,7 +63,7 @@ pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream
 	// earlier versions compatibility dispatch (only std variant)
 	let result: Result<TokenStream> = runtime_interface.all_versions().try_fold(token_stream?, |mut t, (version, method)|
 	{
-		t.extend(function_std_impl(trait_name, method, version, is_wasm_only)?);
+		t.extend(function_std_impl(trait_name, method, version, is_wasm_only, tracing)?);
 		Ok(t)
 	});
 
@@ -145,6 +145,7 @@ fn function_std_impl(
 	method: &TraitItemMethod,
 	version: u32,
 	is_wasm_only: bool,
+	tracing: bool,
 ) -> Result<TokenStream> {
 	let function_name = create_function_ident_with_version(&method.sig.ident, version);
 	let function_name_str = function_name.to_string();
@@ -168,13 +169,21 @@ fn function_std_impl(
 	let attrs = method.attrs.iter().filter(|a| !a.path.is_ident("version"));
 	// Don't make the function public accessible when this is a wasm only interface.
 	let call_to_trait = generate_call_to_trait(trait_name, method, version, is_wasm_only);
+	let call_to_trait = if !tracing {
+		call_to_trait
+	} else {
+		parse_quote!(
+			#crate_::sp_tracing::within_span! { #crate_::sp_tracing::trace_span!(#function_name_str);
+				#call_to_trait
+			}
+		)
+	};
 
 	Ok(
 		quote_spanned! { method.span() =>
 			#[cfg(feature = "std")]
 			#( #attrs )*
 			fn #function_name( #( #args, )* ) #return_value {
-				#crate_::sp_tracing::enter_span!(#function_name_str);
 				#call_to_trait
 			}
 		}
diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs
index 721eed649c25d..7a4dbc5773a28 100644
--- a/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs
+++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs
@@ -227,7 +227,6 @@ fn generate_host_function_implementation(
 						__function_context__: &mut dyn #crate_::sp_wasm_interface::FunctionContext,
 						args: &mut dyn Iterator<Item = #crate_::sp_wasm_interface::Value>,
 					) -> std::result::Result<Option<#crate_::sp_wasm_interface::Value>, String> {
-						#crate_::sp_tracing::enter_span!(#name);
 						#( #wasm_to_ffi_values )*
 						#( #ffi_to_host_values )*
 						#host_function_call
diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs
index c9b6edf68fd5a..02c291975738c 100644
--- a/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs
+++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs
@@ -33,14 +33,20 @@ mod trait_decl_impl;
 pub mod keywords {
 	// Custom keyword `wasm_only` that can be given as attribute to [`runtime_interface`].
 	syn::custom_keyword!(wasm_only);
+	// Disable tracing-macros added to the [`runtime_interface`] by specifying this optional entry
+	syn::custom_keyword!(no_tracing);
 }
 
 /// Implementation of the `runtime_interface` attribute.
 ///
 /// It expects the trait definition the attribute was put above and if this should be an wasm only
 /// interface.
-pub fn runtime_interface_impl(trait_def: ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
-	let bare_functions = bare_function_interface::generate(&trait_def, is_wasm_only)?;
+pub fn runtime_interface_impl(
+	trait_def: ItemTrait,
+	is_wasm_only: bool,
+	tracing: bool,
+) -> Result<TokenStream> {
+	let bare_functions = bare_function_interface::generate(&trait_def, is_wasm_only, tracing)?;
 	let crate_include = generate_runtime_interface_include();
 	let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site());
 	let trait_decl_impl = trait_decl_impl::process(&trait_def, is_wasm_only)?;
diff --git a/primitives/runtime-interface/src/lib.rs b/primitives/runtime-interface/src/lib.rs
index 562f94b278efc..2273e453f104a 100644
--- a/primitives/runtime-interface/src/lib.rs
+++ b/primitives/runtime-interface/src/lib.rs
@@ -284,6 +284,14 @@ pub use sp_std;
 /// 1. The generated functions are not callable from the native side.
 /// 2. The trait as shown above is not implemented for `Externalities` and is instead implemented
 ///    for `FunctionExecutor` (from `sp-wasm-interface`).
+///
+/// # Disable tracing
+/// By addding `no_tracing` to the list of options you can prevent the wasm-side interface from
+/// generating the default `sp-tracing`-calls. Note that this is rarely needed but only meant for
+/// the case when that would create a circular dependency. You usually _do not_ want to add this
+/// flag, as tracing doesn't cost you anything by default anyways (it is added as a no-op) but is
+/// super useful for debugging later.
+///
 pub use sp_runtime_interface_proc_macro::runtime_interface;
 
 #[doc(hidden)]
diff --git a/primitives/runtime-interface/test/Cargo.toml b/primitives/runtime-interface/test/Cargo.toml
index 39a48d10b141e..eb916da245a19 100644
--- a/primitives/runtime-interface/test/Cargo.toml
+++ b/primitives/runtime-interface/test/Cargo.toml
@@ -20,4 +20,5 @@ sp-state-machine = { version = "0.8.0-rc6", path = "../../../primitives/state-ma
 sp-runtime = { version = "2.0.0-rc6", path = "../../runtime" }
 sp-core = { version = "2.0.0-rc6", path = "../../core" }
 sp-io = { version = "2.0.0-rc6", path = "../../io" }
-tracing = "0.1.18"
+tracing = "0.1.19"
+tracing-core = "0.1.15"
diff --git a/primitives/runtime-interface/test/src/lib.rs b/primitives/runtime-interface/test/src/lib.rs
index c213c977829e6..c66609daa2f29 100644
--- a/primitives/runtime-interface/test/src/lib.rs
+++ b/primitives/runtime-interface/test/src/lib.rs
@@ -18,8 +18,6 @@
 //! Integration tests for runtime interface primitives
 #![cfg(test)]
 
-#![cfg(test)]
-
 use sp_runtime_interface::*;
 
 use sp_runtime_interface_test_wasm::{wasm_binary_unwrap, test_api::HostFunctions};
@@ -157,14 +155,26 @@ fn test_versionining_with_new_host_works() {
 
 #[test]
 fn test_tracing() {
-	use tracing::span::Id as SpanId;
+	use std::fmt;
+	use tracing::{span::Id as SpanId};
+	use tracing_core::field::{Field, Visit};
 
 	#[derive(Clone)]
 	struct TracingSubscriber(Arc<Mutex<Inner>>);
 
+	struct FieldConsumer(&'static str, Option<String>);
+	impl Visit for FieldConsumer {
+
+		fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
+			if  field.name() == self.0 {
+				self.1 = Some(format!("{:?}", value))
+			}
+		}
+	}
+
 	#[derive(Default)]
 	struct Inner {
-		spans: HashSet<&'static str>,
+		spans: HashSet<String>,
 	}
 
 	impl tracing::subscriber::Subscriber for TracingSubscriber {
@@ -173,7 +183,9 @@ fn test_tracing() {
 		fn new_span(&self, span: &tracing::span::Attributes) -> tracing::Id {
 			let mut inner = self.0.lock().unwrap();
 			let id = SpanId::from_u64((inner.spans.len() + 1) as _);
-			inner.spans.insert(span.metadata().name());
+			let mut f = FieldConsumer("name", None);
+			span.record(&mut f);
+			inner.spans.insert(f.1.unwrap_or_else(||span.metadata().name().to_owned()));
 			id
 		}
 
@@ -196,5 +208,4 @@ fn test_tracing() {
 
 	let inner = subscriber.0.lock().unwrap();
 	assert!(inner.spans.contains("return_input_version_1"));
-	assert!(inner.spans.contains("ext_test_api_return_input_version_1"));
-}
+}
\ No newline at end of file
diff --git a/primitives/std/src/lib.rs b/primitives/std/src/lib.rs
index 8ff1efc63d8df..b323c43720da1 100644
--- a/primitives/std/src/lib.rs
+++ b/primitives/std/src/lib.rs
@@ -20,7 +20,6 @@
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
-
 #![cfg_attr(feature = "std",
    doc = "Substrate runtime standard library as compiled when linked with Rust's standard library.")]
 #![cfg_attr(not(feature = "std"),
@@ -65,6 +64,30 @@ include!("../with_std.rs");
 #[cfg(not(feature = "std"))]
 include!("../without_std.rs");
 
+
+/// A target for `core::write!` macro - constructs a string in memory.
+#[derive(Default)]
+pub struct Writer(vec::Vec<u8>);
+
+impl fmt::Write for Writer {
+	fn write_str(&mut self, s: &str) -> fmt::Result {
+		self.0.extend(s.as_bytes());
+		Ok(())
+	}
+}
+
+impl Writer {
+	/// Access the content of this `Writer` e.g. for printout
+	pub fn inner(&self) -> &vec::Vec<u8> {
+		&self.0
+	}
+
+	/// Convert into the content of this `Writer`
+	pub fn into_inner(self) -> vec::Vec<u8> {
+		self.0
+	}
+}
+
 /// Prelude of common useful imports.
 ///
 /// This should include only things which are in the normal std prelude.
diff --git a/primitives/tracing/Cargo.toml b/primitives/tracing/Cargo.toml
index 889d116221b5f..98a57bafe5ef9 100644
--- a/primitives/tracing/Cargo.toml
+++ b/primitives/tracing/Cargo.toml
@@ -9,14 +9,33 @@ repository = "https://github.com/paritytech/substrate/"
 description = "Instrumentation primitives and macros for Substrate."
 
 [package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]
+# let's default to wasm32
+default-target =  "wasm32-unknown-unknown"
+# with the tracing enabled
+features = ["with-tracing"]
+# allowing for linux-gnu here, too, allows for `std` to show up as well
+targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"]
 
 [dependencies]
-tracing = { version = "0.1.18", optional = true }
-rental = { version = "0.5.5", optional = true }
+sp-std = { version = "2.0.0-rc6", path = "../std", default-features = false}
+codec = { version = "1.3.1", package = "parity-scale-codec", default-features = false, features = ["derive"]}
+tracing = { version = "0.1.19", default-features = false }
+tracing-core = { version = "0.1.16", default-features = false }
 log = { version = "0.4.8", optional = true }
 tracing-subscriber = { version = "0.2.10", optional = true, features = ["tracing-log"] }
 
 [features]
 default = [ "std" ]
-std = [ "tracing", "rental", "log", "tracing-subscriber" ]
+with-tracing = [
+    "codec/derive",
+    "codec/full",
+]
+std = [
+    "with-tracing",
+    "tracing/std",
+    "tracing-core/std",
+    "codec/std",
+    "sp-std/std",
+    "log",
+    "tracing-subscriber",
+]
diff --git a/primitives/tracing/src/lib.rs b/primitives/tracing/src/lib.rs
index ec692b90dfdec..fb074d5579c82 100644
--- a/primitives/tracing/src/lib.rs
+++ b/primitives/tracing/src/lib.rs
@@ -17,7 +17,7 @@
 
 //! Substrate tracing primitives and macros.
 //!
-//! To trace functions or invidual code in Substrate, this crate provides [`tracing_span`]
+//! To trace functions or invidual code in Substrate, this crate provides [`within_span`]
 //! and [`enter_span`]. See the individual docs for how to use these macros.
 //!
 //! Note that to allow traces from wasm execution environment there are
@@ -28,21 +28,80 @@
 //! Additionally, we have a const: `WASM_TRACE_IDENTIFIER`, which holds a span name used
 //! to signal that the 'actual' span name and target should be retrieved instead from
 //! the associated Fields mentioned above.
+
 #![cfg_attr(not(feature = "std"), no_std)]
 
-#[cfg(feature = "std")]
-#[macro_use]
-extern crate rental;
+/// Tracing facilities and helpers.
+///
+/// This is modeled after the `tracing`/`tracing-core` interface and uses that more or
+/// less directly for the native side. Because of certain optimisations the these crates
+/// have done, the wasm implementation diverges slightly and is optimised for thtat use
+/// case (like being able to cross the wasm/native boundary via scale codecs).
+///
+/// One of said optimisations is that all macros will yield to a `noop` in non-std unless
+/// the `with-tracing` feature is explicitly activated. This allows you to just use the
+/// tracing wherever you deem fit and without any performance impact by default. Only if
+/// the specific `with-tracing`-feature is activated on this crate will it actually include
+/// the tracing code in the non-std environment.
+///
+/// Because of that optimisation, you should not use the `span!` and `span_*!` macros
+/// directly as they yield nothing without the feature present. Instead you should use
+/// `enter_span!` and `within_span!` – which would strip away even any parameter conversion
+/// you do within the span-definition (and thus optimise your performance). For your
+/// convineience you directly specify the `Level` and name of the span or use the full
+/// feature set of `span!`/`span_*!` on it:
+///
+/// # Example
+///
+/// ```rust
+/// sp_tracing::enter_span!(sp_tracing::Level::TRACE, "fn wide span");
+/// {
+///		sp_tracing::enter_span!(sp_tracing::trace_span!("outer-span"));
+///		{
+///			sp_tracing::enter_span!(sp_tracing::Level::TRACE, "inner-span");
+///			// ..
+///		}  // inner span exists here
+///	} // outer span exists here
+///
+/// sp_tracing::within_span! {
+///		sp_tracing::debug_span!("debug-span", you_can_pass="any params");
+///     1 + 1;
+///     // some other complex code
+/// } // debug span ends here
+///
+/// ```
+///
+///
+/// # Setup
+///
+/// This project only provides the macros and facilities to manage tracing
+/// it doesn't implement the tracing subscriber or backend directly – that is
+/// up to the developer integrating it into a specific environment. In native
+/// this can and must be done through the regular `tracing`-facitilies, please
+/// see their documentation for details.
+///
+/// On the wasm-side we've adopted a similar approach of having a global
+/// `TracingSubscriber` that the macros call and that does the actual work
+/// of tracking. To provide your tracking, you must implement `TracingSubscriber`
+/// and call `set_tracing_subscriber` at the very beginning of your execution –
+/// the default subscriber is doing nothing, so any spans or events happening before
+/// will not be recorded!
+///
 
-#[cfg(feature = "std")]
-#[doc(hidden)]
-pub use tracing;
+mod types;
 
 #[cfg(feature = "std")]
-pub mod proxy;
+use tracing;
+
+pub use tracing::{
+	debug, debug_span, error, error_span, info, info_span, trace, trace_span, warn, warn_span,
+	span, event, Level, Span,
+};
+
+pub use crate::types::{
+	WasmMetadata, WasmEntryAttributes, WasmValuesSet, WasmValue, WasmFields, WasmLevel, WasmFieldName
+};
 
-#[cfg(feature = "std")]
-use std::sync::atomic::{AtomicBool, Ordering};
 
 /// Try to init a simple tracing subscriber with log compatibility layer.
 /// Ignores any error. Useful for testing.
@@ -51,74 +110,127 @@ pub fn try_init_simple() {
 	let _ = tracing_subscriber::fmt().with_writer(std::io::stderr).try_init();
 }
 
-/// Flag to signal whether to run wasm tracing
 #[cfg(feature = "std")]
-static WASM_TRACING_ENABLED: AtomicBool = AtomicBool::new(false);
+pub use crate::types::{
+	WASM_NAME_KEY, WASM_TARGET_KEY, WASM_TRACE_IDENTIFIER
+};
+
 
 /// Runs given code within a tracing span, measuring it's execution time.
 ///
-/// If tracing is not enabled, the code is still executed.
+/// If tracing is not enabled, the code is still executed. Pass in level and name or
+/// use any valid `sp_tracing::Span`followe by `;` and the code to execute,
 ///
 /// # Example
 ///
 /// ```
-/// sp_tracing::tracing_span! {
+/// sp_tracing::within_span! {
+///		sp_tracing::Level::TRACE,
 ///     "test-span";
 ///     1 + 1;
 ///     // some other complex code
 /// }
+///
+/// sp_tracing::within_span! {
+///		sp_tracing::span!(sp_tracing::Level::WARN, "warn-span", you_can_pass="any params");
+///     1 + 1;
+///     // some other complex code
+/// }
+///
+/// sp_tracing::within_span! {
+///		sp_tracing::debug_span!("debug-span", you_can_pass="any params");
+///     1 + 1;
+///     // some other complex code
+/// }
 /// ```
+#[cfg(any(feature = "std", feature = "with-tracing"))]
 #[macro_export]
-macro_rules! tracing_span {
+macro_rules! within_span {
+	(
+		$span:expr;
+		$( $code:tt )*
+	) => {
+		$span.in_scope(||
+			{
+				$( $code )*
+			}
+		)
+	};
 	(
+		$lvl:expr,
 		$name:expr;
 		$( $code:tt )*
 	) => {
 		{
-			$crate::enter_span!($name);
-			$( $code )*
+			$crate::within_span!($crate::span!($crate::Level::TRACE, $name); $( $code )*)
 		}
-	}
+	};
+}
+
+#[cfg(all(not(feature = "std"), not(feature = "with-tracing")))]
+#[macro_export]
+macro_rules! within_span {
+	(
+		$span:stmt;
+		$( $code:tt )*
+	) => {
+		$( $code )*
+	};
+	(
+		$lvl:expr,
+		$name:expr;
+		$( $code:tt )*
+	) => {
+		$( $code )*
+	};
+}
+
+
+/// Enter a span - noop for `no_std` without `with-tracing`
+#[cfg(all(not(feature = "std"), not(feature = "with-tracing")))]
+#[macro_export]
+macro_rules! enter_span {
+	( $lvl:expr, $name:expr ) => ( );
+	( $name:expr ) => ( ) // no-op
 }
 
 /// Enter a span.
 ///
-/// The span will be valid, until the scope is left.
+/// The span will be valid, until the scope is left. Use either level and name
+/// or pass in any valid `sp_tracing::Span` for extended usage. The span will
+/// be exited on drop – which is at the end of the block or to the next
+/// `enter_span!` calls, as this overwrites the local variable. For nested
+/// usage or to ensure the span closes at certain time either put it into a block
+/// or use `within_span!`
 ///
 /// # Example
 ///
 /// ```
-/// sp_tracing::enter_span!("test-span");
+/// sp_tracing::enter_span!(sp_tracing::Level::TRACE, "test-span");
+/// // previous will be dropped here
+/// sp_tracing::enter_span!(
+/// 	sp_tracing::span!(sp_tracing::Level::DEBUG, "debug-span", params="value"));
+/// sp_tracing::enter_span!(sp_tracing::info_span!("info-span",  params="value"));
+///
+/// {
+///		sp_tracing::enter_span!(sp_tracing::Level::TRACE, "outer-span");
+///		{
+///			sp_tracing::enter_span!(sp_tracing::Level::TRACE, "inner-span");
+///			// ..
+///		}  // inner span exists here
+///	} // outer span exists here
+///
 /// ```
+#[cfg(any(feature = "std", feature = "with-tracing"))]
 #[macro_export]
 macro_rules! enter_span {
-	( $name:expr ) => {
-		let __tracing_span__ = $crate::if_tracing!(
-			$crate::tracing::span!($crate::tracing::Level::TRACE, $name)
-		);
-		let __tracing_guard__ = $crate::if_tracing!(__tracing_span__.enter());
-	}
-}
-
-/// Generates the given code if the tracing dependency is enabled.
-#[macro_export]
-#[cfg(feature = "std")]
-macro_rules! if_tracing {
-	( $if:expr ) => {{ $if }}
-}
-
-#[macro_export]
-#[cfg(not(feature = "std"))]
-macro_rules! if_tracing {
-	( $if:expr ) => {{}}
-}
-
-#[cfg(feature = "std")]
-pub fn wasm_tracing_enabled() -> bool {
-	WASM_TRACING_ENABLED.load(Ordering::Relaxed)
-}
-
-#[cfg(feature = "std")]
-pub fn set_wasm_tracing(b: bool) {
-	WASM_TRACING_ENABLED.store(b, Ordering::Relaxed)
+	( $span:expr ) => {
+		// Calling this twice in a row will overwrite (and drop) the earlier
+		// that is a _documented feature_!
+		let __within_span__ = $span;
+		let __tracing_guard__ = __within_span__.enter();
+	};
+	( $lvl:expr, $name:expr ) => {
+		$crate::enter_span!($crate::span!($crate::Level::TRACE, $name))
+	};
 }
diff --git a/primitives/tracing/src/proxy.rs b/primitives/tracing/src/proxy.rs
deleted file mode 100644
index 270f57aaa69f0..0000000000000
--- a/primitives/tracing/src/proxy.rs
+++ /dev/null
@@ -1,165 +0,0 @@
-// Copyright 2020 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Proxy to allow entering tracing spans from wasm.
-//!
-//! Use `enter_span` and `exit_span` to surround the code that you wish to trace
-use rental;
-use tracing::info_span;
-
-/// Used to identify a proxied WASM trace
-pub const WASM_TRACE_IDENTIFIER: &'static str = "WASM_TRACE";
-/// Used to extract the real `target` from the associated values of the span
-pub const WASM_TARGET_KEY: &'static str = "proxied_wasm_target";
-/// Used to extract the real `name` from the associated values of the span
-pub const WASM_NAME_KEY: &'static str = "proxied_wasm_name";
-
-const MAX_SPANS_LEN: usize = 1000;
-
-rental! {
-	pub mod rent_span {
-		#[rental]
-		pub struct SpanAndGuard {
-			span: Box<tracing::Span>,
-			guard: tracing::span::Entered<'span>,
-		}
-	}
-}
-
-/// Requires a tracing::Subscriber to process span traces,
-/// this is available when running with client (and relevant cli params).
-pub struct TracingProxy {
-	next_id: u64,
-	spans: Vec<(u64, rent_span::SpanAndGuard)>,
-}
-
-impl Drop for TracingProxy {
-	fn drop(&mut self) {
-		if !self.spans.is_empty() {
-			log::debug!(
-				target: "tracing",
-				"Dropping TracingProxy with {} un-exited spans, marking as not valid", self.spans.len()
-			);
-			while let Some((_, mut sg)) = self.spans.pop() {
-				sg.rent_all_mut(|s| { s.span.record("is_valid_trace", &false); });
-			}
-		}
-	}
-}
-
-impl TracingProxy {
-	pub fn new() -> TracingProxy {
-		TracingProxy {
-			next_id: 0,
-			spans: Vec::new(),
-		}
-	}
-}
-
-impl TracingProxy {
-	/// Create and enter a `tracing` Span, returning the span id,
-	/// which should be passed to `exit_span(id)` to signal that the span should exit.
-	pub fn enter_span(&mut self, proxied_wasm_target: &str, proxied_wasm_name: &str) -> u64 {
-		// The identifiers `proxied_wasm_target` and `proxied_wasm_name` must match their associated const,
-		// WASM_TARGET_KEY and WASM_NAME_KEY.
-		let span = info_span!(WASM_TRACE_IDENTIFIER, is_valid_trace = true, proxied_wasm_target, proxied_wasm_name);
-		self.next_id += 1;
-		let sg = rent_span::SpanAndGuard::new(
-			Box::new(span),
-			|span| span.enter(),
-		);
-		self.spans.push((self.next_id, sg));
-		if self.spans.len() > MAX_SPANS_LEN {
-			// This is to prevent unbounded growth of Vec and could mean one of the following:
-			// 1. Too many nested spans, or MAX_SPANS_LEN is too low.
-			// 2. Not correctly exiting spans due to misconfiguration / misuse
-			log::warn!(
-				target: "tracing",
-				"TracingProxy MAX_SPANS_LEN exceeded, removing oldest span."
-			);
-			let mut sg = self.spans.remove(0).1;
-			sg.rent_all_mut(|s| { s.span.record("is_valid_trace", &false); });
-		}
-		self.next_id
-	}
-
-	/// Exit a span by dropping it along with it's associated guard.
-	pub fn exit_span(&mut self, id: u64) {
-		if self.spans.last().map(|l| id > l.0).unwrap_or(true) {
-			log::warn!(target: "tracing", "Span id not found in TracingProxy: {}", id);
-			return;
-		}
-		let mut last_span = self.spans.pop().expect("Just checked that there is an element to pop; qed");
-		while id < last_span.0 {
-			log::warn!(
-				target: "tracing",
-				"TracingProxy Span ids not equal! id parameter given: {}, last span: {}",
-				id,
-				last_span.0,
-			);
-			last_span.1.rent_all_mut(|s| { s.span.record("is_valid_trace", &false); });
-			if let Some(s) = self.spans.pop() {
-				last_span = s;
-			} else {
-				log::warn!(target: "tracing", "Span id not found in TracingProxy {}", id);
-				return;
-			}
-		}
-	}
-}
-
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-
-	fn create_spans(proxy: &mut TracingProxy, qty: usize) -> Vec<u64> {
-		let mut spans = Vec::new();
-		for n in 0..qty {
-			spans.push(proxy.enter_span("target", &format!("{}", n)));
-		}
-		spans
-	}
-
-	#[test]
-	fn max_spans_len_respected() {
-		let mut proxy = TracingProxy::new();
-		let _spans = create_spans(&mut proxy, MAX_SPANS_LEN + 10);
-		assert_eq!(proxy.spans.len(), MAX_SPANS_LEN);
-		// ensure oldest spans removed
-		assert_eq!(proxy.spans[0].0, 11);
-	}
-
-	#[test]
-	fn handles_span_exit_scenarios() {
-		let mut proxy = TracingProxy::new();
-		let _spans = create_spans(&mut proxy, 10);
-		assert_eq!(proxy.spans.len(), 10);
-		// exit span normally
-		proxy.exit_span(10);
-		assert_eq!(proxy.spans.len(), 9);
-		// skip and exit outer span without exiting inner, id: 8 instead of 9
-		proxy.exit_span(8);
-		// should have also removed the inner span that was lost
-		assert_eq!(proxy.spans.len(), 7);
-		// try to exit span not held
-		proxy.exit_span(9);
-		assert_eq!(proxy.spans.len(), 7);
-		// exit all spans
-		proxy.exit_span(1);
-		assert_eq!(proxy.spans.len(), 0);
-	}
-}
diff --git a/primitives/tracing/src/types.rs b/primitives/tracing/src/types.rs
new file mode 100644
index 0000000000000..050ac4c314166
--- /dev/null
+++ b/primitives/tracing/src/types.rs
@@ -0,0 +1,623 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Types for wasm based tracing. Loosly inspired by `tracing-core` but
+/// optimised for the specific use case.
+
+use core::{format_args, fmt::Debug};
+use sp_std::{
+	vec, vec::Vec,
+};
+use sp_std::Writer;
+use codec::{Encode, Decode};
+
+/// The Tracing Level – the user can filter by this
+#[derive(Clone, Encode, Decode, Debug)]
+pub enum WasmLevel {
+	/// This is a fatal errors
+	ERROR,
+	/// This is a warning you should be aware of
+	WARN,
+	/// Nice to now info
+	INFO,
+	/// Further information for debugging purposes
+	DEBUG,
+	/// The lowest level, keeping track of minute detail
+	TRACE
+}
+
+
+impl From<&tracing_core::Level> for WasmLevel {
+	fn from(l: &tracing_core::Level) -> WasmLevel {
+		match l {
+			&tracing_core::Level::ERROR => WasmLevel::ERROR,
+			&tracing_core::Level::WARN => WasmLevel::WARN,
+			&tracing_core::Level::INFO => WasmLevel::INFO,
+			&tracing_core::Level::DEBUG => WasmLevel::DEBUG,
+			&tracing_core::Level::TRACE => WasmLevel::TRACE,
+		}
+	}
+}
+
+
+
+impl core::default::Default for WasmLevel {
+	fn default() -> Self {
+		WasmLevel::TRACE
+	}
+}
+
+/// A paramter value provided to the span/event
+#[derive(Encode, Decode, Clone)]
+pub enum WasmValue {
+	U8(u8),
+	I8(i8),
+	U32(u32),
+	I32(i32),
+	I64(i64),
+	U64(u64),
+	Bool(bool),
+	Str(Vec<u8>),
+	/// Debug or Display call, this is most-likely a print-able UTF8 String
+	Formatted(Vec<u8>),
+	/// SCALE CODEC encoded object – the name should allow the received to know
+	/// how to decode this.
+	Encoded(Vec<u8>),
+}
+
+impl core::fmt::Debug for WasmValue {
+	fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+		match self {
+			WasmValue::U8(ref i) => {
+				f.write_fmt(format_args!("{}_u8", i))
+			}
+			WasmValue::I8(ref i) => {
+				f.write_fmt(format_args!("{}_i8", i))
+			}
+			WasmValue::U32(ref i) => {
+				f.write_fmt(format_args!("{}_u32", i))
+			}
+			WasmValue::I32(ref i) => {
+				f.write_fmt(format_args!("{}_i32", i))
+			}
+			WasmValue::I64(ref i) => {
+				f.write_fmt(format_args!("{}_i64", i))
+			}
+			WasmValue::U64(ref i) => {
+				f.write_fmt(format_args!("{}_u64", i))
+			}
+			WasmValue::Bool(ref i) => {
+				f.write_fmt(format_args!("{}_bool", i))
+			}
+			WasmValue::Formatted(ref i) | WasmValue::Str(ref i) => {
+				if let Ok(v) = core::str::from_utf8(i) {
+					f.write_fmt(format_args!("{}", v))
+				} else {
+					f.write_fmt(format_args!("{:?}", i))
+				}
+			}
+			WasmValue::Encoded(ref v) => {
+				f.write_str("Scale(")?;
+					for byte in v {
+						f.write_fmt(format_args!("{:02x}", byte))?;
+					}
+				f.write_str(")")
+			}
+		}
+	}
+}
+
+impl From<u8> for WasmValue {
+	fn from(u: u8) -> WasmValue {
+		WasmValue::U8(u)
+	}
+}
+
+impl From<&i8> for WasmValue {
+	fn from(inp: &i8) -> WasmValue {
+		WasmValue::I8(inp.clone())
+	}
+}
+
+impl From<&str> for WasmValue {
+	fn from(inp: &str) -> WasmValue {
+		WasmValue::Str(inp.as_bytes().to_vec())
+	}
+}
+
+impl From<&&str> for WasmValue {
+	fn from(inp: &&str) -> WasmValue {
+		WasmValue::Str((*inp).as_bytes().to_vec())
+	}
+}
+
+impl From<bool> for WasmValue {
+	fn from(inp: bool) -> WasmValue {
+		WasmValue::Bool(inp)
+	}
+}
+
+impl From<core::fmt::Arguments<'_>> for WasmValue {
+	fn from(inp: core::fmt::Arguments<'_>) -> WasmValue {
+		let mut buf = Writer::default();
+		core::fmt::write(&mut buf, inp).expect("Writing of arguments doesn't fail");
+		WasmValue::Formatted(buf.into_inner())
+	}
+}
+
+impl From<i8> for WasmValue {
+	fn from(u: i8) -> WasmValue {
+		WasmValue::I8(u)
+	}
+}
+
+impl From<i32> for WasmValue {
+	fn from(u: i32) -> WasmValue {
+		WasmValue::I32(u)
+	}
+}
+
+impl From<&i32> for WasmValue {
+	fn from(u: &i32) -> WasmValue {
+		WasmValue::I32(*u)
+	}
+}
+
+impl From<u32> for WasmValue {
+	fn from(u: u32) -> WasmValue {
+		WasmValue::U32(u)
+	}
+}
+
+impl From<&u32> for WasmValue {
+	fn from(u: &u32) -> WasmValue {
+		WasmValue::U32(*u)
+	}
+}
+
+impl From<u64> for WasmValue {
+	fn from(u: u64) -> WasmValue {
+		WasmValue::U64(u)
+	}
+}
+
+impl From<i64> for WasmValue {
+	fn from(u: i64) -> WasmValue {
+		WasmValue::I64(u)
+	}
+}
+
+/// The name of a field provided as the argument name when contstructing an
+/// `event!` or `span!`.
+/// Generally generated automaticaly via `stringify` from an `'static &str`.
+/// Likely print-able.
+#[derive(Encode, Decode, Clone)]
+pub struct WasmFieldName(Vec<u8>);
+
+impl core::fmt::Debug for WasmFieldName {
+	fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+		if let Ok(v) = core::str::from_utf8(&self.0) {
+			f.write_fmt(format_args!("{}", v))
+		} else {
+			for byte in self.0.iter() {
+				f.write_fmt(format_args!("{:02x}", byte))?;
+			}
+			Ok(())
+		}
+	}
+}
+
+impl From<Vec<u8>> for WasmFieldName {
+	fn from(v: Vec<u8>) -> Self {
+		WasmFieldName(v)
+	}
+}
+
+impl From<&str> for WasmFieldName {
+	fn from(v: &str) -> Self {
+		WasmFieldName(v.as_bytes().to_vec())
+	}
+}
+
+/// A list of `WasmFieldName`s in the order provided
+#[derive(Encode, Decode, Clone, Debug)]
+pub struct WasmFields(Vec<WasmFieldName>);
+
+impl WasmFields {
+	/// Iterate over the fields
+	pub fn iter(&self) -> core::slice::Iter<'_, WasmFieldName> {
+		self.0.iter()
+	}
+}
+
+impl From<Vec<WasmFieldName>> for WasmFields {
+	fn from(v: Vec<WasmFieldName>) -> WasmFields {
+		WasmFields(v.into())
+	}
+}
+
+impl From<Vec<&str>> for WasmFields {
+	fn from(v: Vec<&str>) -> WasmFields {
+		WasmFields(v.into_iter().map(|v| v.into()).collect())
+	}
+}
+
+impl WasmFields {
+	/// Create an empty entry
+	pub fn empty() -> Self {
+		WasmFields(Vec::with_capacity(0))
+	}
+}
+
+impl From<&tracing_core::field::FieldSet> for WasmFields {
+	fn from(wm: &tracing_core::field::FieldSet) -> WasmFields {
+		WasmFields(wm.iter().map(|s| s.name().into()).collect())
+	}
+}
+
+/// A list of `WasmFieldName`s with the given `WasmValue` (if provided)
+/// in the order specified.
+#[derive(Encode, Decode, Clone)]
+pub struct WasmValuesSet(Vec<(WasmFieldName, Option<WasmValue>)>);
+
+impl core::fmt::Debug for WasmValuesSet {
+	fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+		let mut wrt = f.debug_struct("");
+		let mut non_str = false;
+		for (f, v) in self.0.iter() {
+			if let Ok(s) = core::str::from_utf8(&f.0) {
+				match v {
+					Some(ref i) => wrt.field(s, i),
+					None => wrt.field(s, &(None as Option<WasmValue>)),
+				};
+			} else {
+				non_str = true;
+			}
+		}
+
+		// FIXME: replace with using `finish_non_exhaustive()` once stable
+		//        https://github.com/rust-lang/rust/issues/67364
+		if non_str {
+			wrt.field("..", &"..");
+		}
+
+		wrt.finish()
+	}
+}
+
+
+impl From<Vec<(WasmFieldName, Option<WasmValue>)>> for WasmValuesSet {
+	fn from(v: Vec<(WasmFieldName, Option<WasmValue>)>) -> Self {
+		WasmValuesSet(v)
+	}
+}
+impl From<Vec<(&&WasmFieldName, Option<WasmValue>)>> for WasmValuesSet {
+	fn from(v: Vec<(&&WasmFieldName, Option<WasmValue>)>) -> Self {
+		WasmValuesSet(v.into_iter().map(|(k, v)| ((**k).clone(), v)).collect())
+	}
+}
+
+impl From<Vec<(&&str, Option<WasmValue>)>> for WasmValuesSet {
+	fn from(v: Vec<(&&str, Option<WasmValue>)>) -> Self {
+		WasmValuesSet(v.into_iter().map(|(k, v)| ((*k).into(), v)).collect())
+	}
+}
+
+impl WasmValuesSet {
+	/// Create an empty entry
+	pub fn empty() -> Self {
+		WasmValuesSet(Vec::with_capacity(0))
+	}
+}
+
+impl tracing_core::field::Visit for WasmValuesSet {
+	fn record_debug(&mut self, field: &tracing_core::field::Field, value: &dyn Debug) {
+		self.0.push( (
+			field.name().into(),
+			Some(WasmValue::from(format_args!("{:?}", value)))
+		))
+	}
+	fn record_i64(&mut self, field: &tracing_core::field::Field, value: i64) {
+		self.0.push( (
+			field.name().into(),
+			Some(WasmValue::from(value))
+		))
+	}
+	fn record_u64(&mut self, field: &tracing_core::field::Field, value: u64) {
+		self.0.push( (
+			field.name().into(),
+			Some(WasmValue::from(value))
+		))
+	}
+	fn record_bool(&mut self, field: &tracing_core::field::Field, value: bool) {
+		self.0.push( (
+			field.name().into(),
+			Some(WasmValue::from(value))
+		))
+	}
+	fn record_str(&mut self, field: &tracing_core::field::Field, value: &str) {
+		self.0.push( (
+			field.name().into(),
+			Some(WasmValue::from(value))
+		))
+	}
+}
+/// Metadata provides generic information about the specifc location of the
+/// `span!` or `event!` call on the wasm-side.
+#[derive(Encode, Decode, Clone)]
+pub struct WasmMetadata {
+	/// The name given to `event!`/`span!`, `&'static str` converted to bytes
+	pub name: Vec<u8>,
+	/// The given target to `event!`/`span!` – or module-name, `&'static str` converted to bytes
+	pub target: Vec<u8>,
+	/// The level of this entry
+	pub level: WasmLevel,
+	/// The file this was emitted from – useful for debugging;  `&'static str` converted to bytes
+	pub file: Vec<u8>,
+	/// The specific line number in the file – useful for debugging
+	pub line: u32,
+	/// The module path;  `&'static str` converted to bytes
+	pub module_path: Vec<u8>,
+	/// Whether this is a call  to `span!` or `event!`
+	pub is_span: bool,
+	/// The list of fields specified in the call
+	pub fields: WasmFields,
+}
+
+impl From<&tracing_core::Metadata<'_>> for WasmMetadata {
+	fn from(wm: &tracing_core::Metadata<'_>) -> WasmMetadata {
+		WasmMetadata {
+			name: wm.name().as_bytes().to_vec(),
+			target: wm.target().as_bytes().to_vec(),
+			level: wm.level().into(),
+			file: wm.file().map(|f| f.as_bytes().to_vec()).unwrap_or_default(),
+			line: wm.line().unwrap_or_default(),
+			module_path: wm.module_path().map(|m| m.as_bytes().to_vec()).unwrap_or_default(),
+			is_span: wm.is_span(),
+			fields: wm.fields().into()
+		}
+	}
+}
+
+impl core::fmt::Debug for WasmMetadata {
+	fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+		f.debug_struct("WasmMetadata")
+			.field("name", &decode_field(&self.name))
+			.field("target", &decode_field(&self.target))
+			.field("level", &self.level)
+			.field("file", &decode_field(&self.file))
+			.field("line", &self.line)
+			.field("module_path", &decode_field(&self.module_path))
+			.field("is_span", &self.is_span)
+			.field("fields", &self.fields)
+			.finish()
+	}
+}
+
+impl core::default::Default for WasmMetadata {
+	fn default() -> Self {
+		let target = "default".as_bytes().to_vec();
+		WasmMetadata {
+			target,
+			name: Default::default(),
+			level: Default::default(),
+			file: Default::default(),
+			line: Default::default(),
+			module_path: Default::default(),
+			is_span: true,
+			fields: WasmFields::empty()
+		}
+	}
+}
+
+
+fn decode_field(field: &[u8]) -> &str {
+	core::str::from_utf8(field).unwrap_or_default()
+}
+
+/// Span or Event Attributes
+#[derive(Encode, Decode, Clone, Debug)]
+pub struct WasmEntryAttributes {
+	/// the parent, if directly specified – otherwise assume most inner span
+	pub parent_id: Option<u64>,
+	/// the metadata of the location
+	pub metadata: WasmMetadata,
+	/// the Values provided
+	pub fields: WasmValuesSet,
+}
+
+impl From<&tracing_core::Event<'_>> for WasmEntryAttributes {
+	fn from(evt: &tracing_core::Event<'_>) -> WasmEntryAttributes {
+		let mut fields = WasmValuesSet(Vec::new());
+		evt.record(&mut fields);
+		WasmEntryAttributes {
+			parent_id: evt.parent().map(|id| id.into_u64()),
+			metadata: evt.metadata().into(),
+			fields: fields
+		}
+	}
+}
+
+impl From<&tracing_core::span::Attributes<'_>> for WasmEntryAttributes {
+	fn from(attrs: &tracing_core::span::Attributes<'_>) -> WasmEntryAttributes {
+		let mut fields = WasmValuesSet(Vec::new());
+		attrs.record(&mut fields);
+		WasmEntryAttributes {
+			parent_id: attrs.parent().map(|id| id.into_u64()),
+			metadata: attrs.metadata().into(),
+			fields: fields
+		}
+	}
+}
+
+impl core::default::Default for WasmEntryAttributes {
+	fn default() -> Self {
+		WasmEntryAttributes {
+			parent_id: None,
+			metadata: Default::default(),
+			fields: WasmValuesSet(vec![]),
+		}
+	}
+}
+
+#[cfg(feature = "std")]
+mod std_features {
+
+	use tracing_core::callsite;
+	use tracing;
+
+	/// Static entry use for wasm-originated metadata.
+	pub struct WasmCallsite;
+	impl callsite::Callsite for WasmCallsite {
+		fn set_interest(&self, _: tracing_core::Interest) { unimplemented!() }
+		fn metadata(&self) -> &tracing_core::Metadata { unimplemented!() }
+	}
+	static CALLSITE: WasmCallsite =  WasmCallsite;
+	/// The identifier we are using to inject the wasm events in the generic `tracing` system
+	pub static WASM_TRACE_IDENTIFIER: &'static str = "wasm_tracing";
+	/// The fieldname for the wasm-originated name
+	pub static WASM_NAME_KEY: &'static str = "name";
+	/// The fieldname for the wasm-originated target
+	pub static WASM_TARGET_KEY: &'static str = "target";
+	/// The the list of all static field names we construct from the given metadata
+	pub static GENERIC_FIELDS: &'static [&'static str] = &[WASM_TARGET_KEY, WASM_NAME_KEY,
+		"file", "line", "module_path", "params"];
+
+	// Implementation Note:
+	// the original `tracing` crate generates these static metadata entries at every `span!` and
+	// `event!` location to allow for highly optimised filtering. For us to allow level-based emitting
+	// of wasm events we need these static metadata entries to inject into that system. We then provide
+	// generic `From`-implementations picking the right metadata to refer to.
+
+	static SPAN_ERROR_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::ERROR, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::SPAN
+	);
+
+	static SPAN_WARN_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::WARN, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::SPAN
+	);
+	static SPAN_INFO_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::INFO, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::SPAN
+	);
+
+	static SPAN_DEBUG_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::DEBUG, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::SPAN
+	);
+
+	static SPAN_TRACE_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::TRACE, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::SPAN
+	);
+
+	static EVENT_ERROR_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::ERROR, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::EVENT
+	);
+
+	static EVENT_WARN_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::WARN, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::EVENT
+	);
+
+	static EVENT_INFO_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::INFO, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::EVENT
+	);
+
+	static EVENT_DEBUG_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::DEBUG, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::EVENT
+	);
+
+	static EVENT_TRACE_METADATA : tracing_core::Metadata<'static> = tracing::Metadata::new(
+		WASM_TRACE_IDENTIFIER, WASM_TRACE_IDENTIFIER, tracing::Level::TRACE, None, None, None,
+		tracing_core::field::FieldSet::new(GENERIC_FIELDS, tracing_core::identify_callsite!(&CALLSITE)),
+		tracing_core::metadata::Kind::EVENT
+	);
+
+	// FIXME: this could be done a lot in 0.2 if they opt for using `Cow<str,'static>` instead
+	//			https://github.com/paritytech/substrate/issues/7134
+	impl From<&crate::WasmMetadata> for &'static tracing_core::Metadata<'static> {
+		fn from(wm: &crate::WasmMetadata) -> &'static tracing_core::Metadata<'static> {
+			match (&wm.level, wm.is_span) {
+				(&crate::WasmLevel::ERROR, true) => &SPAN_ERROR_METADATA,
+				(&crate::WasmLevel::WARN, true) => &SPAN_WARN_METADATA,
+				(&crate::WasmLevel::INFO, true) => &SPAN_INFO_METADATA,
+				(&crate::WasmLevel::DEBUG, true) => &SPAN_DEBUG_METADATA,
+				(&crate::WasmLevel::TRACE, true) => &SPAN_TRACE_METADATA,
+				(&crate::WasmLevel::ERROR, false) => &EVENT_ERROR_METADATA,
+				(&crate::WasmLevel::WARN, false) => &EVENT_WARN_METADATA,
+				(&crate::WasmLevel::INFO, false) => &EVENT_INFO_METADATA,
+				(&crate::WasmLevel::DEBUG, false) => &EVENT_DEBUG_METADATA,
+				(&crate::WasmLevel::TRACE, false) => &EVENT_TRACE_METADATA,
+			}
+		}
+	}
+
+	impl From<crate::WasmEntryAttributes> for tracing::Span {
+		fn from(a: crate::WasmEntryAttributes) -> tracing::Span {
+			let name = std::str::from_utf8(&a.metadata.name).unwrap_or_default();
+			let target = std::str::from_utf8(&a.metadata.target).unwrap_or_default();
+			let file = std::str::from_utf8(&a.metadata.file).unwrap_or_default();
+			let line = a.metadata.line;
+			let module_path = std::str::from_utf8(&a.metadata.module_path).unwrap_or_default();
+			let params = a.fields;
+			let metadata : &tracing_core::metadata::Metadata<'static> = (&a.metadata).into();
+
+			tracing::span::Span::child_of(
+				a.parent_id.map(|i|tracing_core::span::Id::from_u64(i)),
+				&metadata,
+				&tracing::valueset!{ metadata.fields(), target, name, file, line, module_path, ?params }
+			)
+		}
+	}
+
+	impl crate::WasmEntryAttributes {
+		/// convert the given Attributes to an event and emit it using `tracing_core`.
+		pub fn emit(self: crate::WasmEntryAttributes) {
+			let name = std::str::from_utf8(&self.metadata.name).unwrap_or_default();
+			let target = std::str::from_utf8(&self.metadata.target).unwrap_or_default();
+			let file = std::str::from_utf8(&self.metadata.file).unwrap_or_default();
+			let line = self.metadata.line;
+			let module_path = std::str::from_utf8(&self.metadata.module_path).unwrap_or_default();
+			let params = self.fields;
+			let metadata : &tracing_core::metadata::Metadata<'static> = (&self.metadata).into();
+
+			tracing_core::Event::child_of(
+				self.parent_id.map(|i|tracing_core::span::Id::from_u64(i)),
+				&metadata,
+				&tracing::valueset!{ metadata.fields(), target, name, file, line, module_path, ?params }
+			)
+		}
+	}
+}
+
+#[cfg(feature = "std")]
+pub use std_features::*;

From b7a3b2b486c290feb573ae3552eb29774bcb6f09 Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Fri, 18 Sep 2020 11:16:41 +0200
Subject: [PATCH 105/122] Pallet Indices (#7137)

---
 bin/node/runtime/src/lib.rs                   |  2 +-
 bin/node/runtime/src/weights/mod.rs           |  1 +
 .../runtime/src/weights/pallet_indices.rs     | 52 +++++++++++++++++++
 frame/indices/src/benchmarking.rs             | 24 +++------
 frame/indices/src/default_weights.rs          | 51 ++++++++++++++++++
 frame/indices/src/lib.rs                      | 36 +++++--------
 6 files changed, 124 insertions(+), 42 deletions(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_indices.rs
 create mode 100644 frame/indices/src/default_weights.rs

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index 16fcc9f70bc16..468605ddb7a09 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -320,7 +320,7 @@ impl pallet_indices::Trait for Runtime {
 	type Currency = Balances;
 	type Deposit = IndexDeposit;
 	type Event = Event;
-	type WeightInfo = ();
+	type WeightInfo = weights::pallet_indices::WeightInfo;
 }
 
 parameter_types! {
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index fd6d3cab49e28..a07aa2e5bc896 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -20,6 +20,7 @@ pub mod pallet_balances;
 pub mod pallet_collective;
 pub mod pallet_democracy;
 pub mod pallet_identity;
+pub mod pallet_indices;
 pub mod pallet_im_online;
 pub mod pallet_proxy;
 pub mod pallet_staking;
diff --git a/bin/node/runtime/src/weights/pallet_indices.rs b/bin/node/runtime/src/weights/pallet_indices.rs
new file mode 100644
index 0000000000000..f6a708bbd46b6
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_indices.rs
@@ -0,0 +1,52 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_indices::WeightInfo for WeightInfo {
+	fn claim() -> Weight {
+		(56_237_000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn transfer() -> Weight {
+		(63_665_000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn free() -> Weight {
+		(50_736_000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_transfer() -> Weight {
+		(52_361_000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn freeze() -> Weight {
+		(46_483_000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+}
diff --git a/frame/indices/src/benchmarking.rs b/frame/indices/src/benchmarking.rs
index e8465c44cdc16..382bf07f1136e 100644
--- a/frame/indices/src/benchmarking.rs
+++ b/frame/indices/src/benchmarking.rs
@@ -32,9 +32,7 @@ benchmarks! {
 	_ { }
 
 	claim {
-		// Index being claimed
-		let i in 0 .. 1000;
-		let account_index = T::AccountIndex::from(i);
+		let account_index = T::AccountIndex::from(SEED);
 		let caller: T::AccountId = whitelisted_caller();
 		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
 	}: _(RawOrigin::Signed(caller.clone()), account_index)
@@ -43,13 +41,11 @@ benchmarks! {
 	}
 
 	transfer {
-		// Index being claimed
-		let i in 0 .. 1000;
-		let account_index = T::AccountIndex::from(i);
+		let account_index = T::AccountIndex::from(SEED);
 		// Setup accounts
 		let caller: T::AccountId = whitelisted_caller();
 		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
-		let recipient: T::AccountId = account("recipient", i, SEED);
+		let recipient: T::AccountId = account("recipient", 0, SEED);
 		T::Currency::make_free_balance_be(&recipient, BalanceOf::<T>::max_value());
 		// Claim the index
 		Indices::<T>::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?;
@@ -59,9 +55,7 @@ benchmarks! {
 	}
 
 	free {
-		// Index being claimed
-		let i in 0 .. 1000;
-		let account_index = T::AccountIndex::from(i);
+		let account_index = T::AccountIndex::from(SEED);
 		// Setup accounts
 		let caller: T::AccountId = whitelisted_caller();
 		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
@@ -73,13 +67,11 @@ benchmarks! {
 	}
 
 	force_transfer {
-		// Index being claimed
-		let i in 0 .. 1000;
-		let account_index = T::AccountIndex::from(i);
+		let account_index = T::AccountIndex::from(SEED);
 		// Setup accounts
 		let original: T::AccountId = account("original", 0, SEED);
 		T::Currency::make_free_balance_be(&original, BalanceOf::<T>::max_value());
-		let recipient: T::AccountId = account("recipient", i, SEED);
+		let recipient: T::AccountId = account("recipient", 0, SEED);
 		T::Currency::make_free_balance_be(&recipient, BalanceOf::<T>::max_value());
 		// Claim the index
 		Indices::<T>::claim(RawOrigin::Signed(original).into(), account_index)?;
@@ -89,9 +81,7 @@ benchmarks! {
 	}
 
 	freeze {
-		// Index being claimed
-		let i in 0 .. 1000;
-		let account_index = T::AccountIndex::from(i);
+		let account_index = T::AccountIndex::from(SEED);
 		// Setup accounts
 		let caller: T::AccountId = whitelisted_caller();
 		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
diff --git a/frame/indices/src/default_weights.rs b/frame/indices/src/default_weights.rs
new file mode 100644
index 0000000000000..6b3b9c13e40a0
--- /dev/null
+++ b/frame/indices/src/default_weights.rs
@@ -0,0 +1,51 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn claim() -> Weight {
+		(56_237_000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn transfer() -> Weight {
+		(63_665_000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn free() -> Weight {
+		(50_736_000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn force_transfer() -> Weight {
+		(52_361_000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn freeze() -> Weight {
+		(46_483_000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+}
diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs
index 3dc0cec9d94bc..edbaed17e536f 100644
--- a/frame/indices/src/lib.rs
+++ b/frame/indices/src/lib.rs
@@ -28,7 +28,7 @@ use sp_runtime::traits::{
 use frame_support::{Parameter, decl_module, decl_error, decl_event, decl_storage, ensure};
 use frame_support::dispatch::DispatchResult;
 use frame_support::traits::{Currency, ReservableCurrency, Get, BalanceStatus::Reserved};
-use frame_support::weights::{Weight, constants::WEIGHT_PER_MICROS};
+use frame_support::weights::Weight;
 use frame_system::{ensure_signed, ensure_root};
 use self::address::Address as RawAddress;
 
@@ -36,24 +36,17 @@ mod mock;
 pub mod address;
 mod tests;
 mod benchmarking;
+mod default_weights;
 
 pub type Address<T> = RawAddress<<T as frame_system::Trait>::AccountId, <T as Trait>::AccountIndex>;
 type BalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
 
 pub trait WeightInfo {
-	fn claim(i: u32, ) -> Weight;
-	fn transfer(i: u32, ) -> Weight;
-	fn free(i: u32, ) -> Weight;
-	fn force_transfer(i: u32, ) -> Weight;
-	fn freeze(i: u32, ) -> Weight;
-}
-
-impl WeightInfo for () {
-	fn claim(_i: u32, ) -> Weight { 1_000_000_000 }
-	fn transfer(_i: u32, ) -> Weight { 1_000_000_000 }
-	fn free(_i: u32, ) -> Weight { 1_000_000_000 }
-	fn force_transfer(_i: u32, ) -> Weight { 1_000_000_000 }
-	fn freeze(_i: u32, ) -> Weight { 1_000_000_000 }
+	fn claim() -> Weight;
+	fn transfer() -> Weight;
+	fn free() -> Weight;
+	fn force_transfer() -> Weight;
+	fn freeze() -> Weight;
 }
 
 /// The module's config trait.
@@ -142,10 +135,9 @@ decl_module! {
 		/// - One reserve operation.
 		/// - One event.
 		/// -------------------
-		/// - Base Weight: 28.69 µs
 		/// - DB Weight: 1 Read/Write (Accounts)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 1) + 30 * WEIGHT_PER_MICROS]
+		#[weight = T::WeightInfo::claim()]
 		fn claim(origin, index: T::AccountIndex) {
 			let who = ensure_signed(origin)?;
 
@@ -173,12 +165,11 @@ decl_module! {
 		/// - One transfer operation.
 		/// - One event.
 		/// -------------------
-		/// - Base Weight: 33.74 µs
 		/// - DB Weight:
 		///    - Reads: Indices Accounts, System Account (recipient)
 		///    - Writes: Indices Accounts, System Account (recipient)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(2, 2) + 35 * WEIGHT_PER_MICROS]
+		#[weight = T::WeightInfo::transfer()]
 		fn transfer(origin, new: T::AccountId, index: T::AccountIndex) {
 			let who = ensure_signed(origin)?;
 			ensure!(who != new, Error::<T>::NotTransfer);
@@ -210,10 +201,9 @@ decl_module! {
 		/// - One reserve operation.
 		/// - One event.
 		/// -------------------
-		/// - Base Weight: 25.53 µs
 		/// - DB Weight: 1 Read/Write (Accounts)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 1) + 25 * WEIGHT_PER_MICROS]
+		#[weight = T::WeightInfo::free()]
 		fn free(origin, index: T::AccountIndex) {
 			let who = ensure_signed(origin)?;
 
@@ -244,12 +234,11 @@ decl_module! {
 		/// - Up to one reserve operation.
 		/// - One event.
 		/// -------------------
-		/// - Base Weight: 26.83 µs
 		/// - DB Weight:
 		///    - Reads: Indices Accounts, System Account (original owner)
 		///    - Writes: Indices Accounts, System Account (original owner)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(2, 2) + 25 * WEIGHT_PER_MICROS]
+		#[weight = T::WeightInfo::force_transfer()]
 		fn force_transfer(origin, new: T::AccountId, index: T::AccountIndex, freeze: bool) {
 			ensure_root(origin)?;
 
@@ -277,10 +266,9 @@ decl_module! {
 		/// - Up to one slash operation.
 		/// - One event.
 		/// -------------------
-		/// - Base Weight: 30.86 µs
 		/// - DB Weight: 1 Read/Write (Accounts)
 		/// # </weight>
-		#[weight = T::DbWeight::get().reads_writes(1, 1) + 30 * WEIGHT_PER_MICROS]
+		#[weight = T::WeightInfo::freeze()]
 		fn freeze(origin, index: T::AccountIndex) {
 			let who = ensure_signed(origin)?;
 

From f8ee0e10149f3d3a6f430beff2bde439ecb81ad3 Mon Sep 17 00:00:00 2001
From: Wei Tang <wei@that.world>
Date: Fri, 18 Sep 2020 11:37:31 +0200
Subject: [PATCH 106/122] pow: replace the thread-base mining loop with a
 future-based mining worker (#7060)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* New worker design

* Remove unused thread import

* Add back missing inherent data provider registration

* Add function to get a Cloned metadata

* Add some docs

* Derive Eq and PartialEq for MiningMetadata

* Fix cargo lock

* Fix line width

* Add docs and fix issues in UntilImportedOrTimeout

* Update client/consensus/pow/src/lib.rs

Co-authored-by: David <dvdplm@gmail.com>

* Add back comments

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: David <dvdplm@gmail.com>
---
 Cargo.lock                         |   2 +
 client/consensus/pow/Cargo.toml    |   2 +
 client/consensus/pow/src/lib.rs    | 309 +++++++++++++----------------
 client/consensus/pow/src/worker.rs | 213 ++++++++++++++++++++
 4 files changed, 358 insertions(+), 168 deletions(-)
 create mode 100644 client/consensus/pow/src/worker.rs

diff --git a/Cargo.lock b/Cargo.lock
index fc293472baecd..df454edaf22f1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6621,8 +6621,10 @@ version = "0.8.0-rc6"
 dependencies = [
  "derive_more",
  "futures 0.3.5",
+ "futures-timer 3.0.2",
  "log",
  "parity-scale-codec",
+ "parking_lot 0.10.2",
  "sc-client-api",
  "sp-api",
  "sp-block-builder",
diff --git a/client/consensus/pow/Cargo.toml b/client/consensus/pow/Cargo.toml
index 993502972f2d0..9e97052373adc 100644
--- a/client/consensus/pow/Cargo.toml
+++ b/client/consensus/pow/Cargo.toml
@@ -24,6 +24,8 @@ sp-consensus-pow = { version = "0.8.0-rc6", path = "../../../primitives/consensu
 sp-consensus = { version = "0.8.0-rc6", path = "../../../primitives/consensus/common" }
 log = "0.4.8"
 futures = { version = "0.3.1", features = ["compat"] }
+futures-timer = "3.0.1"
+parking_lot = "0.10.0"
 sp-timestamp = { version = "2.0.0-rc6", path = "../../../primitives/timestamp" }
 derive_more = "0.99.2"
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc6"}
diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs
index 70a7bb47873f6..b73b9aa91f802 100644
--- a/client/consensus/pow/src/lib.rs
+++ b/client/consensus/pow/src/lib.rs
@@ -31,14 +31,17 @@
 //! as the storage, but it is not recommended as it won't work well with light
 //! clients.
 
-use std::sync::Arc;
-use std::any::Any;
-use std::borrow::Cow;
-use std::thread;
-use std::collections::HashMap;
-use std::marker::PhantomData;
-use std::cmp::Ordering;
-use sc_client_api::{BlockOf, backend::AuxStore};
+mod worker;
+
+pub use crate::worker::{MiningWorker, MiningMetadata, MiningBuild};
+
+use std::{
+	sync::Arc, any::Any, borrow::Cow, collections::HashMap, marker::PhantomData,
+	cmp::Ordering, time::Duration,
+};
+use futures::{prelude::*, future::Either};
+use parking_lot::Mutex;
+use sc_client_api::{BlockOf, backend::AuxStore, BlockchainEvents};
 use sp_blockchain::{HeaderBackend, ProvideCache, well_known_cache_keys::Id as CacheKeyId};
 use sp_block_builder::BlockBuilder as BlockBuilderApi;
 use sp_runtime::{Justification, RuntimeString};
@@ -61,6 +64,8 @@ use sc_client_api;
 use log::*;
 use sp_timestamp::{InherentError as TIError, TimestampInherentData};
 
+use crate::worker::UntilImportedOrTimeout;
+
 #[derive(derive_more::Display, Debug)]
 pub enum Error<B: BlockT> {
 	#[display(fmt = "Header uses the wrong engine {:?}", _0)]
@@ -193,15 +198,6 @@ pub trait PowAlgorithm<B: BlockT> {
 		seal: &Seal,
 		difficulty: Self::Difficulty,
 	) -> Result<bool, Error<B>>;
-	/// Mine a seal that satisfies the given difficulty.
-	fn mine(
-		&self,
-		parent: &BlockId<B>,
-		pre_hash: &B::Hash,
-		pre_digest: Option<&[u8]>,
-		difficulty: Self::Difficulty,
-		round: u32,
-	) -> Result<Option<Seal>, Error<B>>;
 }
 
 /// A block importer for PoW.
@@ -534,194 +530,171 @@ pub fn import_queue<B, Transaction, Algorithm>(
 	))
 }
 
-/// Start the background mining thread for PoW. Note that because PoW mining
-/// is CPU-intensive, it is not possible to use an async future to define this.
-/// However, it's not recommended to use background threads in the rest of the
-/// codebase.
+/// Start the mining worker for PoW. This function provides the necessary helper functions that can
+/// be used to implement a miner. However, it does not do the CPU-intensive mining itself.
+///
+/// Two values are returned -- a worker, which contains functions that allows querying the current
+/// mining metadata and submitting mined blocks, and a future, which must be polled to fill in
+/// information in the worker.
 ///
-/// `pre_runtime` is a parameter that allows a custom additional pre-runtime
-/// digest to be inserted for blocks being built. This can encode authorship
-/// information, or just be a graffiti. `round` is for number of rounds the
-/// CPU miner runs each time. This parameter should be tweaked so that each
-/// mining round is within sub-second time.
-pub fn start_mine<B: BlockT, C, Algorithm, E, SO, S, CAW>(
-	mut block_import: BoxBlockImport<B, sp_api::TransactionFor<C, B>>,
+/// `pre_runtime` is a parameter that allows a custom additional pre-runtime digest to be inserted
+/// for blocks being built. This can encode authorship information, or just be a graffiti.
+pub fn start_mining_worker<Block, C, S, Algorithm, E, SO, CAW>(
+	block_import: BoxBlockImport<Block, sp_api::TransactionFor<C, Block>>,
 	client: Arc<C>,
+	select_chain: S,
 	algorithm: Algorithm,
 	mut env: E,
-	pre_runtime: Option<Vec<u8>>,
-	round: u32,
 	mut sync_oracle: SO,
-	build_time: std::time::Duration,
-	select_chain: Option<S>,
+	pre_runtime: Option<Vec<u8>>,
 	inherent_data_providers: sp_inherents::InherentDataProviders,
+	timeout: Duration,
+	build_time: Duration,
 	can_author_with: CAW,
-) where
-	C: HeaderBackend<B> + AuxStore + ProvideRuntimeApi<B> + 'static,
-	Algorithm: PowAlgorithm<B> + Send + Sync + 'static,
-	E: Environment<B> + Send + Sync + 'static,
+) -> (Arc<Mutex<MiningWorker<Block, Algorithm, C>>>, impl Future<Output = ()>) where
+	Block: BlockT,
+	C: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + 'static,
+	S: SelectChain<Block> + 'static,
+	Algorithm: PowAlgorithm<Block> + Clone,
+	Algorithm::Difficulty: 'static,
+	E: Environment<Block> + Send + Sync + 'static,
 	E::Error: std::fmt::Debug,
-	E::Proposer: Proposer<B, Transaction = sp_api::TransactionFor<C, B>>,
-	SO: SyncOracle + Send + Sync + 'static,
-	S: SelectChain<B> + 'static,
-	CAW: CanAuthorWith<B> + Send + 'static,
+	E::Proposer: Proposer<Block, Transaction = sp_api::TransactionFor<C, Block>>,
+	SO: SyncOracle + Clone + Send + Sync + 'static,
+	CAW: CanAuthorWith<Block> + Clone + Send + 'static,
 {
 	if let Err(_) = register_pow_inherent_data_provider(&inherent_data_providers) {
 		warn!("Registering inherent data provider for timestamp failed");
 	}
 
-	thread::spawn(move || {
-		loop {
-			match mine_loop(
-				&mut block_import,
-				client.as_ref(),
-				&algorithm,
-				&mut env,
-				pre_runtime.as_ref(),
-				round,
-				&mut sync_oracle,
-				build_time.clone(),
-				select_chain.as_ref(),
-				&inherent_data_providers,
-				&can_author_with,
-			) {
-				Ok(()) => (),
-				Err(e) => error!(
-					"Mining block failed with {:?}. Sleep for 1 second before restarting...",
-					e
-				),
-			}
-			std::thread::sleep(std::time::Duration::new(1, 0));
-		}
-	});
-}
+	let timer = UntilImportedOrTimeout::new(client.import_notification_stream(), timeout);
+	let worker = Arc::new(Mutex::new(MiningWorker::<Block, Algorithm, C> {
+		build: None,
+		algorithm: algorithm.clone(),
+		block_import,
+	}));
+	let worker_ret = worker.clone();
+
+	let task = timer.for_each(move |()| {
+		let worker = worker.clone();
 
-fn mine_loop<B: BlockT, C, Algorithm, E, SO, S, CAW>(
-	block_import: &mut BoxBlockImport<B, sp_api::TransactionFor<C, B>>,
-	client: &C,
-	algorithm: &Algorithm,
-	env: &mut E,
-	pre_runtime: Option<&Vec<u8>>,
-	round: u32,
-	sync_oracle: &mut SO,
-	build_time: std::time::Duration,
-	select_chain: Option<&S>,
-	inherent_data_providers: &sp_inherents::InherentDataProviders,
-	can_author_with: &CAW,
-) -> Result<(), Error<B>> where
-	C: HeaderBackend<B> + AuxStore + ProvideRuntimeApi<B>,
-	Algorithm: PowAlgorithm<B>,
-	Algorithm::Difficulty: 'static,
-	E: Environment<B>,
-	E::Proposer: Proposer<B, Transaction = sp_api::TransactionFor<C, B>>,
-	E::Error: std::fmt::Debug,
-	SO: SyncOracle,
-	S: SelectChain<B>,
-	sp_api::TransactionFor<C, B>: 'static,
-	CAW: CanAuthorWith<B>,
-{
-	'outer: loop {
 		if sync_oracle.is_major_syncing() {
 			debug!(target: "pow", "Skipping proposal due to sync.");
-			std::thread::sleep(std::time::Duration::new(1, 0));
-			continue 'outer
+			worker.lock().on_major_syncing();
+			return Either::Left(future::ready(()))
 		}
 
-		let (best_hash, best_header) = match select_chain {
-			Some(select_chain) => {
-				let header = select_chain.best_chain()
-					.map_err(Error::BestHeaderSelectChain)?;
-				let hash = header.hash();
-				(hash, header)
-			},
-			None => {
-				let hash = client.info().best_hash;
-				let header = client.header(BlockId::Hash(hash))
-					.map_err(Error::BestHeader)?
-					.ok_or(Error::NoBestHeader)?;
-				(hash, header)
+		let best_header = match select_chain.best_chain() {
+			Ok(x) => x,
+			Err(err) => {
+				warn!(
+					target: "pow",
+					"Unable to pull new block for authoring. \
+					 Select best chain error: {:?}",
+					err
+				);
+				return Either::Left(future::ready(()))
 			},
 		};
+		let best_hash = best_header.hash();
 
 		if let Err(err) = can_author_with.can_author_with(&BlockId::Hash(best_hash)) {
 			warn!(
 				target: "pow",
 				"Skipping proposal `can_author_with` returned: {} \
-				Probably a node update is required!",
+				 Probably a node update is required!",
 				err,
 			);
-			std::thread::sleep(std::time::Duration::from_secs(1));
-			continue 'outer
+			return Either::Left(future::ready(()))
 		}
 
-		let proposer = futures::executor::block_on(env.init(&best_header))
-			.map_err(|e| Error::Environment(format!("{:?}", e)))?;
-
-		let inherent_data = inherent_data_providers
-			.create_inherent_data().map_err(Error::CreateInherents)?;
-		let mut inherent_digest = Digest::default();
-		if let Some(pre_runtime) = &pre_runtime {
-			inherent_digest.push(DigestItem::PreRuntime(POW_ENGINE_ID, pre_runtime.to_vec()));
+		if worker.lock().best_hash() == Some(best_hash) {
+			return Either::Left(future::ready(()))
 		}
-		let proposal = futures::executor::block_on(proposer.propose(
-			inherent_data,
-			inherent_digest,
-			build_time.clone(),
-			RecordProof::No,
-		)).map_err(|e| Error::BlockProposingError(format!("{:?}", e)))?;
-
-		let (header, body) = proposal.block.deconstruct();
-		let (difficulty, seal) = {
-			let difficulty = algorithm.difficulty(best_hash)?;
-
-			loop {
-				let seal = algorithm.mine(
-					&BlockId::Hash(best_hash),
-					&header.hash(),
-					pre_runtime.map(|v| &v[..]),
-					difficulty,
-					round,
-				)?;
-
-				if let Some(seal) = seal {
-					break (difficulty, seal)
-				}
 
-				if best_hash != client.info().best_hash {
-					continue 'outer
-				}
-			}
+		// The worker is locked for the duration of the whole proposing period. Within this period,
+		// the mining target is outdated and useless anyway.
+
+		let difficulty = match algorithm.difficulty(best_hash) {
+			Ok(x) => x,
+			Err(err) => {
+				warn!(
+					target: "pow",
+					"Unable to propose new block for authoring. \
+					 Fetch difficulty failed: {:?}",
+					err,
+				);
+				return Either::Left(future::ready(()))
+			},
 		};
 
-		log::info!("✅ Successfully mined block: {}", best_hash);
-
-		let (hash, seal) = {
-			let seal = DigestItem::Seal(POW_ENGINE_ID, seal);
-			let mut header = header.clone();
-			header.digest_mut().push(seal);
-			let hash = header.hash();
-			let seal = header.digest_mut().pop()
-				.expect("Pushed one seal above; length greater than zero; qed");
-			(hash, seal)
+		let awaiting_proposer = env.init(&best_header);
+		let inherent_data = match inherent_data_providers.create_inherent_data() {
+			Ok(x) => x,
+			Err(err) => {
+				warn!(
+					target: "pow",
+					"Unable to propose new block for authoring. \
+					 Creating inherent data failed: {:?}",
+					err,
+				);
+				return Either::Left(future::ready(()))
+			},
 		};
+		let mut inherent_digest = Digest::<Block::Hash>::default();
+		if let Some(pre_runtime) = &pre_runtime {
+			inherent_digest.push(DigestItem::PreRuntime(POW_ENGINE_ID, pre_runtime.to_vec()));
+		}
 
-		let intermediate = PowIntermediate::<Algorithm::Difficulty> {
-			difficulty: Some(difficulty),
-		};
+		let pre_runtime = pre_runtime.clone();
+
+		Either::Right(async move {
+			let proposer = match awaiting_proposer.await {
+				Ok(x) => x,
+				Err(err) => {
+					warn!(
+						target: "pow",
+						"Unable to propose new block for authoring. \
+						 Creating proposer failed: {:?}",
+						err,
+					);
+					return
+				},
+			};
+
+			let proposal = match proposer.propose(
+				inherent_data,
+				inherent_digest,
+				build_time.clone(),
+				RecordProof::No,
+			).await {
+				Ok(x) => x,
+				Err(err) => {
+					warn!(
+						target: "pow",
+						"Unable to propose new block for authoring. \
+						 Creating proposal failed: {:?}",
+						err,
+					);
+					return
+				},
+			};
+
+			let build = MiningBuild::<Block, Algorithm, C> {
+				metadata: MiningMetadata {
+					best_hash,
+					pre_hash: proposal.block.header().hash(),
+					pre_runtime: pre_runtime.clone(),
+					difficulty,
+				},
+				proposal,
+			};
 
-		let mut import_block = BlockImportParams::new(BlockOrigin::Own, header);
-		import_block.post_digests.push(seal);
-		import_block.body = Some(body);
-		import_block.storage_changes = Some(proposal.storage_changes);
-		import_block.intermediates.insert(
-			Cow::from(INTERMEDIATE_KEY),
-			Box::new(intermediate) as Box<dyn Any>
-		);
-		import_block.post_hash = Some(hash);
+			worker.lock().on_build(build);
+		})
+	});
 
-		block_import.import_block(import_block, HashMap::default())
-			.map_err(|e| Error::BlockBuiltError(best_hash, e))?;
-	}
+	(worker_ret, task)
 }
 
 /// Find PoW pre-runtime.
diff --git a/client/consensus/pow/src/worker.rs b/client/consensus/pow/src/worker.rs
new file mode 100644
index 0000000000000..4ed863dcd9ed9
--- /dev/null
+++ b/client/consensus/pow/src/worker.rs
@@ -0,0 +1,213 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{pin::Pin, time::Duration, collections::HashMap, any::Any, borrow::Cow};
+use sc_client_api::ImportNotifications;
+use sp_runtime::{DigestItem, traits::Block as BlockT, generic::BlockId};
+use sp_consensus::{Proposal, BlockOrigin, BlockImportParams, import_queue::BoxBlockImport};
+use futures::{prelude::*, task::{Context, Poll}};
+use futures_timer::Delay;
+use log::*;
+
+use crate::{INTERMEDIATE_KEY, POW_ENGINE_ID, Seal, PowAlgorithm, PowIntermediate};
+
+/// Mining metadata. This is the information needed to start an actual mining loop.
+#[derive(Clone, Eq, PartialEq)]
+pub struct MiningMetadata<H, D> {
+	/// Currently known best hash which the pre-hash is built on.
+	pub best_hash: H,
+	/// Mining pre-hash.
+	pub pre_hash: H,
+	/// Pre-runtime digest item.
+	pub pre_runtime: Option<Vec<u8>>,
+	/// Mining target difficulty.
+	pub difficulty: D,
+}
+
+/// A build of mining, containing the metadata and the block proposal.
+pub struct MiningBuild<Block: BlockT, Algorithm: PowAlgorithm<Block>, C: sp_api::ProvideRuntimeApi<Block>> {
+	/// Mining metadata.
+	pub metadata: MiningMetadata<Block::Hash, Algorithm::Difficulty>,
+	/// Mining proposal.
+	pub proposal: Proposal<Block, sp_api::TransactionFor<C, Block>>,
+}
+
+/// Mining worker that exposes structs to query the current mining build and submit mined blocks.
+pub struct MiningWorker<Block: BlockT, Algorithm: PowAlgorithm<Block>, C: sp_api::ProvideRuntimeApi<Block>> {
+	pub(crate) build: Option<MiningBuild<Block, Algorithm, C>>,
+	pub(crate) algorithm: Algorithm,
+	pub(crate) block_import: BoxBlockImport<Block, sp_api::TransactionFor<C, Block>>,
+}
+
+impl<Block, Algorithm, C> MiningWorker<Block, Algorithm, C> where
+	Block: BlockT,
+	C: sp_api::ProvideRuntimeApi<Block>,
+	Algorithm: PowAlgorithm<Block>,
+	Algorithm::Difficulty: 'static,
+{
+	/// Get the current best hash. `None` if the worker has just started or the client is doing
+	/// major syncing.
+	pub fn best_hash(&self) -> Option<Block::Hash> {
+		self.build.as_ref().map(|b| b.metadata.best_hash)
+	}
+
+	pub(crate) fn on_major_syncing(&mut self) {
+		self.build = None;
+	}
+
+	pub(crate) fn on_build(
+		&mut self,
+		build: MiningBuild<Block, Algorithm, C>,
+	) {
+		self.build = Some(build);
+	}
+
+	/// Get a copy of the current mining metadata, if available.
+	pub fn metadata(&self) -> Option<MiningMetadata<Block::Hash, Algorithm::Difficulty>> {
+		self.build.as_ref().map(|b| b.metadata.clone())
+	}
+
+	/// Submit a mined seal. The seal will be validated again. Returns true if the submission is
+	/// successful.
+	pub fn submit(&mut self, seal: Seal) -> bool {
+		if let Some(build) = self.build.take() {
+			match self.algorithm.verify(
+				&BlockId::Hash(build.metadata.best_hash),
+				&build.metadata.pre_hash,
+				build.metadata.pre_runtime.as_ref().map(|v| &v[..]),
+				&seal,
+				build.metadata.difficulty,
+			) {
+				Ok(true) => (),
+				Ok(false) => {
+					warn!(
+						target: "pow",
+						"Unable to import mined block: seal is invalid",
+					);
+					return false
+				},
+				Err(err) => {
+					warn!(
+						target: "pow",
+						"Unable to import mined block: {:?}",
+						err,
+					);
+					return false
+				},
+			}
+
+			let seal = DigestItem::Seal(POW_ENGINE_ID, seal);
+			let (header, body) = build.proposal.block.deconstruct();
+
+			let mut import_block = BlockImportParams::new(BlockOrigin::Own, header);
+			import_block.post_digests.push(seal);
+			import_block.body = Some(body);
+			import_block.storage_changes = Some(build.proposal.storage_changes);
+
+			let intermediate = PowIntermediate::<Algorithm::Difficulty> {
+				difficulty: Some(build.metadata.difficulty),
+			};
+
+			import_block.intermediates.insert(
+				Cow::from(INTERMEDIATE_KEY),
+				Box::new(intermediate) as Box<dyn Any>
+			);
+
+			match self.block_import.import_block(import_block, HashMap::default()) {
+				Ok(_) => {
+					info!(
+						target: "pow",
+						"✅ Successfully mined block on top of: {}",
+						build.metadata.best_hash
+					);
+					true
+				},
+				Err(err) => {
+					warn!(
+						target: "pow",
+						"Unable to import mined block: {:?}",
+						err,
+					);
+					false
+				},
+			}
+		} else {
+			warn!(
+				target: "pow",
+				"Unable to import mined block: build does not exist",
+			);
+			false
+		}
+	}
+}
+
+/// A stream that waits for a block import or timeout.
+pub struct UntilImportedOrTimeout<Block: BlockT> {
+	import_notifications: ImportNotifications<Block>,
+	timeout: Duration,
+	inner_delay: Option<Delay>,
+}
+
+impl<Block: BlockT> UntilImportedOrTimeout<Block> {
+	/// Create a new stream using the given import notification and timeout duration.
+	pub fn new(
+		import_notifications: ImportNotifications<Block>,
+		timeout: Duration,
+	) -> Self {
+		Self {
+			import_notifications,
+			timeout,
+			inner_delay: None,
+		}
+	}
+}
+
+impl<Block: BlockT> Stream for UntilImportedOrTimeout<Block> {
+	type Item = ();
+
+	fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<()>> {
+		let mut fire = false;
+
+		loop {
+			match Stream::poll_next(Pin::new(&mut self.import_notifications), cx) {
+				Poll::Pending => break,
+				Poll::Ready(Some(_)) => {
+					fire = true;
+				},
+				Poll::Ready(None) => return Poll::Ready(None),
+			}
+		}
+
+		let timeout = self.timeout.clone();
+		let inner_delay = self.inner_delay.get_or_insert_with(|| Delay::new(timeout));
+
+		match Future::poll(Pin::new(inner_delay), cx) {
+			Poll::Pending => (),
+			Poll::Ready(()) => {
+				fire = true;
+			},
+		}
+
+		if fire {
+			self.inner_delay = None;
+			Poll::Ready(Some(()))
+		} else {
+			Poll::Pending
+		}
+	}
+}

From ecaf240c0bb1a96a280feeddd6bd71e2fadd4c50 Mon Sep 17 00:00:00 2001
From: Xiliang Chen <xlchen1291@gmail.com>
Date: Fri, 18 Sep 2020 22:19:22 +1200
Subject: [PATCH 107/122] Bounties (#5715)

* add some compact annotation

* implement bounties for treasury

* fix test build

* remove some duplicated code

* fix build

* add tests

* fix build

* fix tests

* rename

* merge deposit byte fee

* add comments

* refactor storage

* support sub bounty

* emit BountyBecameActive when sub bounty is created

* able to contribute bounty

* allow curator to cancel bounty

* remove bounty contribution

* implement bounty expiry

* Able to extend bounty

* fix build and update tests

* create sub bounty test

* add more tests

* add benchmarks for bounties

* fix build

* line width

* fix benchmarking test

* update trait

* fix typo

* Update lib.rs

Missing documentation on Bounties added on this change. Please check the definitions of `propose_bounty` and `create_bounty`.

* update docs

* add MaximumSubBountyDepth

* put BountyValueMinimum into storage

* rework bount depth

* split on_initialize benchmarks

* remove components from constant functions

* Update weight integration into treasury

* Update reject proposal read/writes

* fix weight calculation

* Ignore weights with 0 factor

* Remove 0 multipliers

* add some docs

* allow unused for generated code

* line width

* allow RejectOrigin to cancel a pending payout bounty

* require BountyValueMinimum > ED

* make BountyValueMinimum configurable by chain spec

* remove sub-bounty features

* update curator

* accept curator

* unassign and cancel

* fix tests

* new tests

* Update lib.rs

- Include on `Assign_curator`, `accept_curator` and `unassign_curator` on Bounties Protocol Section
- Include curator fee and curator deposit definitions on Terminology
- Update intro.

* fix test

* update extend_bounty_expiry

* fix benchmarking

* add new benchmarking code

* add docs

* fix tests

* Update benchmarking.rs

* Make BountyValueMinimum a trait config instead of stroage value

* fix runtime build

* Update weights

* Update default_weights.rs

* update weights

* update

* update comments

* unreserve curator fee

* update tests

* update benchmarks

* fix curator deposit handling

* trigger CI

* fix benchmarking

* use append instead of mutate push

* additional noop tests

* improve fee hanlding. update event docs

* RejectOrigin to unassign

* update bounty cancel logic

* use Zero::zero() over 0.into()

* fix tests

* fix benchmarks

* proposed fixes to bounties

* fix tests

* fix benchmarks

* update weightinfo

* use closure

* fix compile

* update weights

Co-authored-by: RRTTI <raul@ost.com>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
---
 bin/node/runtime/src/lib.rs                   |  20 +-
 bin/node/runtime/src/weights/mod.rs           |   1 +
 .../runtime/src/weights/pallet_treasury.rs    | 140 ++++
 frame/treasury/Cargo.toml                     |   1 +
 frame/treasury/src/benchmarking.rs            | 185 ++++-
 frame/treasury/src/default_weights.rs         | 139 ++++
 frame/treasury/src/lib.rs                     | 725 ++++++++++++++++--
 frame/treasury/src/tests.rs                   | 595 ++++++++++++--
 8 files changed, 1668 insertions(+), 138 deletions(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_treasury.rs
 create mode 100644 frame/treasury/src/default_weights.rs

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index 468605ddb7a09..03fe279366d50 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -602,8 +602,14 @@ parameter_types! {
 	pub const TipCountdown: BlockNumber = 1 * DAYS;
 	pub const TipFindersFee: Percent = Percent::from_percent(20);
 	pub const TipReportDepositBase: Balance = 1 * DOLLARS;
-	pub const TipReportDepositPerByte: Balance = 1 * CENTS;
+	pub const DataDepositPerByte: Balance = 1 * CENTS;
+	pub const BountyDepositBase: Balance = 1 * DOLLARS;
+	pub const BountyDepositPayoutDelay: BlockNumber = 1 * DAYS;
 	pub const TreasuryModuleId: ModuleId = ModuleId(*b"py/trsry");
+	pub const BountyUpdatePeriod: BlockNumber = 14 * DAYS;
+	pub const MaximumReasonLength: u32 = 16384;
+	pub const BountyCuratorDeposit: Permill = Permill::from_percent(50);
+	pub const BountyValueMinimum: Balance = 5 * DOLLARS;
 }
 
 impl pallet_treasury::Trait for Runtime {
@@ -623,15 +629,21 @@ impl pallet_treasury::Trait for Runtime {
 	type TipCountdown = TipCountdown;
 	type TipFindersFee = TipFindersFee;
 	type TipReportDepositBase = TipReportDepositBase;
-	type TipReportDepositPerByte = TipReportDepositPerByte;
+	type DataDepositPerByte = DataDepositPerByte;
 	type Event = Event;
-	type ProposalRejection = ();
+	type OnSlash = ();
 	type ProposalBond = ProposalBond;
 	type ProposalBondMinimum = ProposalBondMinimum;
 	type SpendPeriod = SpendPeriod;
 	type Burn = Burn;
+	type BountyDepositBase = BountyDepositBase;
+	type BountyDepositPayoutDelay = BountyDepositPayoutDelay;
+	type BountyUpdatePeriod = BountyUpdatePeriod;
+	type BountyCuratorDeposit = BountyCuratorDeposit;
+	type BountyValueMinimum = BountyValueMinimum;
+	type MaximumReasonLength = MaximumReasonLength;
 	type BurnDestination = ();
-	type WeightInfo = ();
+	type WeightInfo = weights::pallet_treasury::WeightInfo;
 }
 
 parameter_types! {
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index a07aa2e5bc896..199e66888e65d 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -17,6 +17,7 @@
 
 pub mod frame_system;
 pub mod pallet_balances;
+pub mod pallet_treasury;
 pub mod pallet_collective;
 pub mod pallet_democracy;
 pub mod pallet_identity;
diff --git a/bin/node/runtime/src/weights/pallet_treasury.rs b/bin/node/runtime/src/weights/pallet_treasury.rs
new file mode 100644
index 0000000000000..fe1a3166919f0
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_treasury.rs
@@ -0,0 +1,140 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_treasury::WeightInfo for WeightInfo {
+	fn propose_spend() -> Weight {
+		(79604000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn reject_proposal() -> Weight {
+		(61001000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn approve_proposal() -> Weight {
+		(17835000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn report_awesome(r: u32, ) -> Weight {
+		(101602000 as Weight)
+			.saturating_add((2000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	// WARNING! Some components were not used: ["r"]
+	fn retract_tip() -> Weight {
+		(82970000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn tip_new(r: u32, t: u32, ) -> Weight {
+		(63995000 as Weight)
+			.saturating_add((2000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((153000 as Weight).saturating_mul(t as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn tip(t: u32, ) -> Weight {
+		(46765000 as Weight)
+			.saturating_add((711000 as Weight).saturating_mul(t as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn close_tip(t: u32, ) -> Weight {
+		(160874000 as Weight)
+			.saturating_add((379000 as Weight).saturating_mul(t as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn propose_bounty(d: u32, ) -> Weight {
+		(86198000 as Weight)
+			.saturating_add((1000 as Weight).saturating_mul(d as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn approve_bounty() -> Weight {
+		(23063000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn propose_curator() -> Weight {
+		(18890000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn unassign_curator() -> Weight {
+		(66768000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn accept_curator() -> Weight {
+		(69131000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn award_bounty() -> Weight {
+		(48184000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn claim_bounty() -> Weight {
+		(243104000 as Weight)
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(5 as Weight))
+	}
+	fn close_bounty_proposed() -> Weight {
+		(65917000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_bounty_active() -> Weight {
+		(157232000 as Weight)
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn extend_bounty_expiry() -> Weight {
+		(46216000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn on_initialize_proposals(p: u32, ) -> Weight {
+		(119765000 as Weight)
+			.saturating_add((108368000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(p as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(p as Weight)))
+	}
+	fn on_initialize_bounties(b: u32, ) -> Weight {
+		(112536000 as Weight)
+			.saturating_add((107132000 as Weight).saturating_mul(b as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(b as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(b as Weight)))
+	}
+}
diff --git a/frame/treasury/Cargo.toml b/frame/treasury/Cargo.toml
index b6ef83b32eda8..674417cd73e2c 100644
--- a/frame/treasury/Cargo.toml
+++ b/frame/treasury/Cargo.toml
@@ -41,4 +41,5 @@ std = [
 runtime-benchmarks = [
 	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
+	"frame-system/runtime-benchmarks",
 ]
diff --git a/frame/treasury/src/benchmarking.rs b/frame/treasury/src/benchmarking.rs
index a972dc80bd428..5224c1446f452 100644
--- a/frame/treasury/src/benchmarking.rs
+++ b/frame/treasury/src/benchmarking.rs
@@ -47,7 +47,7 @@ fn setup_proposal<T: Trait<I>, I: Instance>(u: u32) -> (
 fn setup_awesome<T: Trait<I>, I: Instance>(length: u32) -> (T::AccountId, Vec<u8>, T::AccountId) {
 	let caller = whitelisted_caller();
 	let value = T::TipReportDepositBase::get()
-		+ T::TipReportDepositPerByte::get() * length.into()
+		+ T::DataDepositPerByte::get() * length.into()
 		+ T::Currency::minimum_balance();
 	let _ = T::Currency::make_free_balance_be(&caller, value);
 	let reason = vec![0; length as usize];
@@ -109,6 +109,58 @@ fn create_approved_proposals<T: Trait<I>, I: Instance>(n: u32) -> Result<(), &'s
 	Ok(())
 }
 
+// Create bounties that are approved for use in `on_initialize`.
+fn create_approved_bounties<T: Trait<I>, I: Instance>(n: u32) -> Result<(), &'static str> {
+	for i in 0 .. n {
+		let (caller, _curator, _fee, value, reason) = setup_bounty::<T, I>(i, MAX_BYTES);
+		Treasury::<T, I>::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?;
+		let bounty_id = BountyCount::<I>::get() - 1;
+		Treasury::<T, I>::approve_bounty(RawOrigin::Root.into(), bounty_id)?;
+	}
+	ensure!(BountyApprovals::<I>::get().len() == n as usize, "Not all bounty approved");
+	Ok(())
+}
+
+// Create the pre-requisite information needed to create a treasury `propose_bounty`.
+fn setup_bounty<T: Trait<I>, I: Instance>(u: u32, d: u32) -> (
+	T::AccountId,
+	T::AccountId,
+	BalanceOf<T, I>,
+	BalanceOf<T, I>,
+	Vec<u8>,
+) {
+	let caller = account("caller", u, SEED);
+	let value: BalanceOf<T, I> = T::Currency::minimum_balance().saturating_mul(100.into());
+	let fee = T::Currency::minimum_balance().saturating_mul(2.into());
+	let deposit = T::BountyDepositBase::get() + T::DataDepositPerByte::get() * MAX_BYTES.into();
+	let _ = T::Currency::make_free_balance_be(&caller, deposit);
+	let curator = account("curator", u, SEED);
+	let _ = T::Currency::make_free_balance_be(&curator, fee / 2.into());
+	let reason = vec![0; d as usize];
+	(caller, curator, fee, value, reason)
+}
+
+fn create_bounty<T: Trait<I>, I: Instance>() -> Result<(
+	<T::Lookup as StaticLookup>::Source,
+	BountyIndex,
+), &'static str> {
+	let (caller, curator, fee, value, reason) = setup_bounty::<T, I>(0, MAX_BYTES);
+	let curator_lookup = T::Lookup::unlookup(curator.clone());
+	Treasury::<T, I>::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?;
+	let bounty_id = BountyCount::<I>::get() - 1;
+	Treasury::<T, I>::approve_bounty(RawOrigin::Root.into(), bounty_id)?;
+	Treasury::<T, I>::on_initialize(T::BlockNumber::zero());
+	Treasury::<T, I>::propose_curator(RawOrigin::Root.into(), bounty_id, curator_lookup.clone(), fee)?;
+	Treasury::<T, I>::accept_curator(RawOrigin::Signed(curator).into(), bounty_id)?;
+	Ok((curator_lookup, bounty_id))
+}
+
+fn setup_pod_account<T: Trait<I>, I: Instance>() {
+	let pot_account = Treasury::<T, I>::account_id();
+	let value = T::Currency::minimum_balance().saturating_mul(1_000_000_000.into());
+	let _ = T::Currency::make_free_balance_be(&pot_account, value);
+}
+
 const MAX_BYTES: u32 = 16384;
 const MAX_TIPPERS: u32 = 100;
 
@@ -116,16 +168,14 @@ benchmarks_instance! {
 	_ { }
 
 	propose_spend {
-		let u in 0 .. 1000;
-		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(u);
+		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(SEED);
 		// Whitelist caller account from further DB operations.
 		let caller_key = frame_system::Account::<T>::hashed_key_for(&caller);
 		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
 	}: _(RawOrigin::Signed(caller), value, beneficiary_lookup)
 
 	reject_proposal {
-		let u in 0 .. 1000;
-		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(u);
+		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(SEED);
 		Treasury::<T, _>::propose_spend(
 			RawOrigin::Signed(caller).into(),
 			value,
@@ -135,8 +185,7 @@ benchmarks_instance! {
 	}: _(RawOrigin::Root, proposal_id)
 
 	approve_proposal {
-		let u in 0 .. 1000;
-		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(u);
+		let (caller, value, beneficiary_lookup) = setup_proposal::<T, _>(SEED);
 		Treasury::<T, _>::propose_spend(
 			RawOrigin::Signed(caller).into(),
 			value,
@@ -202,9 +251,7 @@ benchmarks_instance! {
 		let t in 1 .. MAX_TIPPERS;
 
 		// Make sure pot is funded
-		let pot_account = Treasury::<T, _>::account_id();
-		let value = T::Currency::minimum_balance().saturating_mul(1_000_000_000.into());
-		let _ = T::Currency::make_free_balance_be(&pot_account, value);
+		setup_pod_account::<T, _>();
 
 		// Set up a new tip proposal
 		let (member, reason, beneficiary, value) = setup_tip::<T, _>(0, t)?;
@@ -228,15 +275,112 @@ benchmarks_instance! {
 		frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into());
 	}: _(RawOrigin::Signed(caller), hash)
 
-	on_initialize {
+	propose_bounty {
+		let d in 0 .. MAX_BYTES;
+
+		let (caller, curator, fee, value, description) = setup_bounty::<T, _>(0, d);
+	}: _(RawOrigin::Signed(caller), value, description)
+
+	approve_bounty {
+		let (caller, curator, fee, value, reason) = setup_bounty::<T, _>(0, MAX_BYTES);
+		Treasury::<T, _>::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?;
+		let bounty_id = BountyCount::<I>::get() - 1;
+	}: _(RawOrigin::Root, bounty_id)
+
+	propose_curator {
+		setup_pod_account::<T, _>();
+		let (caller, curator, fee, value, reason) = setup_bounty::<T, _>(0, MAX_BYTES);
+		let curator_lookup = T::Lookup::unlookup(curator.clone());
+		Treasury::<T, _>::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?;
+		let bounty_id = BountyCount::<I>::get() - 1;
+		Treasury::<T, _>::approve_bounty(RawOrigin::Root.into(), bounty_id)?;
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+	}: _(RawOrigin::Root, bounty_id, curator_lookup, fee)
+
+	// Worst case when curator is inactive and any sender unassigns the curator.
+	unassign_curator {
+		setup_pod_account::<T, _>();
+		let (curator_lookup, bounty_id) = create_bounty::<T, _>()?;
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+		let bounty_id = BountyCount::<I>::get() - 1;
+		frame_system::Module::<T>::set_block_number(T::BountyUpdatePeriod::get() + 1.into());
+		let caller = whitelisted_caller();
+	}: _(RawOrigin::Signed(caller), bounty_id)
+
+	accept_curator {
+		setup_pod_account::<T, _>();
+		let (caller, curator, fee, value, reason) = setup_bounty::<T, _>(0, MAX_BYTES);
+		let curator_lookup = T::Lookup::unlookup(curator.clone());
+		Treasury::<T, _>::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?;
+		let bounty_id = BountyCount::<I>::get() - 1;
+		Treasury::<T, _>::approve_bounty(RawOrigin::Root.into(), bounty_id)?;
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+		Treasury::<T, _>::propose_curator(RawOrigin::Root.into(), bounty_id, curator_lookup, fee)?;
+	}: _(RawOrigin::Signed(curator), bounty_id)
+
+	award_bounty {
+		setup_pod_account::<T, _>();
+		let (curator_lookup, bounty_id) = create_bounty::<T, _>()?;
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+
+		let bounty_id = BountyCount::<I>::get() - 1;
+		let curator = T::Lookup::lookup(curator_lookup)?;
+		let beneficiary = T::Lookup::unlookup(account("beneficiary", 0, SEED));
+	}: _(RawOrigin::Signed(curator), bounty_id, beneficiary)
+
+	claim_bounty {
+		setup_pod_account::<T, _>();
+		let (curator_lookup, bounty_id) = create_bounty::<T, _>()?;
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+
+		let bounty_id = BountyCount::<I>::get() - 1;
+		let curator = T::Lookup::lookup(curator_lookup)?;
+
+		let beneficiary = T::Lookup::unlookup(account("beneficiary", 0, SEED));
+		Treasury::<T, _>::award_bounty(RawOrigin::Signed(curator.clone()).into(), bounty_id, beneficiary)?;
+
+		frame_system::Module::<T>::set_block_number(T::BountyDepositPayoutDelay::get());
+
+	}: _(RawOrigin::Signed(curator), bounty_id)
+
+	close_bounty_proposed {
+		setup_pod_account::<T, _>();
+		let (caller, curator, fee, value, reason) = setup_bounty::<T, _>(0, 0);
+		Treasury::<T, _>::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?;
+		let bounty_id = BountyCount::<I>::get() - 1;
+	}: close_bounty(RawOrigin::Root, bounty_id)
+
+	close_bounty_active {
+		setup_pod_account::<T, _>();
+		let (curator_lookup, bounty_id) = create_bounty::<T, _>()?;
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+		let bounty_id = BountyCount::<I>::get() - 1;
+	}: close_bounty(RawOrigin::Root, bounty_id)
+
+	extend_bounty_expiry {
+		setup_pod_account::<T, _>();
+		let (curator_lookup, bounty_id) = create_bounty::<T, _>()?;
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+
+		let bounty_id = BountyCount::<I>::get() - 1;
+		let curator = T::Lookup::lookup(curator_lookup)?;
+	}: _(RawOrigin::Signed(curator), bounty_id, Vec::new())
+
+	on_initialize_proposals {
 		let p in 0 .. 100;
-		let pot_account = Treasury::<T, _>::account_id();
-		let value = T::Currency::minimum_balance().saturating_mul(1_000_000_000.into());
-		let _ = T::Currency::make_free_balance_be(&pot_account, value);
+		setup_pod_account::<T, _>();
 		create_approved_proposals::<T, _>(p)?;
 	}: {
 		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
 	}
+
+	on_initialize_bounties {
+		let b in 0 .. 100;
+		setup_pod_account::<T, _>();
+		create_approved_bounties::<T, _>(b)?;
+	}: {
+		Treasury::<T, _>::on_initialize(T::BlockNumber::zero());
+	}
 }
 
 #[cfg(test)]
@@ -256,7 +400,18 @@ mod tests {
 			assert_ok!(test_benchmark_tip_new::<Test>());
 			assert_ok!(test_benchmark_tip::<Test>());
 			assert_ok!(test_benchmark_close_tip::<Test>());
-			assert_ok!(test_benchmark_on_initialize::<Test>());
+			assert_ok!(test_benchmark_propose_bounty::<Test>());
+			assert_ok!(test_benchmark_approve_bounty::<Test>());
+			assert_ok!(test_benchmark_propose_curator::<Test>());
+			assert_ok!(test_benchmark_unassign_curator::<Test>());
+			assert_ok!(test_benchmark_accept_curator::<Test>());
+			assert_ok!(test_benchmark_award_bounty::<Test>());
+			assert_ok!(test_benchmark_claim_bounty::<Test>());
+			assert_ok!(test_benchmark_close_bounty_proposed::<Test>());
+			assert_ok!(test_benchmark_close_bounty_active::<Test>());
+			assert_ok!(test_benchmark_extend_bounty_expiry::<Test>());
+			assert_ok!(test_benchmark_on_initialize_proposals::<Test>());
+			assert_ok!(test_benchmark_on_initialize_bounties::<Test>());
 		});
 	}
 }
diff --git a/frame/treasury/src/default_weights.rs b/frame/treasury/src/default_weights.rs
new file mode 100644
index 0000000000000..faf6c1a04573d
--- /dev/null
+++ b/frame/treasury/src/default_weights.rs
@@ -0,0 +1,139 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn propose_spend() -> Weight {
+		(79604000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn reject_proposal() -> Weight {
+		(61001000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn approve_proposal() -> Weight {
+		(17835000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn report_awesome(r: u32, ) -> Weight {
+		(101602000 as Weight)
+			.saturating_add((2000 as Weight).saturating_mul(r as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	// WARNING! Some components were not used: ["r"]
+	fn retract_tip() -> Weight {
+		(82970000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn tip_new(r: u32, t: u32, ) -> Weight {
+		(63995000 as Weight)
+			.saturating_add((2000 as Weight).saturating_mul(r as Weight))
+			.saturating_add((153000 as Weight).saturating_mul(t as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn tip(t: u32, ) -> Weight {
+		(46765000 as Weight)
+			.saturating_add((711000 as Weight).saturating_mul(t as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn close_tip(t: u32, ) -> Weight {
+		(160874000 as Weight)
+			.saturating_add((379000 as Weight).saturating_mul(t as Weight))
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn propose_bounty(d: u32, ) -> Weight {
+		(86198000 as Weight)
+			.saturating_add((1000 as Weight).saturating_mul(d as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn approve_bounty() -> Weight {
+		(23063000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn propose_curator() -> Weight {
+		(18890000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn unassign_curator() -> Weight {
+		(66768000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn accept_curator() -> Weight {
+		(69131000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn award_bounty() -> Weight {
+		(48184000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn claim_bounty() -> Weight {
+		(243104000 as Weight)
+			.saturating_add(DbWeight::get().reads(4 as Weight))
+			.saturating_add(DbWeight::get().writes(5 as Weight))
+	}
+	fn close_bounty_proposed() -> Weight {
+		(65917000 as Weight)
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(3 as Weight))
+	}
+	fn close_bounty_active() -> Weight {
+		(157232000 as Weight)
+			.saturating_add(DbWeight::get().reads(3 as Weight))
+			.saturating_add(DbWeight::get().writes(4 as Weight))
+	}
+	fn extend_bounty_expiry() -> Weight {
+		(46216000 as Weight)
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn on_initialize_proposals(p: u32, ) -> Weight {
+		(119765000 as Weight)
+			.saturating_add((108368000 as Weight).saturating_mul(p as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(p as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(p as Weight)))
+	}
+	fn on_initialize_bounties(b: u32, ) -> Weight {
+		(112536000 as Weight)
+			.saturating_add((107132000 as Weight).saturating_mul(b as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().reads((3 as Weight).saturating_mul(b as Weight)))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+			.saturating_add(DbWeight::get().writes((3 as Weight).saturating_mul(b as Weight)))
+	}
+}
diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs
index cedc46eb8c0dd..4d76f2f532151 100644
--- a/frame/treasury/src/lib.rs
+++ b/frame/treasury/src/lib.rs
@@ -44,6 +44,23 @@
 //! countdown period, the median of all declared tips is paid to the reported beneficiary, along
 //! with any finders fee, in case of a public (and bonded) original report.
 //!
+//! ### Bounty
+//!
+//! A Bounty Spending is a reward for a specified body of work - or specified set of objectives - that
+//! needs to be executed for a predefined Treasury amount to be paid out. A curator is assigned after
+//! the bounty is approved and funded by Council, to be delegated
+//! with the responsibility of assigning a payout address once the specified set of objectives is completed.
+//!
+//! After the Council has activated a bounty, it delegates the work that requires expertise to a curator
+//! in exchange of a deposit. Once the curator accepts the bounty, they
+//! get to close the Active bounty. Closing the Active bounty enacts a delayed payout to the payout
+//! address, the curator fee and the return of the curator deposit. The
+//! delay allows for intervention through regular democracy. The Council gets to unassign the curator,
+//! resulting in a new curator election. The Council also gets to cancel
+//! the bounty if deemed necessary before assigning a curator or once the bounty is active or payout
+//! is pending, resulting in the slash of the curator's deposit.
+//!
+//!
 //! ### Terminology
 //!
 //! - **Proposal:** A suggestion to allocate funds from the pot to a beneficiary.
@@ -64,6 +81,22 @@
 //! - **Finders Fee:** Some proportion of the tip amount that is paid to the reporter of the tip,
 //!   rather than the main beneficiary.
 //!
+//! Bounty:
+//! - **Bounty spending proposal:** A proposal to reward a predefined body of work upon completion by
+//! the Treasury.
+//! - **Proposer:** An account proposing a bounty spending.
+//! - **Curator:** An account managing the bounty and assigning a payout address receiving the reward
+//! for the completion of work.
+//! - **Deposit:** The amount held on deposit for placing a bounty proposal plus the amount held on
+//! deposit per byte within the bounty description.
+//! - **Curator deposit:** The payment from a candidate willing to curate an approved bounty. The deposit
+//! is returned when/if the bounty is completed.
+//! - **Bounty value:** The total amount that should be paid to the Payout Address if the bounty is
+//! rewarded.
+//! - **Payout address:** The account to which the total or part of the bounty is assigned to.
+//! - **Payout Delay:** The delay period for which a bounty beneficiary needs to wait before claiming.
+//! - **Curator fee:** The reserved upfront payment for a curator for work related to the bounty.
+//!
 //! ## Interface
 //!
 //! ### Dispatchable Functions
@@ -82,6 +115,19 @@
 //! - `tip` - Declare or redeclare an amount to tip for a particular reason.
 //! - `close_tip` - Close and pay out a tip.
 //!
+//! Bounty protocol:
+//! - `propose_bounty` - Propose a specific treasury amount to be earmarked for a predefined set of
+//! tasks and stake the required deposit.
+//! - `approve_bounty` - Accept a specific treasury amount to be earmarked for a predefined body of work.
+//! - `propose_curator` - Assign an account to a bounty as candidate curator.
+//! - `accept_curator` - Accept a bounty assignment from the Council, setting a curator deposit.
+//! - `extend_bounty_expiry` - Extend the expiry block number of the bounty and stay active.
+//! - `award_bounty` - Close and pay out the specified amount for the completed work.
+//! - `claim_bounty` - Claim a specific bounty amount from the Payout Address.
+//! - `unassign_curator` - Unassign an accepted curator from a specific earmark.
+//! - `close_bounty` - Cancel the earmark for a specific treasury amount and close the bounty.
+//!
+//!
 //! ## GenesisConfig
 //!
 //! The Treasury module depends on the [`GenesisConfig`](./struct.GenesisConfig.html).
@@ -93,12 +139,13 @@ use serde::{Serialize, Deserialize};
 use sp_std::prelude::*;
 use frame_support::{decl_module, decl_storage, decl_event, ensure, print, decl_error, Parameter};
 use frame_support::traits::{
-	Currency, Get, Imbalance, OnUnbalanced, ExistenceRequirement::KeepAlive,
+	Currency, Get, Imbalance, OnUnbalanced, ExistenceRequirement::{KeepAlive, AllowDeath},
 	ReservableCurrency, WithdrawReason
 };
-use sp_runtime::{Permill, ModuleId, Percent, RuntimeDebug, traits::{
+use sp_runtime::{Permill, ModuleId, Percent, RuntimeDebug, DispatchResult, traits::{
 	Zero, StaticLookup, AccountIdConversion, Saturating, Hash, BadOrigin
 }};
+use frame_support::dispatch::DispatchResultWithPostInfo;
 use frame_support::weights::{Weight, DispatchClass};
 use frame_support::traits::{Contains, ContainsLengthBound, EnsureOrigin};
 use codec::{Encode, Decode};
@@ -106,6 +153,7 @@ use frame_system::{self as system, ensure_signed};
 
 mod tests;
 mod benchmarking;
+mod default_weights;
 
 type BalanceOf<T, I> =
 	<<T as Trait<I>>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
@@ -115,27 +163,26 @@ type NegativeImbalanceOf<T, I> =
 	<<T as Trait<I>>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::NegativeImbalance;
 
 pub trait WeightInfo {
-	fn propose_spend(u: u32, ) -> Weight;
-	fn reject_proposal(u: u32, ) -> Weight;
-	fn approve_proposal(u: u32, ) -> Weight;
+	fn propose_spend() -> Weight;
+	fn reject_proposal() -> Weight;
+	fn approve_proposal() -> Weight;
 	fn report_awesome(r: u32, ) -> Weight;
-	fn retract_tip(r: u32, ) -> Weight;
+	fn retract_tip() -> Weight;
 	fn tip_new(r: u32, t: u32, ) -> Weight;
 	fn tip(t: u32, ) -> Weight;
 	fn close_tip(t: u32, ) -> Weight;
-	fn on_initialize(p: u32, ) -> Weight;
-}
-
-impl WeightInfo for () {
-	fn propose_spend(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn reject_proposal(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn approve_proposal(_u: u32, ) -> Weight { 1_000_000_000 }
-	fn report_awesome(_r: u32, ) -> Weight { 1_000_000_000 }
-	fn retract_tip(_r: u32, ) -> Weight { 1_000_000_000 }
-	fn tip_new(_r: u32, _t: u32, ) -> Weight { 1_000_000_000 }
-	fn tip(_t: u32, ) -> Weight { 1_000_000_000 }
-	fn close_tip(_t: u32, ) -> Weight { 1_000_000_000 }
-	fn on_initialize(_p: u32, ) -> Weight { 1_000_000_000 }
+	fn propose_bounty(r: u32, ) -> Weight;
+	fn approve_bounty() -> Weight;
+	fn propose_curator() -> Weight;
+	fn unassign_curator() -> Weight;
+	fn accept_curator() -> Weight;
+	fn award_bounty() -> Weight;
+	fn claim_bounty() -> Weight;
+	fn close_bounty_proposed() -> Weight;
+	fn close_bounty_active() -> Weight;
+	fn extend_bounty_expiry() -> Weight;
+	fn on_initialize_proposals(p: u32, ) -> Weight;
+	fn on_initialize_bounties(b: u32, ) -> Weight;
 }
 
 pub trait Trait<I=DefaultInstance>: frame_system::Trait {
@@ -165,14 +212,14 @@ pub trait Trait<I=DefaultInstance>: frame_system::Trait {
 	/// The amount held on deposit for placing a tip report.
 	type TipReportDepositBase: Get<BalanceOf<Self, I>>;
 
-	/// The amount held on deposit per byte within the tip report reason.
-	type TipReportDepositPerByte: Get<BalanceOf<Self, I>>;
+	/// The amount held on deposit per byte within the tip report reason or bounty description.
+	type DataDepositPerByte: Get<BalanceOf<Self, I>>;
 
 	/// The overarching event type.
 	type Event: From<Event<Self, I>> + Into<<Self as frame_system::Trait>::Event>;
 
-	/// Handler for the unbalanced decrease when slashing for a rejected proposal.
-	type ProposalRejection: OnUnbalanced<NegativeImbalanceOf<Self, I>>;
+	/// Handler for the unbalanced decrease when slashing for a rejected proposal or bounty.
+	type OnSlash: OnUnbalanced<NegativeImbalanceOf<Self, I>>;
 
 	/// Fraction of a proposal's value that should be bonded in order to place the proposal.
 	/// An accepted proposal gets these back. A rejected proposal does not.
@@ -187,6 +234,24 @@ pub trait Trait<I=DefaultInstance>: frame_system::Trait {
 	/// Percentage of spare funds (if any) that are burnt per spend period.
 	type Burn: Get<Permill>;
 
+	/// The amount held on deposit for placing a bounty proposal.
+	type BountyDepositBase: Get<BalanceOf<Self, I>>;
+
+	/// The delay period for which a bounty beneficiary need to wait before claim the payout.
+	type BountyDepositPayoutDelay: Get<Self::BlockNumber>;
+
+	/// Bounty duration in blocks.
+	type BountyUpdatePeriod: Get<Self::BlockNumber>;
+
+	/// Percentage of the curator fee that will be reserved upfront as deposit for bounty curator.
+	type BountyCuratorDeposit: Get<Permill>;
+
+	/// Minimum value for a bounty.
+	type BountyValueMinimum: Get<BalanceOf<Self, I>>;
+
+	/// Maximum acceptable reason length.
+	type MaximumReasonLength: Get<u32>;
+
 	/// Handler for the unbalanced decrease when treasury funds are burned.
 	type BurnDestination: OnUnbalanced<NegativeImbalanceOf<Self, I>>;
 
@@ -238,6 +303,58 @@ pub struct OpenTip<
 	finders_fee: bool,
 }
 
+/// An index of a bounty. Just a `u32`.
+pub type BountyIndex = u32;
+
+/// A bounty proposal.
+#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)]
+pub struct Bounty<AccountId, Balance, BlockNumber> {
+	/// The account proposing it.
+	proposer: AccountId,
+	/// The (total) amount that should be paid if the bounty is rewarded.
+	value: Balance,
+	/// The curator fee. Included in value.
+	fee: Balance,
+	/// The deposit of curator.
+	curator_deposit: Balance,
+	/// The amount held on deposit (reserved) for making this proposal.
+	bond: Balance,
+	/// The status of this bounty.
+	status: BountyStatus<AccountId, BlockNumber>,
+}
+
+/// The status of a bounty proposal.
+#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)]
+pub enum BountyStatus<AccountId, BlockNumber> {
+	/// The bounty is proposed and waiting for approval.
+	Proposed,
+	/// The bounty is approved and waiting to become active at next spend period.
+	Approved,
+	/// The bounty is funded and waiting for curator assignment.
+	Funded,
+	/// A curator has been proposed by the `ApproveOrigin`. Waiting for acceptance from the curator.
+	CuratorProposed {
+		/// The assigned curator of this bounty.
+		curator: AccountId,
+	},
+	/// The bounty is active and waiting to be awarded.
+	Active {
+		/// The curator of this bounty.
+		curator: AccountId,
+		/// An update from the curator is due by this block, else they are considered inactive.
+		update_due: BlockNumber,
+	},
+	/// The bounty is awarded and waiting to released after a delay.
+	PendingPayout {
+		/// The curator of this bounty.
+		curator: AccountId,
+		/// The beneficiary of the bounty.
+		beneficiary: AccountId,
+		/// When the bounty can be claimed.
+		unlock_at: BlockNumber,
+	},
+}
+
 decl_storage! {
 	trait Store for Module<T: Trait<I>, I: Instance=DefaultInstance> as Treasury {
 		/// Number of proposals that have been made.
@@ -261,6 +378,20 @@ decl_storage! {
 		/// Simple preimage lookup from the reason's hash to the original data. Again, has an
 		/// insecure enumerable hash since the key is guaranteed to be the result of a secure hash.
 		pub Reasons get(fn reasons): map hasher(identity) T::Hash => Option<Vec<u8>>;
+
+		/// Number of bounty proposals that have been made.
+		pub BountyCount get(fn bounty_count): BountyIndex;
+
+		/// Bounties that have been made.
+		pub Bounties get(fn bounties):
+			map hasher(twox_64_concat) BountyIndex
+			=> Option<Bounty<T::AccountId, BalanceOf<T, I>, T::BlockNumber>>;
+
+		/// The description of each bounty.
+		pub BountyDescriptions get(fn bounty_descriptions): map hasher(twox_64_concat) BountyIndex => Option<Vec<u8>>;
+
+		/// Bounty indices that have been approved but not yet funded.
+		pub BountyApprovals get(fn bounty_approvals): Vec<BountyIndex>;
 	}
 	add_extra_genesis {
 		build(|_config| {
@@ -303,6 +434,20 @@ decl_event!(
 		TipClosed(Hash, AccountId, Balance),
 		/// A tip suggestion has been retracted. \[tip_hash\]
 		TipRetracted(Hash),
+		/// New bounty proposal. [index]
+		BountyProposed(BountyIndex),
+		/// A bounty proposal was rejected; funds were slashed. [index, bond]
+		BountyRejected(BountyIndex, Balance),
+		/// A bounty proposal is funded and became active. [index]
+		BountyBecameActive(BountyIndex),
+		/// A bounty is awarded to a beneficiary. [index, beneficiary]
+		BountyAwarded(BountyIndex, AccountId),
+		/// A bounty is claimed by beneficiary. [index, payout, beneficiary]
+		BountyClaimed(BountyIndex, Balance, AccountId),
+		/// A bounty is cancelled. [index]
+		BountyCanceled(BountyIndex),
+		/// A bounty expiry is extended. [index]
+		BountyExtended(BountyIndex),
 	}
 );
 
@@ -311,8 +456,8 @@ decl_error! {
 	pub enum Error for Module<T: Trait<I>, I: Instance> {
 		/// Proposer's balance is too low.
 		InsufficientProposersBalance,
-		/// No proposal at that index.
-		InvalidProposalIndex,
+		/// No proposal or bounty at that index.
+		InvalidIndex,
 		/// The reason given is just too big.
 		ReasonTooBig,
 		/// The tip was already found/started.
@@ -325,6 +470,17 @@ decl_error! {
 		StillOpen,
 		/// The tip cannot be claimed/closed because it's still in the countdown period.
 		Premature,
+		/// The bounty status is unexpected.
+		UnexpectedStatus,
+		/// Require bounty curator.
+		RequireCurator,
+		/// Invalid bounty value.
+		InvalidValue,
+		/// Invalid bounty fee.
+		InvalidFee,
+		/// A bounty payout is pending.
+		/// To cancel the bounty, you must unassign and slash the curator.
+		PendingPayout,
 	}
 }
 
@@ -355,12 +511,26 @@ decl_module! {
 		/// The amount held on deposit for placing a tip report.
 		const TipReportDepositBase: BalanceOf<T, I> = T::TipReportDepositBase::get();
 
-		/// The amount held on deposit per byte within the tip report reason.
-		const TipReportDepositPerByte: BalanceOf<T, I> = T::TipReportDepositPerByte::get();
+		/// The amount held on deposit per byte within the tip report reason or bounty description.
+		const DataDepositPerByte: BalanceOf<T, I> = T::DataDepositPerByte::get();
 
 		/// The treasury's module id, used for deriving its sovereign account ID.
 		const ModuleId: ModuleId = T::ModuleId::get();
 
+		/// The amount held on deposit for placing a bounty proposal.
+		const BountyDepositBase: BalanceOf<T, I> = T::BountyDepositBase::get();
+
+		/// The delay period for which a bounty beneficiary need to wait before claim the payout.
+		const BountyDepositPayoutDelay: T::BlockNumber = T::BountyDepositPayoutDelay::get();
+
+		/// Percentage of the curator fee that will be reserved upfront as deposit for bounty curator.
+		const BountyCuratorDeposit: Permill = T::BountyCuratorDeposit::get();
+
+		const BountyValueMinimum: BalanceOf<T, I> = T::BountyValueMinimum::get();
+
+		/// Maximum acceptable reason length.
+		const MaximumReasonLength: u32 = T::MaximumReasonLength::get();
+
 		type Error = Error<T, I>;
 
 		fn deposit_event() = default;
@@ -374,7 +544,7 @@ decl_module! {
 		/// - DbReads: `ProposalCount`, `origin account`
 		/// - DbWrites: `ProposalCount`, `Proposals`, `origin account`
 		/// # </weight>
-		#[weight = 120_000_000 + T::DbWeight::get().reads_writes(1, 2)]
+		#[weight = T::WeightInfo::propose_spend()]
 		fn propose_spend(
 			origin,
 			#[compact] value: BalanceOf<T, I>,
@@ -403,14 +573,14 @@ decl_module! {
 		/// - DbReads: `Proposals`, `rejected proposer account`
 		/// - DbWrites: `Proposals`, `rejected proposer account`
 		/// # </weight>
-		#[weight = (130_000_000 + T::DbWeight::get().reads_writes(2, 2), DispatchClass::Operational)]
+		#[weight = (T::WeightInfo::reject_proposal(), DispatchClass::Operational)]
 		fn reject_proposal(origin, #[compact] proposal_id: ProposalIndex) {
 			T::RejectOrigin::ensure_origin(origin)?;
 
-			let proposal = <Proposals<T, I>>::take(&proposal_id).ok_or(Error::<T, I>::InvalidProposalIndex)?;
+			let proposal = <Proposals<T, I>>::take(&proposal_id).ok_or(Error::<T, I>::InvalidIndex)?;
 			let value = proposal.bond;
 			let imbalance = T::Currency::slash_reserved(&proposal.proposer, value).0;
-			T::ProposalRejection::on_unbalanced(imbalance);
+			T::OnSlash::on_unbalanced(imbalance);
 
 			Self::deposit_event(Event::<T, I>::Rejected(proposal_id, value));
 		}
@@ -425,12 +595,12 @@ decl_module! {
 		/// - DbReads: `Proposals`, `Approvals`
 		/// - DbWrite: `Approvals`
 		/// # </weight>
-		#[weight = (34_000_000 + T::DbWeight::get().reads_writes(2, 1), DispatchClass::Operational)]
+		#[weight = (T::WeightInfo::approve_proposal(), DispatchClass::Operational)]
 		fn approve_proposal(origin, #[compact] proposal_id: ProposalIndex) {
 			T::ApproveOrigin::ensure_origin(origin)?;
 
-			ensure!(<Proposals<T, I>>::contains_key(proposal_id), Error::<T, I>::InvalidProposalIndex);
-			<Approvals<I>>::mutate(|v| v.push(proposal_id));
+			ensure!(<Proposals<T, I>>::contains_key(proposal_id), Error::<T, I>::InvalidIndex);
+			Approvals::<I>::append(proposal_id);
 		}
 
 		/// Report something `reason` that deserves a tip and claim any eventual the finder's fee.
@@ -438,7 +608,7 @@ decl_module! {
 		/// The dispatch origin for this call must be _Signed_.
 		///
 		/// Payment: `TipReportDepositBase` will be reserved from the origin account, as well as
-		/// `TipReportDepositPerByte` for each byte in `reason`.
+		/// `DataDepositPerByte` for each byte in `reason`.
 		///
 		/// - `reason`: The reason for, or the thing that deserves, the tip; generally this will be
 		///   a UTF-8-encoded URL.
@@ -449,15 +619,14 @@ decl_module! {
 		/// # <weight>
 		/// - Complexity: `O(R)` where `R` length of `reason`.
 		///   - encoding and hashing of 'reason'
-		/// - DbReads: `Reasons`, `Tips`, `who account data`
-		/// - DbWrites: `Tips`, `who account data`
+		/// - DbReads: `Reasons`, `Tips`
+		/// - DbWrites: `Reasons`, `Tips`
 		/// # </weight>
-		#[weight = 140_000_000 + 4_000 * reason.len() as Weight + T::DbWeight::get().reads_writes(3, 2)]
+		#[weight = T::WeightInfo::report_awesome(reason.len() as u32)]
 		fn report_awesome(origin, reason: Vec<u8>, who: T::AccountId) {
 			let finder = ensure_signed(origin)?;
 
-			const MAX_SENSIBLE_REASON_LENGTH: usize = 16384;
-			ensure!(reason.len() <= MAX_SENSIBLE_REASON_LENGTH, Error::<T, I>::ReasonTooBig);
+			ensure!(reason.len() <= T::MaximumReasonLength::get() as usize, Error::<T, I>::ReasonTooBig);
 
 			let reason_hash = T::Hashing::hash(&reason[..]);
 			ensure!(!Reasons::<T, I>::contains_key(&reason_hash), Error::<T, I>::AlreadyKnown);
@@ -465,7 +634,7 @@ decl_module! {
 			ensure!(!Tips::<T, I>::contains_key(&hash), Error::<T, I>::AlreadyKnown);
 
 			let deposit = T::TipReportDepositBase::get()
-				+ T::TipReportDepositPerByte::get() * (reason.len() as u32).into();
+				+ T::DataDepositPerByte::get() * (reason.len() as u32).into();
 			T::Currency::reserve(&finder, deposit)?;
 
 			Reasons::<T, I>::insert(&reason_hash, &reason);
@@ -501,7 +670,7 @@ decl_module! {
 		/// - DbReads: `Tips`, `origin account`
 		/// - DbWrites: `Reasons`, `Tips`, `origin account`
 		/// # </weight>
-		#[weight = 120_000_000 + T::DbWeight::get().reads_writes(1, 2)]
+		#[weight = T::WeightInfo::retract_tip()]
 		fn retract_tip(origin, hash: T::Hash) {
 			let who = ensure_signed(origin)?;
 			let tip = Tips::<T, I>::get(&hash).ok_or(Error::<T, I>::UnknownTip)?;
@@ -537,11 +706,8 @@ decl_module! {
 		/// - DbReads: `Tippers`, `Reasons`
 		/// - DbWrites: `Reasons`, `Tips`
 		/// # </weight>
-		#[weight = 110_000_000
-			+ 4_000 * reason.len() as Weight
-			+ 480_000 * T::Tippers::max_len() as Weight
-			+ T::DbWeight::get().reads_writes(2, 2)]
-		fn tip_new(origin, reason: Vec<u8>, who: T::AccountId, tip_value: BalanceOf<T, I>) {
+		#[weight = T::WeightInfo::tip_new(reason.len() as u32, T::Tippers::max_len() as u32)]
+		fn tip_new(origin, reason: Vec<u8>, who: T::AccountId, #[compact] tip_value: BalanceOf<T, I>) {
 			let tipper = ensure_signed(origin)?;
 			ensure!(T::Tippers::contains(&tipper), BadOrigin);
 			let reason_hash = T::Hashing::hash(&reason[..]);
@@ -588,9 +754,8 @@ decl_module! {
 		/// - DbReads: `Tippers`, `Tips`
 		/// - DbWrites: `Tips`
 		/// # </weight>
-		#[weight = 68_000_000 + 2_000_000 * T::Tippers::max_len() as Weight
-			+ T::DbWeight::get().reads_writes(2, 1)]
-		fn tip(origin, hash: T::Hash, tip_value: BalanceOf<T, I>) {
+		#[weight = T::WeightInfo::tip(T::Tippers::max_len() as u32)]
+		fn tip(origin, hash: T::Hash, #[compact] tip_value: BalanceOf<T, I>) {
 			let tipper = ensure_signed(origin)?;
 			ensure!(T::Tippers::contains(&tipper), BadOrigin);
 
@@ -618,8 +783,7 @@ decl_module! {
 		/// - DbReads: `Tips`, `Tippers`, `tip finder`
 		/// - DbWrites: `Reasons`, `Tips`, `Tippers`, `tip finder`
 		/// # </weight>
-		#[weight = 220_000_000 + 1_100_000 * T::Tippers::max_len() as Weight
-			+ T::DbWeight::get().reads_writes(3, 3)]
+		#[weight = T::WeightInfo::close_tip(T::Tippers::max_len() as u32)]
 		fn close_tip(origin, hash: T::Hash) {
 			ensure_signed(origin)?;
 
@@ -632,6 +796,371 @@ decl_module! {
 			Self::payout_tip(hash, tip);
 		}
 
+		/// Propose a new bounty.
+		///
+		/// The dispatch origin for this call must be _Signed_.
+		///
+		/// Payment: `TipReportDepositBase` will be reserved from the origin account, as well as
+		/// `DataDepositPerByte` for each byte in `reason`. It will be unreserved upon approval,
+		/// or slashed when rejected.
+		///
+		/// - `curator`: The curator account whom will manage this bounty.
+		/// - `fee`: The curator fee.
+		/// - `value`: The total payment amount of this bounty, curator fee included.
+		/// - `description`: The description of this bounty.
+		#[weight = T::WeightInfo::propose_bounty(description.len() as u32)]
+		fn propose_bounty(
+			origin,
+			#[compact] value: BalanceOf<T, I>,
+			description: Vec<u8>,
+		) {
+			let proposer = ensure_signed(origin)?;
+			Self::create_bounty(proposer, description, value)?;
+		}
+
+		/// Approve a bounty proposal. At a later time, the bounty will be funded and become active
+		/// and the original deposit will be returned.
+		///
+		/// May only be called from `T::ApproveOrigin`.
+		///
+		/// # <weight>
+		/// - O(1).
+		/// - Limited storage reads.
+		/// - One DB change.
+		/// # </weight>
+		#[weight = T::WeightInfo::approve_bounty()]
+		fn approve_bounty(origin, #[compact] bounty_id: ProposalIndex) {
+			T::ApproveOrigin::ensure_origin(origin)?;
+
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult {
+				let mut bounty = maybe_bounty.as_mut().ok_or(Error::<T, I>::InvalidIndex)?;
+				ensure!(bounty.status == BountyStatus::Proposed, Error::<T, I>::UnexpectedStatus);
+
+				bounty.status = BountyStatus::Approved;
+
+				BountyApprovals::<I>::append(bounty_id);
+
+				Ok(())
+			})?;
+		}
+
+		/// Assign a curator to a funded bounty.
+		///
+		/// May only be called from `T::ApproveOrigin`.
+		///
+		/// # <weight>
+		/// - O(1).
+		/// - Limited storage reads.
+		/// - One DB change.
+		/// # </weight>
+		#[weight = T::WeightInfo::propose_curator()]
+		fn propose_curator(
+			origin,
+			#[compact] bounty_id: ProposalIndex,
+			curator: <T::Lookup as StaticLookup>::Source,
+			#[compact] fee: BalanceOf<T, I>,
+		) {
+			T::ApproveOrigin::ensure_origin(origin)?;
+
+			let curator = T::Lookup::lookup(curator)?;
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult {
+				let mut bounty = maybe_bounty.as_mut().ok_or(Error::<T, I>::InvalidIndex)?;
+				match bounty.status {
+					BountyStatus::Funded | BountyStatus::CuratorProposed { .. } => {},
+					_ => return Err(Error::<T, I>::UnexpectedStatus.into()),
+				};
+
+				ensure!(fee < bounty.value, Error::<T, I>::InvalidFee);
+
+				bounty.status = BountyStatus::CuratorProposed { curator };
+				bounty.fee = fee;
+
+				Ok(())
+			})?;
+		}
+
+		/// Unassign curator from a bounty.
+		///
+		/// This function can only be called by the `RejectOrigin` a signed origin.
+		///
+		/// If this function is called by the `RejectOrigin`, we assume that the curator is malicious
+		/// or inactive. As a result, we will slash the curator when possible.
+		///
+		/// If the origin is the curator, we take this as a sign they are unable to do their job and
+		/// they willingly give up. We could slash them, but for now we allow them to recover their
+		/// deposit and exit without issue. (We may want to change this if it is abused.)
+		///
+		/// Finally, the origin can be anyone if and only if the curator is "inactive". This allows
+		/// anyone in the community to call out that a curator is not doing their due diligence, and
+		/// we should pick a new curator. In this case the curator should also be slashed.
+		///
+		/// # <weight>
+		/// - O(1).
+		/// - Limited storage reads.
+		/// - One DB change.
+		/// # </weight>
+		#[weight = T::WeightInfo::unassign_curator()]
+		fn unassign_curator(
+			origin,
+			#[compact] bounty_id: ProposalIndex,
+		) {
+			let maybe_sender = ensure_signed(origin.clone())
+				.map(Some)
+				.or_else(|_| T::RejectOrigin::ensure_origin(origin).map(|_| None))?;
+
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult {
+				let mut bounty = maybe_bounty.as_mut().ok_or(Error::<T, I>::InvalidIndex)?;
+
+				let slash_curator = |curator: &T::AccountId, curator_deposit: &mut BalanceOf<T, I>| {
+					let imbalance = T::Currency::slash_reserved(curator, *curator_deposit).0;
+					T::OnSlash::on_unbalanced(imbalance);
+					*curator_deposit = Zero::zero();
+				};
+
+				match bounty.status {
+					BountyStatus::Proposed | BountyStatus::Approved | BountyStatus::Funded => {
+						// No curator to unassign at this point.
+						return Err(Error::<T, I>::UnexpectedStatus.into())
+					}
+					BountyStatus::CuratorProposed { ref curator } => {
+						// A curator has been proposed, but not accepted yet.
+						// Either `RejectOrigin` or the proposed curator can unassign the curator.
+						ensure!(maybe_sender.map_or(true, |sender| sender == *curator), BadOrigin);
+					},
+					BountyStatus::Active { ref curator, ref update_due } => {
+						// The bounty is active.
+						match maybe_sender {
+							// If the `RejectOrigin` is calling this function, slash the curator.
+							None => {
+								slash_curator(curator, &mut bounty.curator_deposit);
+								// Continue to change bounty status below...
+							},
+							Some(sender) => {
+								// If the sender is not the curator, and the curator is inactive,
+								// slash the curator.
+								if sender != *curator {
+									let block_number = system::Module::<T>::block_number();
+									if *update_due < block_number {
+										slash_curator(curator, &mut bounty.curator_deposit);
+										// Continue to change bounty status below...
+									} else {
+										// Curator has more time to give an update.
+										return Err(Error::<T, I>::Premature.into())
+									}
+								} else {
+									// Else this is the curator, willingly giving up their role.
+									// Give back their deposit.
+									let _ = T::Currency::unreserve(&curator, bounty.curator_deposit);
+									// Continue to change bounty status below...
+								}
+							},
+						}
+					},
+					BountyStatus::PendingPayout { ref curator, .. } => {
+						// The bounty is pending payout, so only council can unassign a curator.
+						// By doing so, they are claiming the curator is acting maliciously, so
+						// we slash the curator.
+						ensure!(maybe_sender.is_none(), BadOrigin);
+						slash_curator(curator, &mut bounty.curator_deposit);
+						// Continue to change bounty status below...
+					}
+				};
+
+				bounty.status = BountyStatus::Funded;
+				Ok(())
+			})?;
+		}
+
+		/// Accept the curator role for a bounty.
+		/// A deposit will be reserved from curator and refund upon successful payout.
+		///
+		/// May only be called from the curator.
+		///
+		/// # <weight>
+		/// - O(1).
+		/// - Limited storage reads.
+		/// - One DB change.
+		/// # </weight>
+		#[weight = T::WeightInfo::accept_curator()]
+		fn accept_curator(origin, #[compact] bounty_id: ProposalIndex) {
+			let signer = ensure_signed(origin)?;
+
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult {
+				let mut bounty = maybe_bounty.as_mut().ok_or(Error::<T, I>::InvalidIndex)?;
+
+				match bounty.status {
+					BountyStatus::CuratorProposed { ref curator } => {
+						ensure!(signer == *curator, Error::<T, I>::RequireCurator);
+
+						let deposit = T::BountyCuratorDeposit::get() * bounty.fee;
+						T::Currency::reserve(curator, deposit)?;
+						bounty.curator_deposit = deposit;
+
+						let update_due = system::Module::<T>::block_number() + T::BountyUpdatePeriod::get();
+						bounty.status = BountyStatus::Active { curator: curator.clone(), update_due };
+
+						Ok(())
+					},
+					_ => Err(Error::<T, I>::UnexpectedStatus.into()),
+				}
+			})?;
+		}
+
+		/// Award bounty to a beneficiary account. The beneficiary will be able to claim the funds after a delay.
+		///
+		/// The dispatch origin for this call must be the curator of this bounty.
+		///
+		/// - `bounty_id`: Bounty ID to award.
+		/// - `beneficiary`: The beneficiary account whom will receive the payout.
+		#[weight = T::WeightInfo::award_bounty()]
+		fn award_bounty(origin, #[compact] bounty_id: ProposalIndex, beneficiary: <T::Lookup as StaticLookup>::Source) {
+			let signer = ensure_signed(origin)?;
+			let beneficiary = T::Lookup::lookup(beneficiary)?;
+
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult {
+				let mut bounty = maybe_bounty.as_mut().ok_or(Error::<T, I>::InvalidIndex)?;
+				match &bounty.status {
+					BountyStatus::Active {
+						curator,
+						..
+					} => {
+						ensure!(signer == *curator, Error::<T, I>::RequireCurator);
+					},
+					_ => return Err(Error::<T, I>::UnexpectedStatus.into()),
+				}
+				bounty.status = BountyStatus::PendingPayout {
+					curator: signer,
+					beneficiary: beneficiary.clone(),
+					unlock_at: system::Module::<T>::block_number() + T::BountyDepositPayoutDelay::get(),
+				};
+
+				Ok(())
+			})?;
+
+			Self::deposit_event(Event::<T, I>::BountyAwarded(bounty_id, beneficiary));
+		}
+
+		/// Claim the payout from an awarded bounty after payout delay.
+		///
+		/// The dispatch origin for this call must be the beneficiary of this bounty.
+		///
+		/// - `bounty_id`: Bounty ID to claim.
+		#[weight = T::WeightInfo::claim_bounty()]
+		fn claim_bounty(origin, #[compact] bounty_id: BountyIndex) {
+			let _ = ensure_signed(origin)?; // anyone can trigger claim
+
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult {
+				let bounty = maybe_bounty.take().ok_or(Error::<T, I>::InvalidIndex)?;
+				if let BountyStatus::PendingPayout { curator, beneficiary, unlock_at } = bounty.status {
+					ensure!(system::Module::<T>::block_number() >= unlock_at, Error::<T, I>::Premature);
+					let bounty_account = Self::bounty_account_id(bounty_id);
+					let balance = T::Currency::free_balance(&bounty_account);
+					let fee = bounty.fee.min(balance); // just to be safe
+					let payout = balance.saturating_sub(fee);
+					let _ = T::Currency::unreserve(&curator, bounty.curator_deposit);
+					let _ = T::Currency::transfer(&bounty_account, &curator, fee, AllowDeath); // should not fail
+					let _ = T::Currency::transfer(&bounty_account, &beneficiary, payout, AllowDeath); // should not fail
+					*maybe_bounty = None;
+
+					BountyDescriptions::<I>::remove(bounty_id);
+
+					Self::deposit_event(Event::<T, I>::BountyClaimed(bounty_id, payout, beneficiary));
+					Ok(())
+				} else {
+					Err(Error::<T, I>::UnexpectedStatus.into())
+				}
+			})?;
+		}
+
+		/// Cancel a proposed or active bounty. All the funds will be sent to treasury and
+		/// the curator deposit will be unreserved if possible.
+		///
+		/// Only `T::RejectOrigin` is able to cancel a bounty.
+		///
+		/// - `bounty_id`: Bounty ID to cancel.
+		#[weight = T::WeightInfo::close_bounty_proposed().max(T::WeightInfo::close_bounty_active())]
+		fn close_bounty(origin, #[compact] bounty_id: BountyIndex) -> DispatchResultWithPostInfo {
+			T::RejectOrigin::ensure_origin(origin)?;
+
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResultWithPostInfo {
+				let bounty = maybe_bounty.as_ref().ok_or(Error::<T, I>::InvalidIndex)?;
+
+				match &bounty.status {
+					BountyStatus::Proposed => {
+						// The reject origin would like to cancel a proposed bounty.
+						BountyDescriptions::<I>::remove(bounty_id);
+						let value = bounty.bond;
+						let imbalance = T::Currency::slash_reserved(&bounty.proposer, value).0;
+						T::OnSlash::on_unbalanced(imbalance);
+						*maybe_bounty = None;
+
+						Self::deposit_event(Event::<T, I>::BountyRejected(bounty_id, value));
+						// Return early, nothing else to do.
+						return Ok(Some(T::WeightInfo::close_bounty_proposed()).into())
+					},
+					BountyStatus::Approved => {
+						// For weight reasons, we don't allow a council to cancel in this phase.
+						// We ask for them to wait until it is funded before they can cancel.
+						return Err(Error::<T, I>::UnexpectedStatus.into())
+					},
+					BountyStatus::Funded |
+					BountyStatus::CuratorProposed { .. } => {
+						// Nothing extra to do besides the removal of the bounty below.
+					},
+					BountyStatus::Active { curator, .. } => {
+						// Cancelled by council, refund deposit of the working curator.
+						let _ = T::Currency::unreserve(&curator, bounty.curator_deposit);
+						// Then execute removal of the bounty below.
+					},
+					BountyStatus::PendingPayout { .. } => {
+						// Bounty is already pending payout. If council wants to cancel
+						// this bounty, it should mean the curator was acting maliciously.
+						// So the council should first unassign the curator, slashing their
+						// deposit.
+						return Err(Error::<T, I>::PendingPayout.into())
+					}
+				}
+
+				let bounty_account = Self::bounty_account_id(bounty_id);
+
+				BountyDescriptions::<I>::remove(bounty_id);
+
+				let balance = T::Currency::free_balance(&bounty_account);
+				let _ = T::Currency::transfer(&bounty_account, &Self::account_id(), balance, AllowDeath); // should not fail
+				*maybe_bounty = None;
+
+				Self::deposit_event(Event::<T, I>::BountyCanceled(bounty_id));
+				Ok(Some(T::WeightInfo::close_bounty_active()).into())
+			})
+		}
+
+		/// Extend the expiry time of an active bounty.
+		///
+		/// The dispatch origin for this call must be the curator of this bounty.
+		///
+		/// - `bounty_id`: Bounty ID to extend.
+		/// - `remark`: additional information.
+		#[weight = T::WeightInfo::extend_bounty_expiry()]
+		fn extend_bounty_expiry(origin, #[compact] bounty_id: BountyIndex, _remark: Vec<u8>) {
+			let signer = ensure_signed(origin)?;
+
+			Bounties::<T, I>::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult {
+				let bounty = maybe_bounty.as_mut().ok_or(Error::<T, I>::InvalidIndex)?;
+
+				match bounty.status {
+					BountyStatus::Active { ref curator, ref mut update_due } => {
+						ensure!(*curator == signer, Error::<T, I>::RequireCurator);
+						*update_due = (system::Module::<T>::block_number() + T::BountyUpdatePeriod::get()).max(*update_due);
+					},
+					_ => return Err(Error::<T, I>::UnexpectedStatus.into()),
+				}
+
+				Ok(())
+			})?;
+
+			Self::deposit_event(Event::<T, I>::BountyExtended(bounty_id));
+		}
+
 		/// # <weight>
 		/// - Complexity: `O(A)` where `A` is the number of approvals
 		/// - Db reads and writes: `Approvals`, `pot account data`
@@ -642,10 +1171,7 @@ decl_module! {
 		fn on_initialize(n: T::BlockNumber) -> Weight {
 			// Check to see if we should spend some funds!
 			if (n % T::SpendPeriod::get()).is_zero() {
-				let approvals_len = Self::spend_funds();
-
-				270_000_000 * approvals_len
-					+ T::DbWeight::get().reads_writes(2 + approvals_len * 3, 2 + approvals_len * 3)
+				Self::spend_funds()
 			} else {
 				0
 			}
@@ -664,6 +1190,13 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 		T::ModuleId::get().into_account()
 	}
 
+	/// The account ID of a bounty account
+	pub fn bounty_account_id(id: BountyIndex) -> T::AccountId {
+		// only use two byte prefix to support 16 byte account id (used by test)
+		// "modl" ++ "py/trsry" ++ "bt" is 14 bytes, and two bytes remaining for bounty index
+		T::ModuleId::get().into_sub_account(("bt", id))
+	}
+
 	/// The needed bond for a proposal whose spend is `value`.
 	fn calculate_bond(value: BalanceOf<T, I>) -> BalanceOf<T, I> {
 		T::ProposalBondMinimum::get().max(T::ProposalBond::get() * value)
@@ -743,14 +1276,17 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 	}
 
 	/// Spend some money! returns number of approvals before spend.
-	fn spend_funds() -> u64 {
+	fn spend_funds() -> Weight {
+		let mut total_weight: Weight = Zero::zero();
+
 		let mut budget_remaining = Self::pot();
 		Self::deposit_event(RawEvent::Spending(budget_remaining));
+		let account_id = Self::account_id();
 
 		let mut missed_any = false;
 		let mut imbalance = <PositiveImbalanceOf<T, I>>::zero();
-		let prior_approvals_len = <Approvals<I>>::mutate(|v| {
-			let prior_approvals_len = v.len() as u64;
+		let proposals_len = Approvals::<I>::mutate(|v| {
+			let proposals_approvals_len = v.len() as u32;
 			v.retain(|&index| {
 				// Should always be true, but shouldn't panic if false or we're screwed.
 				if let Some(p) = Self::proposals(index) {
@@ -774,9 +1310,44 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 					false
 				}
 			});
-			prior_approvals_len
+			proposals_approvals_len
+		});
+
+		total_weight += T::WeightInfo::on_initialize_proposals(proposals_len);
+
+		let bounties_len = BountyApprovals::<I>::mutate(|v| {
+			let bounties_approval_len = v.len() as u32;
+			v.retain(|&index| {
+				Bounties::<T, I>::mutate(index, |bounty| {
+					// Should always be true, but shouldn't panic if false or we're screwed.
+					if let Some(bounty) = bounty {
+						if bounty.value <= budget_remaining {
+							budget_remaining -= bounty.value;
+
+							bounty.status = BountyStatus::Funded;
+
+							// return their deposit.
+							let _ = T::Currency::unreserve(&bounty.proposer, bounty.bond);
+
+							// fund the bounty account
+							imbalance.subsume(T::Currency::deposit_creating(&Self::bounty_account_id(index), bounty.value));
+
+							Self::deposit_event(RawEvent::BountyBecameActive(index));
+							false
+						} else {
+							missed_any = true;
+							true
+						}
+					} else {
+						false
+					}
+				})
+			});
+			bounties_approval_len
 		});
 
+		total_weight += T::WeightInfo::on_initialize_bounties(bounties_len);
+
 		if !missed_any {
 			// burn some proportion of the remaining budget if we run a surplus.
 			let burn = (T::Burn::get() * budget_remaining).min(budget_remaining);
@@ -793,7 +1364,7 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 		// Thus we can't spend more than account free balance minus ED;
 		// Thus account is kept alive; qed;
 		if let Err(problem) = T::Currency::settle(
-			&Self::account_id(),
+			&account_id,
 			imbalance,
 			WithdrawReason::Transfer.into(),
 			KeepAlive
@@ -805,7 +1376,7 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 
 		Self::deposit_event(RawEvent::Rollover(budget_remaining));
 
-		prior_approvals_len
+		total_weight
 	}
 
 	/// Return the amount of money in the pot.
@@ -816,6 +1387,36 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 			.saturating_sub(T::Currency::minimum_balance())
 	}
 
+	fn create_bounty(
+		proposer: T::AccountId,
+		description: Vec<u8>,
+		value: BalanceOf<T, I>,
+	) -> DispatchResult {
+		ensure!(description.len() <= T::MaximumReasonLength::get() as usize, Error::<T, I>::ReasonTooBig);
+		ensure!(value >= T::BountyValueMinimum::get(), Error::<T, I>::InvalidValue);
+
+		let index = Self::bounty_count();
+
+		// reserve deposit for new bounty
+		let bond = T::BountyDepositBase::get()
+			+ T::DataDepositPerByte::get() * (description.len() as u32).into();
+		T::Currency::reserve(&proposer, bond)
+			.map_err(|_| Error::<T, I>::InsufficientProposersBalance)?;
+
+		BountyCount::<I>::put(index + 1);
+
+		let bounty = Bounty {
+			proposer, value, fee: 0.into(), curator_deposit: 0.into(), bond, status: BountyStatus::Proposed,
+		};
+
+		Bounties::<T, I>::insert(index, &bounty);
+		BountyDescriptions::<I>::insert(index, description);
+
+		Self::deposit_event(RawEvent::BountyProposed(index));
+
+		Ok(())
+	}
+
 	pub fn migrate_retract_tip_for_tip_new() {
 		/// An open tipping "motion". Retains all details of a tip including information on the finder
 		/// and the members who have voted.
diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs
index a4e1e3d8d77b2..a092a14f05d84 100644
--- a/frame/treasury/src/tests.rs
+++ b/frame/treasury/src/tests.rs
@@ -67,7 +67,7 @@ impl frame_system::Trait for Test {
 	type Call = ();
 	type Hash = H256;
 	type Hashing = BlakeTwo256;
-	type AccountId = u64;
+	type AccountId = u128; // u64 is not enough to hold bytes used to generate bounty account
 	type Lookup = IdentityLookup<Self::AccountId>;
 	type Header = Header;
 	type Event = Event;
@@ -99,17 +99,17 @@ impl pallet_balances::Trait for Test {
 	type WeightInfo = ();
 }
 thread_local! {
-	static TEN_TO_FOURTEEN: RefCell<Vec<u64>> = RefCell::new(vec![10,11,12,13,14]);
+	static TEN_TO_FOURTEEN: RefCell<Vec<u128>> = RefCell::new(vec![10,11,12,13,14]);
 }
 pub struct TenToFourteen;
-impl Contains<u64> for TenToFourteen {
-	fn sorted_members() -> Vec<u64> {
+impl Contains<u128> for TenToFourteen {
+	fn sorted_members() -> Vec<u128> {
 		TEN_TO_FOURTEEN.with(|v| {
 			v.borrow().clone()
 		})
 	}
 	#[cfg(feature = "runtime-benchmarks")]
-	fn add(new: &u64) {
+	fn add(new: &u128) {
 		TEN_TO_FOURTEEN.with(|v| {
 			let mut members = v.borrow_mut();
 			members.push(*new);
@@ -131,25 +131,37 @@ parameter_types! {
 	pub const TipCountdown: u64 = 1;
 	pub const TipFindersFee: Percent = Percent::from_percent(20);
 	pub const TipReportDepositBase: u64 = 1;
-	pub const TipReportDepositPerByte: u64 = 1;
+	pub const DataDepositPerByte: u64 = 1;
+	pub const BountyDepositBase: u64 = 80;
+	pub const BountyDepositPayoutDelay: u64 = 3;
 	pub const TreasuryModuleId: ModuleId = ModuleId(*b"py/trsry");
+	pub const BountyUpdatePeriod: u32 = 20;
+	pub const MaximumReasonLength: u32 = 16384;
+	pub const BountyCuratorDeposit: Permill = Permill::from_percent(50);
+	pub const BountyValueMinimum: u64 = 1;
 }
 impl Trait for Test {
 	type ModuleId = TreasuryModuleId;
 	type Currency = pallet_balances::Module<Test>;
-	type ApproveOrigin = frame_system::EnsureRoot<u64>;
-	type RejectOrigin = frame_system::EnsureRoot<u64>;
+	type ApproveOrigin = frame_system::EnsureRoot<u128>;
+	type RejectOrigin = frame_system::EnsureRoot<u128>;
 	type Tippers = TenToFourteen;
 	type TipCountdown = TipCountdown;
 	type TipFindersFee = TipFindersFee;
 	type TipReportDepositBase = TipReportDepositBase;
-	type TipReportDepositPerByte = TipReportDepositPerByte;
+	type DataDepositPerByte = DataDepositPerByte;
 	type Event = Event;
-	type ProposalRejection = ();
+	type OnSlash = ();
 	type ProposalBond = ProposalBond;
 	type ProposalBondMinimum = ProposalBondMinimum;
 	type SpendPeriod = SpendPeriod;
 	type Burn = Burn;
+	type BountyDepositBase = BountyDepositBase;
+	type BountyDepositPayoutDelay = BountyDepositPayoutDelay;
+	type BountyUpdatePeriod = BountyUpdatePeriod;
+	type BountyCuratorDeposit = BountyCuratorDeposit;
+	type BountyValueMinimum = BountyValueMinimum;
+	type MaximumReasonLength = MaximumReasonLength;
 	type BurnDestination = ();  // Just gets burned.
 	type WeightInfo = ();
 }
@@ -167,6 +179,15 @@ pub fn new_test_ext() -> sp_io::TestExternalities {
 	t.into()
 }
 
+fn last_event() -> RawEvent<u64, u128, H256, DefaultInstance> {
+	System::events().into_iter().map(|r| r.event)
+		.filter_map(|e| {
+			if let Event::treasury(inner) = e { Some(inner) } else { None }
+		})
+		.last()
+		.unwrap()
+}
+
 #[test]
 fn genesis_config_works() {
 	new_test_ext().execute_with(|| {
@@ -176,7 +197,7 @@ fn genesis_config_works() {
 }
 
 fn tip_hash() -> H256 {
-	BlakeTwo256::hash_of(&(BlakeTwo256::hash(b"awesome.dot"), 3u64))
+	BlakeTwo256::hash_of(&(BlakeTwo256::hash(b"awesome.dot"), 3u128))
 }
 
 #[test]
@@ -225,7 +246,7 @@ fn report_awesome_from_beneficiary_and_tip_works() {
 		assert_ok!(Treasury::report_awesome(Origin::signed(0), b"awesome.dot".to_vec(), 0));
 		assert_eq!(Balances::reserved_balance(0), 12);
 		assert_eq!(Balances::free_balance(0), 88);
-		let h = BlakeTwo256::hash_of(&(BlakeTwo256::hash(b"awesome.dot"), 0u64));
+		let h = BlakeTwo256::hash_of(&(BlakeTwo256::hash(b"awesome.dot"), 0u128));
 		assert_ok!(Treasury::tip(Origin::signed(10), h.clone(), 10));
 		assert_ok!(Treasury::tip(Origin::signed(11), h.clone(), 10));
 		assert_ok!(Treasury::tip(Origin::signed(12), h.clone(), 10));
@@ -248,15 +269,7 @@ fn close_tip_works() {
 
 		let h = tip_hash();
 
-		assert_eq!(
-			System::events().into_iter().map(|r| r.event)
-				.filter_map(|e| {
-					if let Event::treasury(inner) = e { Some(inner) } else { None }
-				})
-				.last()
-				.unwrap(),
-			RawEvent::NewTip(h),
-		);
+		assert_eq!(last_event(), RawEvent::NewTip(h));
 
 		assert_ok!(Treasury::tip(Origin::signed(11), h.clone(), 10));
 
@@ -264,15 +277,7 @@ fn close_tip_works() {
 
 		assert_ok!(Treasury::tip(Origin::signed(12), h.clone(), 10));
 
-		assert_eq!(
-			System::events().into_iter().map(|r| r.event)
-				.filter_map(|e| {
-					if let Event::treasury(inner) = e { Some(inner) } else { None }
-				})
-				.last()
-				.unwrap(),
-			RawEvent::TipClosing(h),
-		);
+		assert_eq!(last_event(), RawEvent::TipClosing(h));
 
 		assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::<Test, _>::Premature);
 
@@ -281,15 +286,7 @@ fn close_tip_works() {
 		assert_ok!(Treasury::close_tip(Origin::signed(0), h.into()));
 		assert_eq!(Balances::free_balance(3), 10);
 
-		assert_eq!(
-			System::events().into_iter().map(|r| r.event)
-				.filter_map(|e| {
-					if let Event::treasury(inner) = e { Some(inner) } else { None }
-				})
-				.last()
-				.unwrap(),
-			RawEvent::TipClosed(h, 3, 10),
-		);
+		assert_eq!(last_event(), RawEvent::TipClosed(h, 3, 10));
 
 		assert_noop!(Treasury::close_tip(Origin::signed(100), h.into()), Error::<Test, _>::UnknownTip);
 	});
@@ -441,30 +438,21 @@ fn reject_already_rejected_spend_proposal_fails() {
 
 		assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3));
 		assert_ok!(Treasury::reject_proposal(Origin::root(), 0));
-		assert_noop!(
-			Treasury::reject_proposal(Origin::root(), 0),
-			Error::<Test, _>::InvalidProposalIndex,
-		);
+		assert_noop!(Treasury::reject_proposal(Origin::root(), 0), Error::<Test, _>::InvalidIndex);
 	});
 }
 
 #[test]
 fn reject_non_existent_spend_proposal_fails() {
 	new_test_ext().execute_with(|| {
-		assert_noop!(
-			Treasury::reject_proposal(Origin::root(), 0),
-			Error::<Test, _>::InvalidProposalIndex,
-		);
+		assert_noop!(Treasury::reject_proposal(Origin::root(), 0), Error::<Test, _>::InvalidIndex);
 	});
 }
 
 #[test]
 fn accept_non_existent_spend_proposal_fails() {
 	new_test_ext().execute_with(|| {
-		assert_noop!(
-			Treasury::approve_proposal(Origin::root(), 0),
-			Error::<Test, _>::InvalidProposalIndex,
-		);
+		assert_noop!(Treasury::approve_proposal(Origin::root(), 0), Error::<Test, _>::InvalidIndex);
 	});
 }
 
@@ -475,10 +463,7 @@ fn accept_already_rejected_spend_proposal_fails() {
 
 		assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3));
 		assert_ok!(Treasury::reject_proposal(Origin::root(), 0));
-		assert_noop!(
-			Treasury::approve_proposal(Origin::root(), 0),
-			Error::<Test, _>::InvalidProposalIndex,
-		);
+		assert_noop!(Treasury::approve_proposal(Origin::root(), 0), Error::<Test, _>::InvalidIndex);
 	});
 }
 
@@ -574,6 +559,502 @@ fn inexistent_account_works() {
 	});
 }
 
+#[test]
+fn propose_bounty_works() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_eq!(Treasury::pot(), 100);
+
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 10, b"1234567890".to_vec()));
+
+		assert_eq!(last_event(), RawEvent::BountyProposed(0));
+
+		let deposit: u64 = 85 + 5;
+		assert_eq!(Balances::reserved_balance(0), deposit);
+		assert_eq!(Balances::free_balance(0), 100 - deposit);
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 0,
+			curator_deposit: 0,
+			value: 10,
+			bond: deposit,
+			status: BountyStatus::Proposed,
+		});
+
+		assert_eq!(Treasury::bounty_descriptions(0).unwrap(), b"1234567890".to_vec());
+
+		assert_eq!(Treasury::bounty_count(), 1);
+	});
+}
+
+#[test]
+fn propose_bounty_validation_works() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_eq!(Treasury::pot(), 100);
+
+		assert_noop!(
+			Treasury::propose_bounty(Origin::signed(1), 0, [0; 17_000].to_vec()),
+			Error::<Test, _>::ReasonTooBig
+		);
+
+		assert_noop!(
+			Treasury::propose_bounty(Origin::signed(1), 10, b"12345678901234567890".to_vec()),
+			Error::<Test, _>::InsufficientProposersBalance
+		);
+
+		assert_noop!(
+			Treasury::propose_bounty(Origin::signed(1), 0, b"12345678901234567890".to_vec()),
+			Error::<Test, _>::InvalidValue
+		);
+	});
+}
+
+#[test]
+fn close_bounty_works() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_noop!(Treasury::close_bounty(Origin::root(), 0), Error::<Test, _>::InvalidIndex);
+
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 10, b"12345".to_vec()));
+
+		assert_ok!(Treasury::close_bounty(Origin::root(), 0));
+
+		let deposit: u64 = 80 + 5;
+
+		assert_eq!(last_event(), RawEvent::BountyRejected(0, deposit));
+
+		assert_eq!(Balances::reserved_balance(0), 0);
+		assert_eq!(Balances::free_balance(0), 100 - deposit);
+
+		assert_eq!(Treasury::bounties(0), None);
+		assert!(!Bounties::<Test>::contains_key(0));
+		assert_eq!(Treasury::bounty_descriptions(0), None);
+	});
+}
+
+#[test]
+fn approve_bounty_works() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_noop!(Treasury::approve_bounty(Origin::root(), 0), Error::<Test, _>::InvalidIndex);
+
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		let deposit: u64 = 80 + 5;
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 0,
+			value: 50,
+			curator_deposit: 0,
+			bond: deposit,
+			status: BountyStatus::Approved,
+		});
+		assert_eq!(Treasury::bounty_approvals(), vec![0]);
+
+		assert_noop!(Treasury::close_bounty(Origin::root(), 0), Error::<Test, _>::UnexpectedStatus);
+
+		// deposit not returned yet
+		assert_eq!(Balances::reserved_balance(0), deposit);
+		assert_eq!(Balances::free_balance(0), 100 - deposit);
+
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		// return deposit
+		assert_eq!(Balances::reserved_balance(0), 0);
+		assert_eq!(Balances::free_balance(0), 100);
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 0,
+			curator_deposit: 0,
+			value: 50,
+			bond: deposit,
+			status: BountyStatus::Funded,
+		});
+		assert_eq!(Treasury::pot(), 100 - 50 - 25); // burn 25
+		assert_eq!(Balances::free_balance(Treasury::bounty_account_id(0)), 50);
+	});
+}
+
+#[test]
+fn assign_curator_works() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+
+		assert_noop!(Treasury::propose_curator(Origin::root(), 0, 4, 4), Error::<Test, _>::InvalidIndex);
+
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_noop!(Treasury::propose_curator(Origin::root(), 0, 4, 50), Error::<Test, _>::InvalidFee);
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 4, 4));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 4,
+			curator_deposit: 0,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::CuratorProposed {
+				curator: 4,
+			},
+		});
+
+		assert_noop!(Treasury::accept_curator(Origin::signed(1), 0), Error::<Test, _>::RequireCurator);
+		assert_noop!(Treasury::accept_curator(Origin::signed(4), 0), pallet_balances::Error::<Test, _>::InsufficientBalance);
+
+		Balances::make_free_balance_be(&4, 10);
+
+		assert_ok!(Treasury::accept_curator(Origin::signed(4), 0));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 4,
+			curator_deposit: 2,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::Active {
+				curator: 4,
+				update_due: 22,
+			},
+		});
+
+		assert_eq!(Balances::free_balance(&4), 8);
+		assert_eq!(Balances::reserved_balance(&4), 2);
+	});
+}
+
+#[test]
+fn unassign_curator_works() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 4, 4));
+
+		assert_noop!(Treasury::unassign_curator(Origin::signed(1), 0), BadOrigin);
+
+		assert_ok!(Treasury::unassign_curator(Origin::signed(4), 0));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 4,
+			curator_deposit: 0,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::Funded,
+		});
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 4, 4));
+
+		Balances::make_free_balance_be(&4, 10);
+
+		assert_ok!(Treasury::accept_curator(Origin::signed(4), 0));
+
+		assert_ok!(Treasury::unassign_curator(Origin::root(), 0));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 4,
+			curator_deposit: 0,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::Funded,
+		});
+
+		assert_eq!(Balances::free_balance(&4), 8);
+		assert_eq!(Balances::reserved_balance(&4), 0); // slashed 2
+	});
+}
+
+#[test]
+fn award_and_claim_bounty_works() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		Balances::make_free_balance_be(&4, 10);
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 4, 4));
+		assert_ok!(Treasury::accept_curator(Origin::signed(4), 0));
+
+		assert_eq!(Balances::free_balance(4), 8); // inital 10 - 2 deposit
+
+		assert_noop!(Treasury::award_bounty(Origin::signed(1), 0, 3), Error::<Test, _>::RequireCurator);
+
+		assert_ok!(Treasury::award_bounty(Origin::signed(4), 0, 3));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 4,
+			curator_deposit: 2,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::PendingPayout {
+				curator: 4,
+				beneficiary: 3,
+				unlock_at: 5
+			},
+		});
+
+		assert_noop!(Treasury::claim_bounty(Origin::signed(1), 0), Error::<Test, _>::Premature);
+
+		System::set_block_number(5);
+		<Treasury as OnInitialize<u64>>::on_initialize(5);
+
+		assert_ok!(Balances::transfer(Origin::signed(0), Treasury::bounty_account_id(0), 10));
+
+		assert_ok!(Treasury::claim_bounty(Origin::signed(1), 0));
+
+		assert_eq!(last_event(), RawEvent::BountyClaimed(0, 56, 3));
+
+		assert_eq!(Balances::free_balance(4), 14); // initial 10 + fee 4
+		assert_eq!(Balances::free_balance(3), 56);
+		assert_eq!(Balances::free_balance(Treasury::bounty_account_id(0)), 0);
+
+		assert_eq!(Treasury::bounties(0), None);
+		assert_eq!(Treasury::bounty_descriptions(0), None);
+	});
+}
+
+#[test]
+fn claim_handles_high_fee() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		Balances::make_free_balance_be(&4, 30);
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 4, 49));
+		assert_ok!(Treasury::accept_curator(Origin::signed(4), 0));
+
+		assert_ok!(Treasury::award_bounty(Origin::signed(4), 0, 3));
+
+		System::set_block_number(5);
+		<Treasury as OnInitialize<u64>>::on_initialize(5);
+
+		// make fee > balance
+		let _ = Balances::slash(&Treasury::bounty_account_id(0), 10);
+
+		assert_ok!(Treasury::claim_bounty(Origin::signed(1), 0));
+
+		assert_eq!(last_event(), RawEvent::BountyClaimed(0, 0, 3));
+
+		assert_eq!(Balances::free_balance(4), 70); // 30 + 50 - 10
+		assert_eq!(Balances::free_balance(3), 0);
+		assert_eq!(Balances::free_balance(Treasury::bounty_account_id(0)), 0);
+
+		assert_eq!(Treasury::bounties(0), None);
+		assert_eq!(Treasury::bounty_descriptions(0), None);
+	});
+}
+
+#[test]
+fn cancel_and_refund() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_ok!(Balances::transfer(Origin::signed(0), Treasury::bounty_account_id(0), 10));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 0,
+			curator_deposit: 0,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::Funded,
+		});
+
+		assert_eq!(Balances::free_balance(Treasury::bounty_account_id(0)), 60);
+
+		assert_noop!(Treasury::close_bounty(Origin::signed(0), 0), BadOrigin);
+
+		assert_ok!(Treasury::close_bounty(Origin::root(), 0));
+
+		assert_eq!(Treasury::pot(), 85); // - 25 + 10
+	});
+}
+
+#[test]
+fn award_and_cancel() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 0, 10));
+		assert_ok!(Treasury::accept_curator(Origin::signed(0), 0));
+
+		assert_eq!(Balances::free_balance(0), 95);
+		assert_eq!(Balances::reserved_balance(0), 5);
+
+		assert_ok!(Treasury::award_bounty(Origin::signed(0), 0, 3));
+
+		// Cannot close bounty directly when payout is happening...
+		assert_noop!(Treasury::close_bounty(Origin::root(), 0), Error::<Test, _>::PendingPayout);
+
+		// Instead unassign the curator to slash them and then close.
+		assert_ok!(Treasury::unassign_curator(Origin::root(), 0));
+		assert_ok!(Treasury::close_bounty(Origin::root(), 0));
+
+		assert_eq!(last_event(), RawEvent::BountyCanceled(0));
+
+		assert_eq!(Balances::free_balance(Treasury::bounty_account_id(0)), 0);
+		// Slashed.
+		assert_eq!(Balances::free_balance(0), 95);
+		assert_eq!(Balances::reserved_balance(0), 0);
+
+		assert_eq!(Treasury::bounties(0), None);
+		assert_eq!(Treasury::bounty_descriptions(0), None);
+	});
+}
+
+#[test]
+fn expire_and_unassign() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 1, 10));
+		assert_ok!(Treasury::accept_curator(Origin::signed(1), 0));
+
+		assert_eq!(Balances::free_balance(1), 93);
+		assert_eq!(Balances::reserved_balance(1), 5);
+
+		System::set_block_number(22);
+		<Treasury as OnInitialize<u64>>::on_initialize(22);
+
+		assert_noop!(Treasury::unassign_curator(Origin::signed(0), 0), Error::<Test, _>::Premature);
+
+		System::set_block_number(23);
+		<Treasury as OnInitialize<u64>>::on_initialize(23);
+
+		assert_ok!(Treasury::unassign_curator(Origin::signed(0), 0));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 10,
+			curator_deposit: 0,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::Funded,
+		});
+
+		assert_eq!(Balances::free_balance(1), 93);
+		assert_eq!(Balances::reserved_balance(1), 0); // slashed
+
+	});
+}
+
+#[test]
+fn extend_expiry() {
+	new_test_ext().execute_with(|| {
+		System::set_block_number(1);
+		Balances::make_free_balance_be(&Treasury::account_id(), 101);
+		Balances::make_free_balance_be(&4, 10);
+		assert_ok!(Treasury::propose_bounty(Origin::signed(0), 50, b"12345".to_vec()));
+
+		assert_ok!(Treasury::approve_bounty(Origin::root(), 0));
+
+		assert_noop!(Treasury::extend_bounty_expiry(Origin::signed(1), 0, Vec::new()), Error::<Test, _>::UnexpectedStatus);
+
+		System::set_block_number(2);
+		<Treasury as OnInitialize<u64>>::on_initialize(2);
+
+		assert_ok!(Treasury::propose_curator(Origin::root(), 0, 4, 10));
+		assert_ok!(Treasury::accept_curator(Origin::signed(4), 0));
+
+		assert_eq!(Balances::free_balance(4), 5);
+		assert_eq!(Balances::reserved_balance(4), 5);
+
+		System::set_block_number(10);
+		<Treasury as OnInitialize<u64>>::on_initialize(10);
+
+		assert_noop!(Treasury::extend_bounty_expiry(Origin::signed(0), 0, Vec::new()), Error::<Test, _>::RequireCurator);
+		assert_ok!(Treasury::extend_bounty_expiry(Origin::signed(4), 0, Vec::new()));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 10,
+			curator_deposit: 5,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::Active { curator: 4, update_due: 30 },
+		});
+
+		assert_ok!(Treasury::extend_bounty_expiry(Origin::signed(4), 0, Vec::new()));
+
+		assert_eq!(Treasury::bounties(0).unwrap(), Bounty {
+			proposer: 0,
+			fee: 10,
+			curator_deposit: 5,
+			value: 50,
+			bond: 85,
+			status: BountyStatus::Active { curator: 4, update_due: 30 }, // still the same
+		});
+
+		System::set_block_number(25);
+		<Treasury as OnInitialize<u64>>::on_initialize(25);
+
+		assert_noop!(Treasury::unassign_curator(Origin::signed(0), 0), Error::<Test, _>::Premature);
+		assert_ok!(Treasury::unassign_curator(Origin::signed(4), 0));
+
+		assert_eq!(Balances::free_balance(4), 10); // not slashed
+		assert_eq!(Balances::reserved_balance(4), 0);
+	});
+}
+
 #[test]
 fn test_last_reward_migration() {
 	use sp_storage::Storage;
@@ -604,7 +1085,7 @@ fn test_last_reward_migration() {
 	let reason1 = BlakeTwo256::hash(b"reason1");
 	let hash1 = BlakeTwo256::hash_of(&(reason1, 10u64));
 
-	let old_tip_finder = OldOpenTip::<u64, u64, u64, H256> {
+	let old_tip_finder = OldOpenTip::<u128, u64, u64, H256> {
 		reason: reason1,
 		who: 10,
 		finder: Some((20, 30)),
@@ -615,7 +1096,7 @@ fn test_last_reward_migration() {
 	let reason2 = BlakeTwo256::hash(b"reason2");
 	let hash2 = BlakeTwo256::hash_of(&(reason2, 20u64));
 
-	let old_tip_no_finder = OldOpenTip::<u64, u64, u64, H256> {
+	let old_tip_no_finder = OldOpenTip::<u128, u64, u64, H256> {
 		reason: reason2,
 		who: 20,
 		finder: None,

From 6b1d600adba22fff64c624eeec3b1ad339f37958 Mon Sep 17 00:00:00 2001
From: Jianping Deng <djptux@gmail.com>
Date: Fri, 18 Sep 2020 19:23:41 +0800
Subject: [PATCH 108/122] Update SS58 configuration for Bifrost (#7142)

* Add 6 as address type of ss58 for Bifrost Network

* Update SS58 configuration for Bifrost
---
 ss58-registry.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ss58-registry.json b/ss58-registry.json
index db3ab18d9854f..31177f6b61911 100644
--- a/ss58-registry.json
+++ b/ss58-registry.json
@@ -68,10 +68,10 @@
 			"prefix": 6,
 			"network": "bitfrost",
 			"displayName": "Bitfrost",
-			"symbols": null,
-			"decimals": null,
+			"symbols": ["BNC"],
+			"decimals": [12],
 			"standardAccount": "*25519",
-			"website": null
+			"website": "https://bifrost.finance/"
 		},
 		{
 			"prefix": 7,

From 7cc397fcd7399f73b552fbb0f82e88591ee404cd Mon Sep 17 00:00:00 2001
From: Anton Gavrilov <AntonE.Gavrilov@gmail.com>
Date: Fri, 18 Sep 2020 15:15:19 +0200
Subject: [PATCH 109/122] Prometheus metrics for RPC calls (#7088)

* WS and HTTP middlewares added

* Prometheus endpoint added

* Counters renamed

* Proper style for inc

* Metrics initialization re-written

* Rework to handler middleware

* Introduce transport prefix for metrics

* String shortened

* Commented code removed, new line inserted

* One more string shortened

* Wasm build fixed

* Wasm build fixed once again

* Rework to shared metrics

* Added collectors label

* Tilde removed from cargo

* Switch to owned metrics in parameters
---
 Cargo.lock                           |  2 +
 client/rpc-servers/Cargo.toml        |  4 +-
 client/rpc-servers/src/lib.rs        | 13 +++--
 client/rpc-servers/src/middleware.rs | 87 ++++++++++++++++++++++++++++
 client/service/src/builder.rs        | 42 +++++++++-----
 client/service/src/lib.rs            | 38 +++++++++---
 6 files changed, 157 insertions(+), 29 deletions(-)
 create mode 100644 client/rpc-servers/src/middleware.rs

diff --git a/Cargo.lock b/Cargo.lock
index df454edaf22f1..f635807f6fbc6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7111,6 +7111,7 @@ dependencies = [
 name = "sc-rpc-server"
 version = "2.0.0-rc6"
 dependencies = [
+ "futures 0.1.29",
  "jsonrpc-core",
  "jsonrpc-http-server",
  "jsonrpc-ipc-server",
@@ -7120,6 +7121,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sp-runtime",
+ "substrate-prometheus-endpoint",
 ]
 
 [[package]]
diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml
index 3af5cdd039d8b..0d3589a00a65e 100644
--- a/client/rpc-servers/Cargo.toml
+++ b/client/rpc-servers/Cargo.toml
@@ -12,14 +12,16 @@ description = "Substrate RPC servers."
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+futures = "0.1.6"
 jsonrpc-core = "14.2.0"
 pubsub = { package = "jsonrpc-pubsub", version = "14.2.0" }
 log = "0.4.8"
+prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc6"}
 serde = "1.0.101"
 serde_json = "1.0.41"
 sp-runtime = { version = "2.0.0-rc6", path = "../../primitives/runtime" }
 
 [target.'cfg(not(target_os = "unknown"))'.dependencies]
 http = { package = "jsonrpc-http-server", version = "14.2.0" }
-ws = { package = "jsonrpc-ws-server", version = "14.2.0" }
 ipc = { version = "14.2.0", package = "jsonrpc-ipc-server" }
+ws = { package = "jsonrpc-ws-server", version = "14.2.0" }
diff --git a/client/rpc-servers/src/lib.rs b/client/rpc-servers/src/lib.rs
index 1e476262acea8..1f99e8bb0d242 100644
--- a/client/rpc-servers/src/lib.rs
+++ b/client/rpc-servers/src/lib.rs
@@ -20,8 +20,10 @@
 
 #![warn(missing_docs)]
 
+mod middleware;
+
 use std::io;
-use jsonrpc_core::IoHandlerExtension;
+use jsonrpc_core::{IoHandlerExtension, MetaIoHandler};
 use log::error;
 use pubsub::PubSubMetadata;
 
@@ -32,15 +34,18 @@ const MAX_PAYLOAD: usize = 15 * 1024 * 1024;
 const WS_MAX_CONNECTIONS: usize = 100;
 
 /// The RPC IoHandler containing all requested APIs.
-pub type RpcHandler<T> = pubsub::PubSubHandler<T>;
+pub type RpcHandler<T> = pubsub::PubSubHandler<T, RpcMiddleware>;
 
 pub use self::inner::*;
+pub use middleware::{RpcMiddleware, RpcMetrics};
 
 /// Construct rpc `IoHandler`
 pub fn rpc_handler<M: PubSubMetadata>(
-	extension: impl IoHandlerExtension<M>
+	extension: impl IoHandlerExtension<M>,
+	rpc_middleware: RpcMiddleware,
 ) -> RpcHandler<M> {
-	let mut io = pubsub::PubSubHandler::default();
+	let io_handler = MetaIoHandler::with_middleware(rpc_middleware);
+	let mut io = pubsub::PubSubHandler::new(io_handler);
 	extension.augment(&mut io);
 
 	// add an endpoint to list all available methods.
diff --git a/client/rpc-servers/src/middleware.rs b/client/rpc-servers/src/middleware.rs
new file mode 100644
index 0000000000000..74139714c8cb7
--- /dev/null
+++ b/client/rpc-servers/src/middleware.rs
@@ -0,0 +1,87 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Middleware for RPC requests.
+
+use jsonrpc_core::{
+	Middleware as RequestMiddleware, Metadata,
+	Request, Response, FutureResponse, FutureOutput
+};
+use prometheus_endpoint::{
+	Registry, CounterVec, PrometheusError,
+	Opts, register, U64
+};
+
+use futures::{future::Either, Future};
+
+/// Metrics for RPC middleware
+#[derive(Debug, Clone)]
+pub struct RpcMetrics {
+	rpc_calls: CounterVec<U64>,
+}
+
+impl RpcMetrics {
+	/// Create an instance of metrics
+	pub fn new(metrics_registry: Option<&Registry>) -> Result<Self, PrometheusError> {
+		metrics_registry.and_then(|r| {
+			Some(RpcMetrics {
+				rpc_calls: register(CounterVec::new(
+					Opts::new(
+						"rpc_calls_total",
+						"Number of rpc calls received",
+					),
+					&["protocol"]
+				).ok()?, r).ok()?,
+			})
+		}).ok_or(PrometheusError::Msg("Cannot register metric".to_string()))
+	}
+}
+
+/// Middleware for RPC calls
+pub struct RpcMiddleware {
+	metrics: Option<RpcMetrics>,
+	transport_label: String,
+}
+
+impl RpcMiddleware {
+	/// Create an instance of middleware with provided metrics
+	/// transport_label is used as a label for Prometheus collector
+	pub fn new(metrics: Option<RpcMetrics>, transport_label: &str) -> Self {
+		RpcMiddleware {
+			metrics,
+			transport_label: String::from(transport_label),
+		}
+	}
+}
+
+impl<M: Metadata> RequestMiddleware<M> for RpcMiddleware {
+	type Future = FutureResponse;
+	type CallFuture = FutureOutput;
+
+	fn on_request<F, X>(&self, request: Request, meta: M, next: F) -> Either<FutureResponse, X>
+	where
+		F: Fn(Request, M) -> X + Send + Sync,
+		X: Future<Item = Option<Response>, Error = ()> + Send + 'static,
+	{
+		if let Some(ref metrics) = self.metrics {
+			metrics.rpc_calls.with_label_values(&[self.transport_label.as_str()]).inc();
+		}
+
+		Either::B(next(request, meta))
+	}
+}
diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs
index 25abfdffed845..410198af26da3 100644
--- a/client/service/src/builder.rs
+++ b/client/service/src/builder.rs
@@ -545,14 +545,22 @@ pub fn spawn_tasks<TBl, TBackend, TExPool, TRpc, TCl>(
 	);
 
 	// RPC
-	let gen_handler = |deny_unsafe: sc_rpc::DenyUnsafe| gen_handler(
-		deny_unsafe, &config, task_manager.spawn_handle(), client.clone(), transaction_pool.clone(),
-		keystore.clone(), on_demand.clone(), remote_blockchain.clone(), &*rpc_extensions_builder,
+	let gen_handler = |
+		deny_unsafe: sc_rpc::DenyUnsafe,
+		rpc_middleware: sc_rpc_server::RpcMiddleware
+	| gen_handler(
+		deny_unsafe, rpc_middleware, &config, task_manager.spawn_handle(),
+		client.clone(), transaction_pool.clone(), keystore.clone(),
+		on_demand.clone(), remote_blockchain.clone(), &*rpc_extensions_builder,
 		backend.offchain_storage(), system_rpc_tx.clone()
 	);
-	let rpc = start_rpc_servers(&config, gen_handler)?;
+	let rpc_metrics = sc_rpc_server::RpcMetrics::new(config.prometheus_registry()).ok();
+	let rpc = start_rpc_servers(&config, gen_handler, rpc_metrics.as_ref())?;
 	// This is used internally, so don't restrict access to unsafe RPC
-	let rpc_handlers = RpcHandlers(Arc::new(gen_handler(sc_rpc::DenyUnsafe::No).into()));
+	let rpc_handlers = RpcHandlers(Arc::new(gen_handler(
+		sc_rpc::DenyUnsafe::No,
+		sc_rpc_server::RpcMiddleware::new(rpc_metrics.as_ref().cloned(), "inbrowser")
+	).into()));
 
 	// Telemetry
 	let telemetry = config.telemetry_endpoints.clone().and_then(|endpoints| {
@@ -660,6 +668,7 @@ fn build_telemetry<TBl: BlockT>(
 
 fn gen_handler<TBl, TBackend, TExPool, TRpc, TCl>(
 	deny_unsafe: sc_rpc::DenyUnsafe,
+	rpc_middleware: sc_rpc_server::RpcMiddleware,
 	config: &Configuration,
 	spawn_handle: SpawnTaskHandle,
 	client: Arc<TCl>,
@@ -670,7 +679,7 @@ fn gen_handler<TBl, TBackend, TExPool, TRpc, TCl>(
 	rpc_extensions_builder: &(dyn RpcExtensionBuilder<Output = TRpc> + Send),
 	offchain_storage: Option<<TBackend as sc_client_api::backend::Backend<TBl>>::OffchainStorage>,
 	system_rpc_tx: TracingUnboundedSender<sc_rpc::system::Request<TBl>>
-) -> jsonrpc_pubsub::PubSubHandler<sc_rpc::Metadata>
+) -> sc_rpc_server::RpcHandler<sc_rpc::Metadata>
 	where
 		TBl: BlockT,
 		TCl: ProvideRuntimeApi<TBl> + BlockchainEvents<TBl> + HeaderBackend<TBl> +
@@ -735,15 +744,18 @@ fn gen_handler<TBl, TBackend, TExPool, TRpc, TCl>(
 		offchain::OffchainApi::to_delegate(offchain)
 	});
 
-	sc_rpc_server::rpc_handler((
-		state::StateApi::to_delegate(state),
-		state::ChildStateApi::to_delegate(child_state),
-		chain::ChainApi::to_delegate(chain),
-		maybe_offchain_rpc,
-		author::AuthorApi::to_delegate(author),
-		system::SystemApi::to_delegate(system),
-		rpc_extensions_builder.build(deny_unsafe, task_executor),
-	))
+	sc_rpc_server::rpc_handler(
+		(
+			state::StateApi::to_delegate(state),
+			state::ChildStateApi::to_delegate(child_state),
+			chain::ChainApi::to_delegate(chain),
+			maybe_offchain_rpc,
+			author::AuthorApi::to_delegate(author),
+			system::SystemApi::to_delegate(system),
+			rpc_extensions_builder.build(deny_unsafe, task_executor),
+		),
+		rpc_middleware
+	)
 }
 
 /// Parameters to pass into `build_network`.
diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs
index d5d503d22d171..39f1dff289a1a 100644
--- a/client/service/src/lib.rs
+++ b/client/service/src/lib.rs
@@ -97,7 +97,7 @@ impl<T> MallocSizeOfWasm for T {}
 
 /// RPC handlers that can perform RPC queries.
 #[derive(Clone)]
-pub struct RpcHandlers(Arc<jsonrpc_core::MetaIoHandler<sc_rpc::Metadata>>);
+pub struct RpcHandlers(Arc<jsonrpc_core::MetaIoHandler<sc_rpc::Metadata, sc_rpc_server::RpcMiddleware>>);
 
 impl RpcHandlers {
 	/// Starts an RPC query.
@@ -118,7 +118,8 @@ impl RpcHandlers {
 	}
 
 	/// Provides access to the underlying `MetaIoHandler`
-	pub fn io_handler(&self) -> Arc<jsonrpc_core::MetaIoHandler<sc_rpc::Metadata>> {
+	pub fn io_handler(&self)
+		-> Arc<jsonrpc_core::MetaIoHandler<sc_rpc::Metadata, sc_rpc_server::RpcMiddleware>> {
 		self.0.clone()
 	}
 }
@@ -382,9 +383,13 @@ mod waiting {
 
 /// Starts RPC servers that run in their own thread, and returns an opaque object that keeps them alive.
 #[cfg(not(target_os = "unknown"))]
-fn start_rpc_servers<H: FnMut(sc_rpc::DenyUnsafe) -> sc_rpc_server::RpcHandler<sc_rpc::Metadata>>(
+fn start_rpc_servers<
+	H: FnMut(sc_rpc::DenyUnsafe, sc_rpc_server::RpcMiddleware)
+	-> sc_rpc_server::RpcHandler<sc_rpc::Metadata>
+>(
 	config: &Configuration,
-	mut gen_handler: H
+	mut gen_handler: H,
+	rpc_metrics: Option<&sc_rpc_server::RpcMetrics>
 ) -> Result<Box<dyn std::any::Any + Send + Sync>, error::Error> {
 	fn maybe_start_server<T, F>(address: Option<SocketAddr>, mut start: F) -> Result<Option<T>, io::Error>
 		where F: FnMut(&SocketAddr) -> Result<T, io::Error>,
@@ -414,13 +419,21 @@ fn start_rpc_servers<H: FnMut(sc_rpc::DenyUnsafe) -> sc_rpc_server::RpcHandler<s
 	}
 
 	Ok(Box::new((
-		config.rpc_ipc.as_ref().map(|path| sc_rpc_server::start_ipc(&*path, gen_handler(sc_rpc::DenyUnsafe::No))),
+		config.rpc_ipc.as_ref().map(|path| sc_rpc_server::start_ipc(
+			&*path, gen_handler(
+				sc_rpc::DenyUnsafe::No,
+				sc_rpc_server::RpcMiddleware::new(rpc_metrics.cloned(), "ipc")
+			)
+		)),
 		maybe_start_server(
 			config.rpc_http,
 			|address| sc_rpc_server::start_http(
 				address,
 				config.rpc_cors.as_ref(),
-				gen_handler(deny_unsafe(&address, &config.rpc_methods)),
+				gen_handler(
+					deny_unsafe(&address, &config.rpc_methods),
+					sc_rpc_server::RpcMiddleware::new(rpc_metrics.cloned(), "http")
+				),
 			),
 		)?.map(|s| waiting::HttpServer(Some(s))),
 		maybe_start_server(
@@ -429,7 +442,10 @@ fn start_rpc_servers<H: FnMut(sc_rpc::DenyUnsafe) -> sc_rpc_server::RpcHandler<s
 				address,
 				config.rpc_ws_max_connections,
 				config.rpc_cors.as_ref(),
-				gen_handler(deny_unsafe(&address, &config.rpc_methods)),
+				gen_handler(
+					deny_unsafe(&address, &config.rpc_methods),
+					sc_rpc_server::RpcMiddleware::new(rpc_metrics.cloned(), "ws")
+				),
 			),
 		)?.map(|s| waiting::WsServer(Some(s))),
 	)))
@@ -437,9 +453,13 @@ fn start_rpc_servers<H: FnMut(sc_rpc::DenyUnsafe) -> sc_rpc_server::RpcHandler<s
 
 /// Starts RPC servers that run in their own thread, and returns an opaque object that keeps them alive.
 #[cfg(target_os = "unknown")]
-fn start_rpc_servers<H: FnMut(sc_rpc::DenyUnsafe) -> sc_rpc_server::RpcHandler<sc_rpc::Metadata>>(
+fn start_rpc_servers<
+	H: FnMut(sc_rpc::DenyUnsafe, sc_rpc_server::RpcMiddleware)
+	-> sc_rpc_server::RpcHandler<sc_rpc::Metadata>
+>(
 	_: &Configuration,
-	_: H
+	_: H,
+	_: Option<&sc_rpc_server::RpcMetrics>
 ) -> Result<Box<dyn std::any::Any + Send + Sync>, error::Error> {
 	Ok(Box::new(()))
 }

From 98a6a8c3ebf4ac57e82cb78b3259d77dcbc3905f Mon Sep 17 00:00:00 2001
From: Shawn Tabrizi <shawntabrizi@gmail.com>
Date: Fri, 18 Sep 2020 16:15:40 +0200
Subject: [PATCH 110/122] WeightInfo for Scheduler (#7138)

* initial scheduler stuff

* integrate weightinfo

* Update pallet_scheduler.rs
---
 bin/node/runtime/src/lib.rs                   |  4 +-
 bin/node/runtime/src/weights/mod.rs           |  1 +
 .../runtime/src/weights/pallet_scheduler.rs   | 51 +++++++++++++++++
 frame/democracy/src/tests.rs                  |  1 +
 frame/scheduler/src/benchmarking.rs           | 13 +++--
 frame/scheduler/src/default_weights.rs        | 50 +++++++++++++++++
 frame/scheduler/src/lib.rs                    | 56 ++++++++++++-------
 7 files changed, 148 insertions(+), 28 deletions(-)
 create mode 100644 bin/node/runtime/src/weights/pallet_scheduler.rs
 create mode 100644 frame/scheduler/src/default_weights.rs

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
index 03fe279366d50..0de78464bddec 100644
--- a/bin/node/runtime/src/lib.rs
+++ b/bin/node/runtime/src/lib.rs
@@ -273,6 +273,7 @@ impl pallet_proxy::Trait for Runtime {
 
 parameter_types! {
 	pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get();
+	pub const MaxScheduledPerBlock: u32 = 50;
 }
 
 impl pallet_scheduler::Trait for Runtime {
@@ -282,7 +283,8 @@ impl pallet_scheduler::Trait for Runtime {
 	type Call = Call;
 	type MaximumWeight = MaximumSchedulerWeight;
 	type ScheduleOrigin = EnsureRoot<AccountId>;
-	type WeightInfo = ();
+	type MaxScheduledPerBlock = MaxScheduledPerBlock;
+	type WeightInfo = weights::pallet_scheduler::WeightInfo;
 }
 
 parameter_types! {
diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs
index 199e66888e65d..668b9462a7d6a 100644
--- a/bin/node/runtime/src/weights/mod.rs
+++ b/bin/node/runtime/src/weights/mod.rs
@@ -24,6 +24,7 @@ pub mod pallet_identity;
 pub mod pallet_indices;
 pub mod pallet_im_online;
 pub mod pallet_proxy;
+pub mod pallet_scheduler;
 pub mod pallet_staking;
 pub mod pallet_timestamp;
 pub mod pallet_utility;
diff --git a/bin/node/runtime/src/weights/pallet_scheduler.rs b/bin/node/runtime/src/weights/pallet_scheduler.rs
new file mode 100644
index 0000000000000..110a0545ed839
--- /dev/null
+++ b/bin/node/runtime/src/weights/pallet_scheduler.rs
@@ -0,0 +1,51 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+pub struct WeightInfo;
+impl pallet_scheduler::WeightInfo for WeightInfo {
+	fn schedule(s: u32, ) -> Weight {
+		(37_835_000 as Weight)
+			.saturating_add((81_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn cancel(s: u32, ) -> Weight {
+		(34_707_000 as Weight)
+			.saturating_add((3_125_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn schedule_named(s: u32, ) -> Weight {
+		(48_065_000 as Weight)
+			.saturating_add((110_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn cancel_named(s: u32, ) -> Weight {
+		(38_776_000 as Weight)
+			.saturating_add((3_138_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+}
diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs
index aed6739c77ebb..4e569c98aee7f 100644
--- a/frame/democracy/src/tests.rs
+++ b/frame/democracy/src/tests.rs
@@ -128,6 +128,7 @@ impl pallet_scheduler::Trait for Test {
 	type Call = Call;
 	type MaximumWeight = MaximumSchedulerWeight;
 	type ScheduleOrigin = EnsureRoot<u64>;
+	type MaxScheduledPerBlock = ();
 	type WeightInfo = ();
 }
 parameter_types! {
diff --git a/frame/scheduler/src/benchmarking.rs b/frame/scheduler/src/benchmarking.rs
index 847460fe85a50..17b3a298f49ea 100644
--- a/frame/scheduler/src/benchmarking.rs
+++ b/frame/scheduler/src/benchmarking.rs
@@ -28,7 +28,6 @@ use frame_benchmarking::benchmarks;
 use crate::Module as Scheduler;
 use frame_system::Module as System;
 
-const MAX_SCHEDULED: u32 = 50;
 const BLOCK_NUMBER: u32 = 2;
 
 // Add `n` named items to the schedule
@@ -56,7 +55,7 @@ benchmarks! {
 	_ { }
 
 	schedule {
-		let s in 0 .. MAX_SCHEDULED;
+		let s in 0 .. T::MaxScheduledPerBlock::get();
 		let when = BLOCK_NUMBER.into();
 		let periodic = Some((T::BlockNumber::one(), 100));
 		let priority = 0;
@@ -73,7 +72,7 @@ benchmarks! {
 	}
 
 	cancel {
-		let s in 1 .. MAX_SCHEDULED;
+		let s in 1 .. T::MaxScheduledPerBlock::get();
 		let when = BLOCK_NUMBER.into();
 
 		fill_schedule::<T>(when, s)?;
@@ -92,7 +91,7 @@ benchmarks! {
 	}
 
 	schedule_named {
-		let s in 0 .. MAX_SCHEDULED;
+		let s in 0 .. T::MaxScheduledPerBlock::get();
 		let id = s.encode();
 		let when = BLOCK_NUMBER.into();
 		let periodic = Some((T::BlockNumber::one(), 100));
@@ -110,7 +109,7 @@ benchmarks! {
 	}
 
 	cancel_named {
-		let s in 1 .. MAX_SCHEDULED;
+		let s in 1 .. T::MaxScheduledPerBlock::get();
 		let when = BLOCK_NUMBER.into();
 
 		fill_schedule::<T>(when, s)?;
@@ -127,8 +126,10 @@ benchmarks! {
 		);
 	}
 
+	// TODO: Make this more complex and flexible so it can be used in automation.
+	#[extra]
 	on_initialize {
-		let s in 0 .. MAX_SCHEDULED;
+		let s in 0 .. T::MaxScheduledPerBlock::get();
 		let when = BLOCK_NUMBER.into();
 		fill_schedule::<T>(when, s)?;
 	}: { Scheduler::<T>::on_initialize(BLOCK_NUMBER.into()); }
diff --git a/frame/scheduler/src/default_weights.rs b/frame/scheduler/src/default_weights.rs
new file mode 100644
index 0000000000000..920de1d37a07c
--- /dev/null
+++ b/frame/scheduler/src/default_weights.rs
@@ -0,0 +1,50 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc6
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight};
+
+impl crate::WeightInfo for () {
+	fn schedule(s: u32, ) -> Weight {
+		(37_835_000 as Weight)
+			.saturating_add((81_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(1 as Weight))
+	}
+	fn cancel(s: u32, ) -> Weight {
+		(34_707_000 as Weight)
+			.saturating_add((3_125_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(1 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn schedule_named(s: u32, ) -> Weight {
+		(48_065_000 as Weight)
+			.saturating_add((110_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+	fn cancel_named(s: u32, ) -> Weight {
+		(38_776_000 as Weight)
+			.saturating_add((3_138_000 as Weight).saturating_mul(s as Weight))
+			.saturating_add(DbWeight::get().reads(2 as Weight))
+			.saturating_add(DbWeight::get().writes(2 as Weight))
+	}
+}
diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs
index edd112bd89299..99b9ea4ea5741 100644
--- a/frame/scheduler/src/lib.rs
+++ b/frame/scheduler/src/lib.rs
@@ -52,6 +52,7 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 
 mod benchmarking;
+mod default_weights;
 
 use sp_std::{prelude::*, marker::PhantomData, borrow::Borrow};
 use codec::{Encode, Decode, Codec};
@@ -69,15 +70,6 @@ pub trait WeightInfo {
 	fn cancel(s: u32, ) -> Weight;
 	fn schedule_named(s: u32, ) -> Weight;
 	fn cancel_named(s: u32, ) -> Weight;
-	fn on_initialize(s: u32, ) -> Weight;
-}
-
-impl WeightInfo for () {
-	fn schedule(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn cancel(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn schedule_named(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn cancel_named(_s: u32, ) -> Weight { 1_000_000_000 }
-	fn on_initialize(_s: u32, ) -> Weight { 1_000_000_000 }
 }
 
 /// Our pallet's configuration trait. All our types and constants go in here. If the
@@ -106,6 +98,10 @@ pub trait Trait: system::Trait {
 	/// Required origin to schedule or cancel calls.
 	type ScheduleOrigin: EnsureOrigin<<Self as system::Trait>::Origin>;
 
+	/// The maximum number of scheduled calls in the queue for a single block.
+	/// Not strictly enforced, but used for weight estimation.
+	type MaxScheduledPerBlock: Get<u32>;
+
 	/// Weight information for extrinsics in this pallet.
 	type WeightInfo: WeightInfo;
 }
@@ -213,7 +209,7 @@ decl_module! {
 		///     - Write: Agenda
 		/// - Will use base weight of 25 which should be good for up to 30 scheduled calls
 		/// # </weight>
-		#[weight = 25_000_000 + T::DbWeight::get().reads_writes(1, 1)]
+		#[weight = T::WeightInfo::schedule(T::MaxScheduledPerBlock::get())]
 		fn schedule(origin,
 			when: T::BlockNumber,
 			maybe_periodic: Option<schedule::Period<T::BlockNumber>>,
@@ -235,7 +231,7 @@ decl_module! {
 		///     - Write: Agenda, Lookup
 		/// - Will use base weight of 100 which should be good for up to 30 scheduled calls
 		/// # </weight>
-		#[weight = 100_000_000 + T::DbWeight::get().reads_writes(1, 2)]
+		#[weight = T::WeightInfo::cancel(T::MaxScheduledPerBlock::get())]
 		fn cancel(origin, when: T::BlockNumber, index: u32) {
 			T::ScheduleOrigin::ensure_origin(origin.clone())?;
 			let origin = <T as Trait>::Origin::from(origin);
@@ -252,7 +248,7 @@ decl_module! {
 		///     - Write: Agenda, Lookup
 		/// - Will use base weight of 35 which should be good for more than 30 scheduled calls
 		/// # </weight>
-		#[weight = 35_000_000 + T::DbWeight::get().reads_writes(2, 2)]
+		#[weight = T::WeightInfo::schedule_named(T::MaxScheduledPerBlock::get())]
 		fn schedule_named(origin,
 			id: Vec<u8>,
 			when: T::BlockNumber,
@@ -277,7 +273,7 @@ decl_module! {
 		///     - Write: Agenda, Lookup
 		/// - Will use base weight of 100 which should be good for up to 30 scheduled calls
 		/// # </weight>
-		#[weight = 100_000_000 + T::DbWeight::get().reads_writes(2, 2)]
+		#[weight = T::WeightInfo::cancel_named(T::MaxScheduledPerBlock::get())]
 		fn cancel_named(origin, id: Vec<u8>) {
 			T::ScheduleOrigin::ensure_origin(origin.clone())?;
 			let origin = <T as Trait>::Origin::from(origin);
@@ -289,7 +285,7 @@ decl_module! {
 		/// # <weight>
 		/// Same as [`schedule`].
 		/// # </weight>
-		#[weight = 25_000_000 + T::DbWeight::get().reads_writes(1, 1)]
+		#[weight = T::WeightInfo::schedule(T::MaxScheduledPerBlock::get())]
 		fn schedule_after(origin,
 			after: T::BlockNumber,
 			maybe_periodic: Option<schedule::Period<T::BlockNumber>>,
@@ -308,7 +304,7 @@ decl_module! {
 		/// # <weight>
 		/// Same as [`schedule_named`].
 		/// # </weight>
-		#[weight = 35_000_000 + T::DbWeight::get().reads_writes(2, 2)]
+		#[weight = T::WeightInfo::schedule_named(T::MaxScheduledPerBlock::get())]
 		fn schedule_named_after(origin,
 			id: Vec<u8>,
 			after: T::BlockNumber,
@@ -340,16 +336,20 @@ decl_module! {
 				.enumerate()
 				.filter_map(|(index, s)| s.map(|inner| (index as u32, inner)))
 				.collect::<Vec<_>>();
+			if queued.len() as u32 > T::MaxScheduledPerBlock::get() {
+				frame_support::debug::warn!(
+					"Warning: This block has more items queued in Scheduler than \
+					expected from the runtime configuration. An update might be needed."
+				);
+			}
 			queued.sort_by_key(|(_, s)| s.priority);
-			let base_weight: Weight = T::DbWeight::get().reads_writes(1, 2) // Agenda + Agenda(next)
-				.saturating_add(10_000_000); // Base Weight
+			let base_weight: Weight = T::DbWeight::get().reads_writes(1, 2); // Agenda + Agenda(next)
 			let mut total_weight: Weight = 0;
 			queued.into_iter()
 				.enumerate()
 				.scan(base_weight, |cumulative_weight, (order, (index, s))| {
 					*cumulative_weight = cumulative_weight
-						.saturating_add(s.call.get_dispatch_info().weight)
-						.saturating_add(25_000_000); // Base multiplier
+						.saturating_add(s.call.get_dispatch_info().weight);
 
 					if s.maybe_id.is_some() {
 						// Remove/Modify Lookup
@@ -466,6 +466,12 @@ impl<T: Trait> Module<T> {
 		});
 		Agenda::<T>::append(when, s);
 		let index = Agenda::<T>::decode_len(when).unwrap_or(1) as u32 - 1;
+		if index > T::MaxScheduledPerBlock::get() {
+			frame_support::debug::warn!(
+				"Warning: There are more items queued in the Scheduler than \
+				expected from the runtime configuration. An update might be needed."
+			);
+		}
 		Self::deposit_event(RawEvent::Scheduled(when, index));
 
 		Ok((when, index))
@@ -535,6 +541,12 @@ impl<T: Trait> Module<T> {
 		};
 		Agenda::<T>::append(when, Some(s));
 		let index = Agenda::<T>::decode_len(when).unwrap_or(1) as u32 - 1;
+		if index > T::MaxScheduledPerBlock::get() {
+			frame_support::debug::warn!(
+				"Warning: There are more items queued in the Scheduler than \
+				expected from the runtime configuration. An update might be needed."
+			);
+		}
 		let address = (when, index);
 		Lookup::<T>::insert(&id, &address);
 		Self::deposit_event(RawEvent::Scheduled(when, index));
@@ -734,6 +746,7 @@ mod tests {
 	}
 	parameter_types! {
 		pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get();
+		pub const MaxScheduledPerBlock: u32 = 10;
 	}
 	ord_parameter_types! {
 		pub const One: u64 = 1;
@@ -746,6 +759,7 @@ mod tests {
 		type Call = Call;
 		type MaximumWeight = MaximumSchedulerWeight;
 		type ScheduleOrigin = EnsureOneOf<u64, EnsureRoot<u64>, EnsureSignedBy<One, u64>>;
+		type MaxScheduledPerBlock = MaxScheduledPerBlock;
 		type WeightInfo = ();
 	}
 	type System = system::Module<Test>;
@@ -982,8 +996,8 @@ mod tests {
 	#[test]
 	fn on_initialize_weight_is_correct() {
 		new_test_ext().execute_with(|| {
-			let base_weight: Weight = <Test as frame_system::Trait>::DbWeight::get().reads_writes(1, 2) + 10_000_000;
-			let base_multiplier = 25_000_000;
+			let base_weight: Weight = <Test as frame_system::Trait>::DbWeight::get().reads_writes(1, 2);
+			let base_multiplier = 0;
 			let named_multiplier = <Test as frame_system::Trait>::DbWeight::get().writes(1);
 			let periodic_multiplier = <Test as frame_system::Trait>::DbWeight::get().reads_writes(1, 1);
 

From f75c54041bd4ad752d27447ce3ba54c2d65e13dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= <jon.haggblad@gmail.com>
Date: Fri, 18 Sep 2020 18:39:27 +0200
Subject: [PATCH 111/122] grandpa-rpc: use FinalityProofProvider to check
 finality for rpc (#6215)

* grandpa-rpc: use FinalityProofProvider to check finality for rpc

* grandpa-rpc: minor tidy

* grandpa-rpc: remove dyn FinalityProofProvider

* grandpa-rpc: remove unused dependencies

* node: move finality_proof_provider setup

* grandpa-rpc: print error reported by finality_proof_provider

* grandpa-rpc: add note about unnecessary encode/decode

* grandpa-rpc: dont encode/decode and use correct hash

* grandpa-rpc: set_id is optional

* grandpa-rpc: create test for prove_finality

* grandpa-rpc: set visibility back to how it was

* grandpa-rpc: remove unused dependency

* grandpa-rpc: minor tidy

* grandpa: doc strings

* grandpa-rpc: rename to prove_finality

* grandpa-rpc: use current set id if none is provided

* grandpa-rpc: remove unnecessary check in test

* node: group finality_proof_provider in rpc_setup

* grandpa: make prove_finality concrete in FinalityProofProvider

* grandpa-rpc: wrap finality output in struct and store as Bytes

* grandpa-rpc: exhaustive error codes and wrap

* grandpa-rpc: let prove_finality take a range instead of a starting point

* grandpa-rpc: fix test for changed API

* grandpa-rpc: fix line length

* grandpa: fix reviewer nits

* node/rpc: fix reviewer comments
---
 Cargo.lock                                    |   2 +
 bin/node/cli/src/service.rs                   |  14 +-
 bin/node/rpc/Cargo.toml                       |   1 +
 bin/node/rpc/src/lib.rs                       |  20 ++-
 client/finality-grandpa/rpc/Cargo.toml        |   3 +-
 client/finality-grandpa/rpc/src/error.rs      |  34 ++++-
 client/finality-grandpa/rpc/src/finality.rs   |  54 +++++++
 client/finality-grandpa/rpc/src/lib.rs        | 137 ++++++++++++++++--
 client/finality-grandpa/src/finality_proof.rs |  27 +++-
 client/finality-grandpa/src/lib.rs            |   2 +-
 10 files changed, 263 insertions(+), 31 deletions(-)
 create mode 100644 client/finality-grandpa/rpc/src/finality.rs

diff --git a/Cargo.lock b/Cargo.lock
index f635807f6fbc6..3b08f91f2a8dc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3823,6 +3823,7 @@ dependencies = [
  "sp-blockchain",
  "sp-consensus",
  "sp-consensus-babe",
+ "sp-runtime",
  "sp-transaction-pool",
  "substrate-frame-rpc-system",
 ]
@@ -6821,6 +6822,7 @@ dependencies = [
  "log",
  "parity-scale-codec",
  "sc-block-builder",
+ "sc-client-api",
  "sc-finality-grandpa",
  "sc-network-test",
  "sc-rpc",
diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs
index 03347e455e6a3..b15ace6181a8f 100644
--- a/bin/node/cli/src/service.rs
+++ b/bin/node/cli/src/service.rs
@@ -58,7 +58,10 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 			grandpa::LinkHalf<Block, FullClient, FullSelectChain>,
 			sc_consensus_babe::BabeLink<Block>,
 		),
-		grandpa::SharedVoterState,
+		(
+			grandpa::SharedVoterState,
+			Arc<GrandpaFinalityProofProvider<FullBackend, Block>>,
+		),
 	)
 >, ServiceError> {
 	let (client, backend, keystore, task_manager) =
@@ -108,8 +111,10 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 		let justification_stream = grandpa_link.justification_stream();
 		let shared_authority_set = grandpa_link.shared_authority_set().clone();
 		let shared_voter_state = grandpa::SharedVoterState::empty();
+		let finality_proof_provider =
+			GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone());
 
-		let rpc_setup = shared_voter_state.clone();
+		let rpc_setup = (shared_voter_state.clone(), finality_proof_provider.clone());
 
 		let babe_config = babe_link.config().clone();
 		let shared_epoch_changes = babe_link.epoch_changes().clone();
@@ -135,6 +140,7 @@ pub fn new_partial(config: &Configuration) -> Result<sc_service::PartialComponen
 					shared_authority_set: shared_authority_set.clone(),
 					justification_stream: justification_stream.clone(),
 					subscription_executor,
+					finality_provider: finality_proof_provider.clone(),
 				},
 			};
 
@@ -174,8 +180,7 @@ pub fn new_full_base(
 		other: (rpc_extensions_builder, import_setup, rpc_setup),
 	} = new_partial(&config)?;
 
-	let finality_proof_provider =
-		GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone());
+	let (shared_voter_state, finality_proof_provider) = rpc_setup;
 
 	let (network, network_status_sinks, system_rpc_tx, network_starter) =
 		sc_service::build_network(sc_service::BuildNetworkParams {
@@ -220,7 +225,6 @@ pub fn new_full_base(
 	})?;
 
 	let (block_import, grandpa_link, babe_link) = import_setup;
-	let shared_voter_state = rpc_setup;
 
 	(with_startup_data)(&block_import, &babe_link);
 
diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml
index e9abaef35f655..020fc88a3d0f2 100644
--- a/bin/node/rpc/Cargo.toml
+++ b/bin/node/rpc/Cargo.toml
@@ -30,5 +30,6 @@ sp-block-builder = { version = "2.0.0-rc6", path = "../../../primitives/block-bu
 sp-blockchain = { version = "2.0.0-rc6", path = "../../../primitives/blockchain" }
 sp-consensus = { version = "0.8.0-rc6", path = "../../../primitives/consensus/common" }
 sp-consensus-babe = { version = "0.8.0-rc6", path = "../../../primitives/consensus/babe" }
+sp-runtime = { version = "2.0.0-rc6", path = "../../../primitives/runtime" }
 sp-transaction-pool = { version = "2.0.0-rc6", path = "../../../primitives/transaction-pool" }
 substrate-frame-rpc-system = { version = "2.0.0-rc6", path = "../../../utils/frame/rpc/system" }
diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs
index 420fd0eb1475b..55a7adf612119 100644
--- a/bin/node/rpc/src/lib.rs
+++ b/bin/node/rpc/src/lib.rs
@@ -36,7 +36,9 @@ use node_primitives::{Block, BlockNumber, AccountId, Index, Balance, Hash};
 use sc_consensus_babe::{Config, Epoch};
 use sc_consensus_babe_rpc::BabeRpcHandler;
 use sc_consensus_epochs::SharedEpochChanges;
-use sc_finality_grandpa::{SharedVoterState, SharedAuthoritySet, GrandpaJustificationStream};
+use sc_finality_grandpa::{
+	SharedVoterState, SharedAuthoritySet, FinalityProofProvider, GrandpaJustificationStream
+};
 use sc_finality_grandpa_rpc::GrandpaRpcHandler;
 use sc_keystore::KeyStorePtr;
 pub use sc_rpc_api::DenyUnsafe;
@@ -71,7 +73,7 @@ pub struct BabeDeps {
 }
 
 /// Extra dependencies for GRANDPA
-pub struct GrandpaDeps {
+pub struct GrandpaDeps<B> {
 	/// Voting round info.
 	pub shared_voter_state: SharedVoterState,
 	/// Authority set info.
@@ -80,10 +82,12 @@ pub struct GrandpaDeps {
 	pub justification_stream: GrandpaJustificationStream<Block>,
 	/// Executor to drive the subscription manager in the Grandpa RPC handler.
 	pub subscription_executor: SubscriptionTaskExecutor,
+	/// Finality proof provider.
+	pub finality_provider: Arc<FinalityProofProvider<B, Block>>,
 }
 
 /// Full client dependencies.
-pub struct FullDeps<C, P, SC> {
+pub struct FullDeps<C, P, SC, B> {
 	/// The client instance to use.
 	pub client: Arc<C>,
 	/// Transaction pool instance.
@@ -95,15 +99,15 @@ pub struct FullDeps<C, P, SC> {
 	/// BABE specific dependencies.
 	pub babe: BabeDeps,
 	/// GRANDPA specific dependencies.
-	pub grandpa: GrandpaDeps,
+	pub grandpa: GrandpaDeps<B>,
 }
 
 /// A IO handler that uses all Full RPC extensions.
 pub type IoHandler = jsonrpc_core::IoHandler<sc_rpc::Metadata>;
 
 /// Instantiate all Full RPC extensions.
-pub fn create_full<C, P, SC>(
-	deps: FullDeps<C, P, SC>,
+pub fn create_full<C, P, SC, B>(
+	deps: FullDeps<C, P, SC, B>,
 ) -> jsonrpc_core::IoHandler<sc_rpc_api::Metadata> where
 	C: ProvideRuntimeApi<Block>,
 	C: HeaderBackend<Block> + HeaderMetadata<Block, Error=BlockChainError> + 'static,
@@ -115,6 +119,8 @@ pub fn create_full<C, P, SC>(
 	C::Api: BlockBuilder<Block>,
 	P: TransactionPool + 'static,
 	SC: SelectChain<Block> +'static,
+	B: sc_client_api::Backend<Block> + Send + Sync + 'static,
+	B::State: sc_client_api::backend::StateBackend<sp_runtime::traits::HashFor<Block>>,
 {
 	use substrate_frame_rpc_system::{FullSystem, SystemApi};
 	use pallet_contracts_rpc::{Contracts, ContractsApi};
@@ -140,6 +146,7 @@ pub fn create_full<C, P, SC>(
 		shared_authority_set,
 		justification_stream,
 		subscription_executor,
+		finality_provider,
 	} = grandpa;
 
 	io.extend_with(
@@ -173,6 +180,7 @@ pub fn create_full<C, P, SC>(
 				shared_voter_state,
 				justification_stream,
 				subscription_executor,
+				finality_provider,
 			)
 		)
 	);
diff --git a/client/finality-grandpa/rpc/Cargo.toml b/client/finality-grandpa/rpc/Cargo.toml
index 6f3014644eaa4..0112ddd420c99 100644
--- a/client/finality-grandpa/rpc/Cargo.toml
+++ b/client/finality-grandpa/rpc/Cargo.toml
@@ -10,6 +10,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
 [dependencies]
 sc-finality-grandpa = { version = "0.8.0-rc6", path = "../" }
 sc-rpc = { version = "2.0.0-rc6", path = "../../rpc" }
+sp-blockchain = { version = "2.0.0-rc6", path = "../../../primitives/blockchain" }
 sp-core = { version = "2.0.0-rc6", path = "../../../primitives/core" }
 sp-runtime = { version = "2.0.0-rc6", path = "../../../primitives/runtime" }
 finality-grandpa = { version = "0.12.3", features = ["derive-codec"] }
@@ -23,12 +24,12 @@ serde_json = "1.0.50"
 log = "0.4.8"
 derive_more = "0.99.2"
 parity-scale-codec = { version = "1.3.0", features = ["derive"] }
+sc-client-api = { version = "2.0.0-rc6", path = "../../api" }
 
 [dev-dependencies]
 sc-block-builder = { version = "0.8.0-rc6", path = "../../block-builder" }
 sc-network-test = { version = "0.8.0-rc6", path = "../../network/test" }
 sc-rpc = { version = "2.0.0-rc6", path = "../../rpc", features = ["test-helpers"] }
-sp-blockchain = { version = "2.0.0-rc6", path = "../../../primitives/blockchain" }
 sp-consensus = { version = "0.8.0-rc6", path = "../../../primitives/consensus/common" }
 sp-core = { version = "2.0.0-rc6", path = "../../../primitives/core" }
 sp-finality-grandpa = { version = "2.0.0-rc6", path = "../../../primitives/finality-grandpa" }
diff --git a/client/finality-grandpa/rpc/src/error.rs b/client/finality-grandpa/rpc/src/error.rs
index bfd0596fdf320..6464acbe10ea0 100644
--- a/client/finality-grandpa/rpc/src/error.rs
+++ b/client/finality-grandpa/rpc/src/error.rs
@@ -16,8 +16,6 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use crate::NOT_READY_ERROR_CODE;
-
 #[derive(derive_more::Display, derive_more::From)]
 /// Top-level error type for the RPC handler
 pub enum Error {
@@ -30,13 +28,41 @@ pub enum Error {
 	/// GRANDPA reports voter state with round id or weights larger than 32-bits.
 	#[display(fmt = "GRANDPA reports voter state as unreasonably large")]
 	VoterStateReportsUnreasonablyLargeNumbers,
+	/// GRANDPA prove finality failed.
+	#[display(fmt = "GRANDPA prove finality rpc failed: {}", _0)]
+	ProveFinalityFailed(sp_blockchain::Error),
+}
+
+/// The error codes returned by jsonrpc.
+pub enum ErrorCode {
+	/// Returned when Grandpa RPC endpoint is not ready.
+	NotReady = 1,
+	/// Authority set ID is larger than 32-bits.
+	AuthoritySetTooLarge,
+	/// Voter state with round id or weights larger than 32-bits.
+	VoterStateTooLarge,
+	/// Failed to prove finality.
+	ProveFinality,
+}
+
+impl From<Error> for ErrorCode {
+	fn from(error: Error) -> Self {
+		match error {
+			Error::EndpointNotReady => ErrorCode::NotReady,
+			Error::AuthoritySetIdReportedAsUnreasonablyLarge => ErrorCode::AuthoritySetTooLarge,
+			Error::VoterStateReportsUnreasonablyLargeNumbers => ErrorCode::VoterStateTooLarge,
+			Error::ProveFinalityFailed(_) => ErrorCode::ProveFinality,
+		}
+	}
 }
 
 impl From<Error> for jsonrpc_core::Error {
 	fn from(error: Error) -> Self {
+		let message = format!("{}", error);
+		let code = ErrorCode::from(error);
 		jsonrpc_core::Error {
-			message: format!("{}", error),
-			code: jsonrpc_core::ErrorCode::ServerError(NOT_READY_ERROR_CODE),
+			message,
+			code: jsonrpc_core::ErrorCode::ServerError(code as i64),
 			data: None,
 		}
 	}
diff --git a/client/finality-grandpa/rpc/src/finality.rs b/client/finality-grandpa/rpc/src/finality.rs
new file mode 100644
index 0000000000000..1f288b86a0e46
--- /dev/null
+++ b/client/finality-grandpa/rpc/src/finality.rs
@@ -0,0 +1,54 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{Serialize, Deserialize};
+
+use sc_finality_grandpa::FinalityProofProvider;
+use sp_runtime::traits::{Block as BlockT, NumberFor};
+
+#[derive(Serialize, Deserialize)]
+pub struct EncodedFinalityProofs(pub sp_core::Bytes);
+
+/// Local trait mainly to allow mocking in tests.
+pub trait RpcFinalityProofProvider<Block: BlockT> {
+	/// Return finality proofs for the given authorities set id, if it is provided, otherwise the
+	/// current one will be used.
+	fn rpc_prove_finality(
+		&self,
+		begin: Block::Hash,
+		end: Block::Hash,
+		authorities_set_id: u64,
+	) -> Result<Option<EncodedFinalityProofs>, sp_blockchain::Error>;
+}
+
+impl<B, Block> RpcFinalityProofProvider<Block> for FinalityProofProvider<B, Block>
+where
+	Block: BlockT,
+	NumberFor<Block>: finality_grandpa::BlockNumberOps,
+	B: sc_client_api::backend::Backend<Block> + Send + Sync + 'static,
+{
+	fn rpc_prove_finality(
+		&self,
+		begin: Block::Hash,
+		end: Block::Hash,
+		authorities_set_id: u64,
+	) -> Result<Option<EncodedFinalityProofs>, sp_blockchain::Error> {
+		self.prove_finality(begin, end, authorities_set_id)
+			.map(|x| x.map(|y| EncodedFinalityProofs(y.into())))
+	}
+}
diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs
index fedd7220d3115..172473ad6518b 100644
--- a/client/finality-grandpa/rpc/src/lib.rs
+++ b/client/finality-grandpa/rpc/src/lib.rs
@@ -32,24 +32,23 @@ use jsonrpc_core::futures::{
 };
 
 mod error;
+mod finality;
 mod notification;
 mod report;
 
 use sc_finality_grandpa::GrandpaJustificationStream;
 use sp_runtime::traits::Block as BlockT;
 
+use finality::{EncodedFinalityProofs, RpcFinalityProofProvider};
 use report::{ReportAuthoritySet, ReportVoterState, ReportedRoundStates};
 use notification::JustificationNotification;
 
-/// Returned when Grandpa RPC endpoint is not ready.
-pub const NOT_READY_ERROR_CODE: i64 = 1;
-
 type FutureResult<T> =
 	Box<dyn jsonrpc_core::futures::Future<Item = T, Error = jsonrpc_core::Error> + Send>;
 
 /// Provides RPC methods for interacting with GRANDPA.
 #[rpc]
-pub trait GrandpaApi<Notification> {
+pub trait GrandpaApi<Notification, Hash> {
 	/// RPC Metadata
 	type Metadata;
 
@@ -82,23 +81,37 @@ pub trait GrandpaApi<Notification> {
 		metadata: Option<Self::Metadata>,
 		id: SubscriptionId
 	) -> jsonrpc_core::Result<bool>;
+
+	/// Prove finality for the range (begin; end] hash. Returns None if there are no finalized blocks
+	/// unknown in the range. If no authorities set is provided, the current one will be attempted.
+	#[rpc(name = "grandpa_proveFinality")]
+	fn prove_finality(
+		&self,
+		begin: Hash,
+		end: Hash,
+		authorities_set_id: Option<u64>,
+	) -> FutureResult<Option<EncodedFinalityProofs>>;
 }
 
 /// Implements the GrandpaApi RPC trait for interacting with GRANDPA.
-pub struct GrandpaRpcHandler<AuthoritySet, VoterState, Block: BlockT> {
+pub struct GrandpaRpcHandler<AuthoritySet, VoterState, Block: BlockT, ProofProvider> {
 	authority_set: AuthoritySet,
 	voter_state: VoterState,
 	justification_stream: GrandpaJustificationStream<Block>,
 	manager: SubscriptionManager,
+	finality_proof_provider: Arc<ProofProvider>,
 }
 
-impl<AuthoritySet, VoterState, Block: BlockT> GrandpaRpcHandler<AuthoritySet, VoterState, Block> {
+impl<AuthoritySet, VoterState, Block: BlockT, ProofProvider>
+	GrandpaRpcHandler<AuthoritySet, VoterState, Block, ProofProvider>
+{
 	/// Creates a new GrandpaRpcHandler instance.
 	pub fn new<E>(
 		authority_set: AuthoritySet,
 		voter_state: VoterState,
 		justification_stream: GrandpaJustificationStream<Block>,
 		executor: E,
+		finality_proof_provider: Arc<ProofProvider>,
 	) -> Self
 	where
 		E: Executor01<Box<dyn Future01<Item = (), Error = ()> + Send>> + Send + Sync + 'static,
@@ -109,16 +122,18 @@ impl<AuthoritySet, VoterState, Block: BlockT> GrandpaRpcHandler<AuthoritySet, Vo
 			voter_state,
 			justification_stream,
 			manager,
+			finality_proof_provider,
 		}
 	}
 }
 
-impl<AuthoritySet, VoterState, Block> GrandpaApi<JustificationNotification>
-	for GrandpaRpcHandler<AuthoritySet, VoterState, Block>
+impl<AuthoritySet, VoterState, Block, ProofProvider> GrandpaApi<JustificationNotification, Block::Hash>
+	for GrandpaRpcHandler<AuthoritySet, VoterState, Block, ProofProvider>
 where
 	VoterState: ReportVoterState + Send + Sync + 'static,
 	AuthoritySet: ReportAuthoritySet + Send + Sync + 'static,
 	Block: BlockT,
+	ProofProvider: RpcFinalityProofProvider<Block> + Send + Sync + 'static,
 {
 	type Metadata = sc_rpc::Metadata;
 
@@ -153,6 +168,30 @@ where
 	) -> jsonrpc_core::Result<bool> {
 		Ok(self.manager.cancel(id))
 	}
+
+	fn prove_finality(
+		&self,
+		begin: Block::Hash,
+		end: Block::Hash,
+		authorities_set_id: Option<u64>,
+	) -> FutureResult<Option<EncodedFinalityProofs>> {
+		// If we are not provided a set_id, try with the current one.
+		let authorities_set_id = authorities_set_id
+			.unwrap_or_else(|| self.authority_set.get().0);
+		let result = self
+			.finality_proof_provider
+			.rpc_prove_finality(begin, end, authorities_set_id);
+		let future = async move { result }.boxed();
+		Box::new(
+			future
+				.map_err(|e| {
+					warn!("Error proving finality: {}", e);
+					error::Error::ProveFinalityFailed(e)
+				})
+				.map_err(jsonrpc_core::Error::from)
+				.compat()
+		)
+	}
 }
 
 #[cfg(test)]
@@ -161,16 +200,19 @@ mod tests {
 	use std::{collections::HashSet, convert::TryInto, sync::Arc};
 	use jsonrpc_core::{Notification, Output, types::Params};
 
-	use parity_scale_codec::Decode;
+	use parity_scale_codec::{Encode, Decode};
 	use sc_block_builder::BlockBuilder;
-	use sc_finality_grandpa::{report, AuthorityId, GrandpaJustificationSender, GrandpaJustification};
+	use sc_finality_grandpa::{
+		report, AuthorityId, GrandpaJustificationSender, GrandpaJustification,
+		FinalityProofFragment,
+	};
 	use sp_blockchain::HeaderBackend;
 	use sp_consensus::RecordProof;
 	use sp_core::crypto::Public;
 	use sp_keyring::Ed25519Keyring;
-	use sp_runtime::traits::Header as HeaderT;
+	use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 	use substrate_test_runtime_client::{
-		runtime::Block,
+		runtime::{Block, Header, H256},
 		DefaultTestClientBuilderExt,
 		TestClientBuilderExt,
 		TestClientBuilder,
@@ -180,6 +222,10 @@ mod tests {
 	struct TestVoterState;
 	struct EmptyVoterState;
 
+	struct TestFinalityProofProvider {
+		finality_proofs: Vec<FinalityProofFragment<Header>>,
+	}
+
 	fn voters() -> HashSet<AuthorityId> {
 		let voter_id_1 = AuthorityId::from_slice(&[1; 32]);
 		let voter_id_2 = AuthorityId::from_slice(&[2; 32]);
@@ -199,6 +245,31 @@ mod tests {
 		}
 	}
 
+	fn header(number: u64) -> Header {
+		let parent_hash = match number {
+			0 => Default::default(),
+			_ => header(number - 1).hash(),
+		};
+		Header::new(
+			number,
+			H256::from_low_u64_be(0),
+			H256::from_low_u64_be(0),
+			parent_hash,
+			Default::default(),
+		)
+	}
+
+	impl<Block: BlockT> RpcFinalityProofProvider<Block> for TestFinalityProofProvider {
+		fn rpc_prove_finality(
+			&self,
+			_begin: Block::Hash,
+			_end: Block::Hash,
+			_authoritites_set_id: u64,
+		) -> Result<Option<EncodedFinalityProofs>, sp_blockchain::Error> {
+			Ok(Some(EncodedFinalityProofs(self.finality_proofs.encode().into())))
+		}
+	}
+
 	impl ReportVoterState for TestVoterState {
 		fn get(&self) -> Option<report::VoterState<AuthorityId>> {
 			let voter_id_1 = AuthorityId::from_slice(&[1; 32]);
@@ -236,14 +307,28 @@ mod tests {
 		GrandpaJustificationSender<Block>,
 	) where
 		VoterState: ReportVoterState + Send + Sync + 'static,
+	{
+		setup_io_handler_with_finality_proofs(voter_state, Default::default())
+	}
+
+	fn setup_io_handler_with_finality_proofs<VoterState>(
+		voter_state: VoterState,
+		finality_proofs: Vec<FinalityProofFragment<Header>>,
+	) -> (
+		jsonrpc_core::MetaIoHandler<sc_rpc::Metadata>,
+		GrandpaJustificationSender<Block>,
+	) where
+		VoterState: ReportVoterState + Send + Sync + 'static,
 	{
 		let (justification_sender, justification_stream) = GrandpaJustificationStream::channel();
+		let finality_proof_provider = Arc::new(TestFinalityProofProvider { finality_proofs });
 
 		let handler = GrandpaRpcHandler::new(
 			TestAuthoritySet,
 			voter_state,
 			justification_stream,
 			sc_rpc::testing::TaskExecutor,
+			finality_proof_provider,
 		);
 
 		let mut io = jsonrpc_core::MetaIoHandler::default();
@@ -432,4 +517,32 @@ mod tests {
 		assert_eq!(recv_sub_id, sub_id);
 		assert_eq!(recv_justification, justification);
 	}
+
+	#[test]
+	fn prove_finality_with_test_finality_proof_provider() {
+		let finality_proofs = vec![FinalityProofFragment {
+			block: header(42).hash(),
+			justification: create_justification().encode(),
+			unknown_headers: vec![header(2)],
+			authorities_proof: None,
+		}];
+		let (io,  _) = setup_io_handler_with_finality_proofs(
+			TestVoterState,
+			finality_proofs.clone(),
+		);
+
+		let request = "{\"jsonrpc\":\"2.0\",\"method\":\"grandpa_proveFinality\",\"params\":[\
+			\"0x0000000000000000000000000000000000000000000000000000000000000000\",\
+			\"0x0000000000000000000000000000000000000000000000000000000000000001\",\
+			42\
+		],\"id\":1}";
+
+		let meta = sc_rpc::Metadata::default();
+		let resp = io.handle_request_sync(request, meta);
+		let mut resp: serde_json::Value = serde_json::from_str(&resp.unwrap()).unwrap();
+		let result: sp_core::Bytes = serde_json::from_value(resp["result"].take()).unwrap();
+		let fragments: Vec<FinalityProofFragment<Header>> =
+			Decode::decode(&mut &result[..]).unwrap();
+		assert_eq!(fragments, finality_proofs);
+	}
 }
diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs
index 2ac9ec57f3df4..33dd69cc11d6e 100644
--- a/client/finality-grandpa/src/finality_proof.rs
+++ b/client/finality-grandpa/src/finality_proof.rs
@@ -180,7 +180,30 @@ impl<B, Block: BlockT> FinalityProofProvider<B, Block>
 	) -> Arc<Self> {
 		Arc::new(Self::new(backend, storage_and_proof_provider))
 	}
+}
 
+impl<B, Block> FinalityProofProvider<B, Block>
+	where
+		Block: BlockT,
+		NumberFor<Block>: BlockNumberOps,
+		B: Backend<Block> + Send + Sync + 'static,
+{
+	/// Prove finality for the range (begin; end] hash. Returns None if there are no finalized blocks
+	/// unknown in the range.
+	pub fn prove_finality(
+		&self,
+		begin: Block::Hash,
+		end: Block::Hash,
+		authorities_set_id: u64,
+	) -> Result<Option<Vec<u8>>, ClientError> {
+		prove_finality::<_, _, GrandpaJustification<Block>>(
+			&*self.backend.blockchain(),
+			&*self.authority_provider,
+			authorities_set_id,
+			begin,
+			end,
+		)
+	}
 }
 
 impl<B, Block> sc_network::config::FinalityProofProvider<Block> for FinalityProofProvider<B, Block>
@@ -232,8 +255,8 @@ pub struct FinalityEffects<Header: HeaderT> {
 /// 1) the justification for the descendant block F;
 /// 2) headers sub-chain (B; F] if B != F;
 /// 3) proof of GRANDPA::authorities() if the set changes at block F.
-#[derive(Debug, PartialEq, Encode, Decode)]
-pub(crate) struct FinalityProofFragment<Header: HeaderT> {
+#[derive(Debug, PartialEq, Encode, Decode, Clone)]
+pub struct FinalityProofFragment<Header: HeaderT> {
 	/// The hash of block F for which justification is provided.
 	pub block: Header::Hash,
 	/// Justification of the block F.
diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs
index ab84591f9cdec..a15130942c30f 100644
--- a/client/finality-grandpa/src/lib.rs
+++ b/client/finality-grandpa/src/lib.rs
@@ -125,7 +125,7 @@ mod until_imported;
 mod voting_rule;
 
 pub use authorities::SharedAuthoritySet;
-pub use finality_proof::{FinalityProofProvider, StorageAndProofProvider};
+pub use finality_proof::{FinalityProofFragment, FinalityProofProvider, StorageAndProofProvider};
 pub use notification::{GrandpaJustificationSender, GrandpaJustificationStream};
 pub use import::GrandpaBlockImport;
 pub use justification::GrandpaJustification;

From 3a0cb2ab9c3da8ad476dd915c1b8f1f0fa29a6df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Fri, 18 Sep 2020 22:01:57 +0200
Subject: [PATCH 112/122] Make it compile

---
 client/network/src/protocol/sync.rs | 24 ++++++++++++++----------
 client/network/test/src/sync.rs     | 18 ++++++++----------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 5d922e67c2e58..027d63d662c0c 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -50,7 +50,7 @@ use sp_runtime::{
 };
 use sp_arithmetic::traits::Saturating;
 use std::{fmt, ops::Range, collections::{HashMap, HashSet, VecDeque}, sync::Arc};
-use futures::future::{Either as FEither, Future, ready, FutureExt};
+use futures::future::{Either as FEither, Future, ready};
 
 mod blocks;
 mod extra_requests;
@@ -1201,13 +1201,16 @@ impl<B: BlockT> ChainSync<B> {
 	/// [`PreValidateBlockAnnounce`]. This result needs to be feed into
 	/// [`ChainSync::process_block_announce_pre_validation`] to finish
 	/// the block announcement processing.
-	pub async fn pre_validate_block_announce(
+	pub fn pre_validate_block_announce(
 		&mut self,
 		who: PeerId,
 		hash: &B::Hash,
 		announce: BlockAnnounce<B::Header>,
 		is_best: bool,
-	) -> PreValidateBlockAnnounce<B::Header> {
+	) -> FEither<
+		impl Future<Output = PreValidateBlockAnnounce<B::Header>>,
+		impl Future<Output = PreValidateBlockAnnounce<B::Header>>
+	> {
 		let header = &announce.header;
 		let number = *header.number();
 		debug!(
@@ -1225,7 +1228,7 @@ impl<B: BlockT> ChainSync<B> {
 				who,
 				hash,
 			);
-			return PreValidateBlockAnnounce::Nothing { is_best, who, announce }
+			return FEither::Left(ready(PreValidateBlockAnnounce::Nothing { is_best, who, announce }))
 		}
 
 		// Let external validator check the block announcement.
@@ -1233,13 +1236,13 @@ impl<B: BlockT> ChainSync<B> {
 		let future = self.block_announce_validator.validate(&header, assoc_data);
 		let hash = hash.clone();
 
+		FEither::Right(async move {
 			match future.await {
-				Ok(Validation::Success { is_new_best }) =>
-					PreValidateBlockAnnounce::Process {
-						is_new_best: is_new_best || is_best,
-						announce,
-						who,
-					},
+				Ok(Validation::Success { is_new_best }) => PreValidateBlockAnnounce::Process {
+					is_new_best: is_new_best || is_best,
+					announce,
+					who,
+				},
 				Ok(Validation::Failure) => {
 					debug!(
 						target: "sync",
@@ -1254,6 +1257,7 @@ impl<B: BlockT> ChainSync<B> {
 					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
 				}
 			}
+		})
 	}
 
 	/// Needs to be called with the result of [`ChainSync::pre_validate_block_announce`].
diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs
index 17f94d4a860f1..64985871d85e0 100644
--- a/client/network/test/src/sync.rs
+++ b/client/network/test/src/sync.rs
@@ -713,8 +713,8 @@ impl BlockAnnounceValidator<Block> for NewBestBlockAnnounceValidator {
 		&mut self,
 		_: &Header,
 		_: &[u8],
-	) -> Box<dyn Future<Output = Result<Validation, Box<dyn std::error::Error + Send>>> + Unpin + Send> {
-		Box::new(async { Ok(Validation::Success { is_new_best: true }) }.boxed())
+	) -> Pin<Box<dyn Future<Output = Result<Validation, Box<dyn std::error::Error + Send>>> + Send>> {
+		async { Ok(Validation::Success { is_new_best: true }) }.boxed()
 	}
 }
 
@@ -752,19 +752,17 @@ impl BlockAnnounceValidator<Block> for DeferredBlockAnnounceValidator {
 		&mut self,
 		_: &Header,
 		_: &[u8],
-	) -> Box<dyn Future<Output = Result<Validation, Box<dyn std::error::Error + Send>>> + Unpin + Send> {
-		Box::new(
-			async {
-				futures_timer::Delay::new(std::time::Duration::from_millis(500)).await;
-				Ok(Validation::Success { is_new_best: false })
-			}.boxed()
-		)
+	) -> Pin<Box<dyn Future<Output = Result<Validation, Box<dyn std::error::Error + Send>>> + Send>> {
+		async {
+			futures_timer::Delay::new(std::time::Duration::from_millis(500)).await;
+			Ok(Validation::Success { is_new_best: false })
+		}.boxed()
 	}
 }
 
 #[test]
 fn wait_until_deferred_block_announce_validation_is_ready() {
-	let _ = env_logger::try_init();
+	sp_tracing::try_init_simple();
 	log::trace!(target: "sync", "Test");
 	let mut net = TestNet::with_fork_choice(ForkChoiceStrategy::Custom(false));
 	net.add_full_peer_with_config(Default::default());

From a83703155ec675c2be99c313529e11321a14d39a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Fri, 18 Sep 2020 23:33:53 +0200
Subject: [PATCH 113/122] Rework the implementation and decrease the peer
 reputation on invalid block announcement

---
 client/network/src/protocol.rs                | 52 +++++++------
 client/network/src/protocol/sync.rs           | 76 ++++++++++++++-----
 .../consensus/common/src/block_validation.rs  |  5 ++
 3 files changed, 88 insertions(+), 45 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index ae96106473231..27d5706c10c07 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -122,6 +122,8 @@ mod rep {
 	pub const BAD_ROLE: Rep = Rep::new_fatal("Unsupported role");
 	/// Peer response data does not have requested bits.
 	pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response");
+	/// Peer send us a block announcement that failed at validation.
+	pub const BAD_BLOCK_ANNOUNCEMENT: Rep = Rep::new(-(1 << 12), "Bad block announcement");
 }
 
 struct Metrics {
@@ -241,10 +243,6 @@ pub struct Protocol<B: BlockT, H: ExHashT> {
 	metrics: Option<Metrics>,
 	/// The `PeerId`'s of all boot nodes.
 	boot_node_ids: Arc<HashSet<PeerId>>,
-	/// All the block announcement pre-validations that are currently active.
-	block_announce_pre_validation: FuturesUnordered<
-		Pin<Box<dyn Future<Output = sync::PreValidateBlockAnnounce<B::Header>> + Send>>
-	>,
 }
 
 #[derive(Default)]
@@ -458,7 +456,6 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 				None
 			},
 			boot_node_ids,
-			block_announce_pre_validation: FuturesUnordered::new(),
 		};
 
 		Ok((protocol, peerset_handle))
@@ -603,7 +600,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			GenericMessage::Status(_) =>
 				debug!(target: "sub-libp2p", "Received unexpected Status"),
 			GenericMessage::BlockAnnounce(announce) => {
-				self.pre_validate_block_announce(who.clone(), announce);
+				return self.push_block_announce_validation(who.clone(), announce)
 			},
 			GenericMessage::Transactions(m) =>
 				self.on_transactions(who, m),
@@ -1166,16 +1163,17 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		}
 	}
 
-	/// Pre-validate the given block announcement.
+	/// Push a block announce validation.
 	///
-	/// The pre-validation is an async operation that will be run asyncly
-	/// until it is finished. After finishing the pre-validation it will
-	/// call the [`Protocol::process_block_announce_pre_validation`].
-	fn pre_validate_block_announce(
+	/// It is required that [`ChainSync::poll_block_announce_validation`] is
+	/// called later to check for finished validations. The result of the validation
+	/// needs to be passed to [`Protocol::process_block_announce_validation_result`]
+	/// finish the processing.
+	fn push_block_announce_validation(
 		&mut self,
 		who: PeerId,
 		announce: BlockAnnounce<B::Header>,
-	) {
+	) -> CustomMessageOutcome<B> {
 		let hash = announce.header.hash();
 
 		if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) {
@@ -1187,16 +1185,19 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			message::BlockState::Normal => false,
 		};
 
-		let future = self.sync.pre_validate_block_announce(who, &hash, announce, is_best);
-		self.block_announce_pre_validation.push(Box::pin(future));
+		if let Some(res) = self.sync.push_block_announce_validation(who, &hash, announce, is_best) {
+			self.process_block_announce_validation_result(res)
+		} else {
+			CustomMessageOutcome::None
+		}
 	}
 
-	/// Process the result of the pre-validation of a block announcement.
-	fn process_block_announce_pre_validation(
+	/// Process the result of the block announce validation.
+	fn process_block_announce_validation_result(
 		&mut self,
-		pre_validation_result: sync::PreValidateBlockAnnounce<B::Header>,
+		validation_result: sync::BlockAnnounceResult<B::Header>,
 	) -> CustomMessageOutcome<B> {
-		let (header, is_best, who) = match self.sync.process_block_announce_pre_validation(pre_validation_result) {
+		let (header, is_best, who) = match validation_result {
 			sync::BlockAnnounceResult::Nothing { is_best, who, header } => {
 				self.update_peer_info(&who);
 
@@ -1216,6 +1217,10 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 				self.update_peer_info(&who);
 				(header, is_best, who)
 			}
+			sync::BlockAnnounceResult::Failure { who } => {
+				self.report_peer(who, rep::BAD_BLOCK_ANNOUNCEMENT);
+				return CustomMessageOutcome::None
+			}
 		};
 
 		let number = *header.number();
@@ -1675,12 +1680,11 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 					}
 					Some(Fallback::BlockAnnounce) => {
 						if let Ok(announce) = message::BlockAnnounce::decode(&mut message.as_ref()) {
-							self.pre_validate_block_announce(peer_id, announce);
+							self.push_block_announce_validation(peer_id, announce)
 						} else {
 							warn!(target: "sub-libp2p", "Failed to decode block announce");
+							CustomMessageOutcome::None
 						}
-
-						CustomMessageOutcome::None
 					}
 					None => {
 						debug!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name);
@@ -1690,9 +1694,9 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 		};
 
 		if let CustomMessageOutcome::None = outcome {
-			// Check if any pre-validations are ready to be processed.
-			while let Poll::Ready(Some(result)) = self.block_announce_pre_validation.poll_next_unpin(cx) {
-				match self.process_block_announce_pre_validation(result) {
+			// Check if there is any block announcement validation finished.
+			while let Poll::Ready(result) = self.sync.poll_block_announce_validation(cx) {
+				match self.process_block_announce_validation_result(result) {
 					CustomMessageOutcome::None => continue,
 					outcome => return Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome))
 				}
diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 027d63d662c0c..a93be4755ab99 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -49,8 +49,8 @@ use sp_runtime::{
 	traits::{Block as BlockT, Header, NumberFor, Zero, One, CheckedSub, SaturatedConversion, Hash, HashFor}
 };
 use sp_arithmetic::traits::Saturating;
-use std::{fmt, ops::Range, collections::{HashMap, HashSet, VecDeque}, sync::Arc};
-use futures::future::{Either as FEither, Future, ready};
+use std::{fmt, ops::Range, collections::{HashMap, HashSet, VecDeque}, sync::Arc, pin::Pin};
+use futures::{task::Poll, Future, stream::FuturesUnordered, FutureExt, StreamExt};
 
 mod blocks;
 mod extra_requests;
@@ -193,6 +193,10 @@ pub struct ChainSync<B: BlockT> {
 	max_parallel_downloads: u32,
 	/// Total number of downloaded blocks.
 	downloaded_blocks: usize,
+	/// All block announcement pre-validations that are currently being validated.
+	block_announce_pre_validation: FuturesUnordered<
+		Pin<Box<dyn Future<Output = PreValidateBlockAnnounce<B::Header>> + Send>>
+	>,
 }
 
 /// All the data we have about a Peer that we are trying to sync with
@@ -310,6 +314,13 @@ pub enum OnBlockData<B: BlockT> {
 /// Result of [`ChainSync::process_block_announce_pre_validation`].
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum BlockAnnounceResult<H> {
+	/// The announcement failed at validation.
+	///
+	/// The peer reputation should be decreased.
+	Failure {
+		/// Who sent the processed block announcement?
+		who: PeerId,
+	},
 	/// The announcement does not require further handling.
 	Nothing {
 		/// Who sent the processed block announcement?
@@ -330,9 +341,16 @@ pub enum BlockAnnounceResult<H> {
 	},
 }
 
-/// Result of [`ChainSync::pre_validate_block_announce`].
+/// Result of [`ChainSync::block_announce_pre_validation`].
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum PreValidateBlockAnnounce<H> {
+enum PreValidateBlockAnnounce<H> {
+	/// The announcement failed at validation.
+	///
+	/// The peer reputation should be decreased.
+	Failure {
+		/// Who sent the processed block announcement?
+		who: PeerId,
+	},
 	/// The announcement does not require further handling.
 	Nothing {
 		/// Who sent the processed block announcement?
@@ -416,6 +434,7 @@ impl<B: BlockT> ChainSync<B> {
 			block_announce_validator,
 			max_parallel_downloads,
 			downloaded_blocks: 0,
+			block_announce_pre_validation: Default::default(),
 		}
 	}
 
@@ -1195,22 +1214,19 @@ impl<B: BlockT> ChainSync<B> {
 		self.pending_requests.set_all();
 	}
 
-	/// Pre-validate the given block announcement.
+	/// Push a block announce validation.
 	///
-	/// The pre-validation will return a [`Future`] that resolves to a
-	/// [`PreValidateBlockAnnounce`]. This result needs to be feed into
-	/// [`ChainSync::process_block_announce_pre_validation`] to finish
-	/// the block announcement processing.
-	pub fn pre_validate_block_announce(
+	/// It is required that [`ChainSync::poll_block_announce_validation`] is called
+	/// to check for finished block announce validations.
+	///
+	/// Returns `Some(_)` if the block announce was validated immediately.
+	pub fn push_block_announce_validation(
 		&mut self,
 		who: PeerId,
 		hash: &B::Hash,
 		announce: BlockAnnounce<B::Header>,
 		is_best: bool,
-	) -> FEither<
-		impl Future<Output = PreValidateBlockAnnounce<B::Header>>,
-		impl Future<Output = PreValidateBlockAnnounce<B::Header>>
-	> {
+	) -> Option<BlockAnnounceResult<B::Header>> {
 		let header = &announce.header;
 		let number = *header.number();
 		debug!(
@@ -1228,7 +1244,7 @@ impl<B: BlockT> ChainSync<B> {
 				who,
 				hash,
 			);
-			return FEither::Left(ready(PreValidateBlockAnnounce::Nothing { is_best, who, announce }))
+			return Some(BlockAnnounceResult::Nothing { is_best, who, header: announce.header })
 		}
 
 		// Let external validator check the block announcement.
@@ -1236,7 +1252,7 @@ impl<B: BlockT> ChainSync<B> {
 		let future = self.block_announce_validator.validate(&header, assoc_data);
 		let hash = hash.clone();
 
-		FEither::Right(async move {
+		self.block_announce_pre_validation.push(async move {
 			match future.await {
 				Ok(Validation::Success { is_new_best }) => PreValidateBlockAnnounce::Process {
 					is_new_best: is_new_best || is_best,
@@ -1250,29 +1266,47 @@ impl<B: BlockT> ChainSync<B> {
 						hash,
 						who,
 					);
-					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
+					PreValidateBlockAnnounce::Failure { who }
 				}
 				Err(e) => {
 					error!(target: "sync", "💔 Block announcement validation errored: {}", e);
 					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
 				}
 			}
-		})
+		}.boxed());
+
+		None
 	}
 
-	/// Needs to be called with the result of [`ChainSync::pre_validate_block_announce`].
+	/// Poll block announce validation.
 	///
-	/// This will finish processing of the block announcement.
+	/// Block announce validations can be pushed by using
+	/// [`ChainSync::push_block_announce_validation`].
+	///
+	/// This should be polled until it returns [`Poll::Pending`].
 	///
 	/// If [`BlockAnnounceResult::ImportHeader`] is returned, then the caller MUST try to import passed
 	/// header (call `on_block_data`). The network request isn't sent in this case.
-	pub fn process_block_announce_pre_validation(
+	pub fn poll_block_announce_validation(
+		&mut self,
+		cx: &mut std::task::Context,
+	) -> Poll<BlockAnnounceResult<B::Header>> {
+		match self.block_announce_pre_validation.poll_next_unpin(cx) {
+			Poll::Ready(Some(res)) => Poll::Ready(self.finish_block_announce_validation(res)),
+			_ => Poll::Pending,
+		}
+	}
+
+	/// This will finish processing of the block announcement.
+	fn finish_block_announce_validation(
 		&mut self,
 		pre_validation_result: PreValidateBlockAnnounce<B::Header>,
 	) -> BlockAnnounceResult<B::Header> {
 		let (announce, is_best, who) = match pre_validation_result {
 			PreValidateBlockAnnounce::Nothing { is_best, who, announce } =>
 				return BlockAnnounceResult::Nothing { is_best, who, header: announce.header },
+			PreValidateBlockAnnounce::Failure { who } =>
+				return BlockAnnounceResult::Failure { who },
 			PreValidateBlockAnnounce::Process { announce, is_new_best, who } =>
 				(announce, is_new_best, who),
 		};
diff --git a/primitives/consensus/common/src/block_validation.rs b/primitives/consensus/common/src/block_validation.rs
index c3be7f3ffa8ba..c45d076caad5e 100644
--- a/primitives/consensus/common/src/block_validation.rs
+++ b/primitives/consensus/common/src/block_validation.rs
@@ -48,6 +48,11 @@ pub enum Validation {
 /// Type which checks incoming block announcements.
 pub trait BlockAnnounceValidator<B: Block> {
 	/// Validate the announced header and its associated data.
+	///
+	/// # Note
+	///
+	/// Returning [`Validation::Failure`] will led to a decrease of the
+	/// peers reputation as it send us invalid data.
 	fn validate(
 		&mut self,
 		header: &B::Header,

From a0f3789e00ecb0bec301bd638667a780e0e421ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Thu, 24 Sep 2020 14:05:38 +0200
Subject: [PATCH 114/122] Implement limits for block announce validation

---
 client/network/src/protocol/sync.rs | 130 +++++++++++++++++++++++++---
 1 file changed, 117 insertions(+), 13 deletions(-)

diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index a93be4755ab99..119a7070b1ef2 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -49,7 +49,10 @@ use sp_runtime::{
 	traits::{Block as BlockT, Header, NumberFor, Zero, One, CheckedSub, SaturatedConversion, Hash, HashFor}
 };
 use sp_arithmetic::traits::Saturating;
-use std::{fmt, ops::Range, collections::{HashMap, HashSet, VecDeque}, sync::Arc, pin::Pin};
+use std::{
+	fmt, ops::Range, collections::{HashMap, hash_map::Entry, HashSet, VecDeque},
+	sync::Arc, pin::Pin,
+};
 use futures::{task::Poll, Future, stream::FuturesUnordered, FutureExt, StreamExt};
 
 mod blocks;
@@ -64,6 +67,17 @@ const MAX_IMPORTING_BLOCKS: usize = 2048;
 /// Maximum blocks to download ahead of any gap.
 const MAX_DOWNLOAD_AHEAD: u32 = 2048;
 
+/// Maximum number of concurrent block announce validations.
+///
+/// If the queue reaches the maximum, we drop any new block
+/// announcements.
+const MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS: usize = 256;
+
+/// Maximum number of concurrent block announce validations per peer.
+///
+/// See [`MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS`] for more information.
+const MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER: usize = 4;
+
 /// We use a heuristic that with a high likelihood, by the time
 /// `MAJOR_SYNC_BLOCKS` have been imported we'll be on the same
 /// chain as (or at least closer to) the peer so we want to delay
@@ -193,10 +207,12 @@ pub struct ChainSync<B: BlockT> {
 	max_parallel_downloads: u32,
 	/// Total number of downloaded blocks.
 	downloaded_blocks: usize,
-	/// All block announcement pre-validations that are currently being validated.
-	block_announce_pre_validation: FuturesUnordered<
+	/// All block announcement that are currently being validated.
+	block_announce_validation: FuturesUnordered<
 		Pin<Box<dyn Future<Output = PreValidateBlockAnnounce<B::Header>> + Send>>
 	>,
+	/// Stats per peer about the number of concurrent block announce validations.
+	block_announce_validation_per_peer_stats: HashMap<PeerId, usize>,
 }
 
 /// All the data we have about a Peer that we are trying to sync with
@@ -341,7 +357,7 @@ pub enum BlockAnnounceResult<H> {
 	},
 }
 
-/// Result of [`ChainSync::block_announce_pre_validation`].
+/// Result of [`ChainSync::block_announce_validation`].
 #[derive(Debug, Clone, PartialEq, Eq)]
 enum PreValidateBlockAnnounce<H> {
 	/// The announcement failed at validation.
@@ -400,6 +416,16 @@ pub enum OnBlockFinalityProof<B: BlockT> {
 	}
 }
 
+/// Result of [`ChainSync::has_slot_for_block_announce_validation`].
+enum HasSlotForBlockAnnounceValidation {
+	/// Yes, there is a slot for the block announce validation.
+	Yes,
+	/// We reached the total maximum number of validation slots.
+	TotalMaximumSlotsReached,
+	/// We reached the maximum number of validation slots for the given peer.
+	MaximumPeerSlotsReached,
+}
+
 impl<B: BlockT> ChainSync<B> {
 	/// Create a new instance.
 	pub fn new(
@@ -434,7 +460,8 @@ impl<B: BlockT> ChainSync<B> {
 			block_announce_validator,
 			max_parallel_downloads,
 			downloaded_blocks: 0,
-			block_announce_pre_validation: Default::default(),
+			block_announce_validation: Default::default(),
+			block_announce_validation_per_peer_stats: Default::default(),
 		}
 	}
 
@@ -1214,6 +1241,33 @@ impl<B: BlockT> ChainSync<B> {
 		self.pending_requests.set_all();
 	}
 
+	/// Checks if there is a slot for a block announce validation.
+	///
+	/// The total number and the number per peer of concurrent block announce validations
+	/// is caped.
+	///
+	/// It will return `true` when there is still a slot for a block announce validation.
+	fn has_slot_for_block_announce_validation(&mut self, peer: &PeerId) -> HasSlotForBlockAnnounceValidation {
+		if self.block_announce_validation.len() >= MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS {
+			return HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached
+		}
+
+		match self.block_announce_validation_per_peer_stats.entry(peer.clone()) {
+			Entry::Vacant(entry) => {
+				entry.insert(1);
+				HasSlotForBlockAnnounceValidation::Yes
+			},
+			Entry::Occupied(mut entry) => {
+				if *entry.get() < MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER {
+					*entry.get_mut() += 1;
+					HasSlotForBlockAnnounceValidation::Yes
+				} else {
+					HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached
+				}
+			},
+		}
+	}
+
 	/// Push a block announce validation.
 	///
 	/// It is required that [`ChainSync::poll_block_announce_validation`] is called
@@ -1247,12 +1301,37 @@ impl<B: BlockT> ChainSync<B> {
 			return Some(BlockAnnounceResult::Nothing { is_best, who, header: announce.header })
 		}
 
+		// Check if there is a slot for this block announce validation.
+		match self.has_slot_for_block_announce_validation(&who) {
+			HasSlotForBlockAnnounceValidation::Yes => {},
+			HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached => {
+				warn!(
+					target: "sync",
+					"💔 Ignored block (#{} -- {}) announcement from {} because all validation slots are occupied.",
+					number,
+					hash,
+					who,
+				);
+				return Some(BlockAnnounceResult::Nothing { is_best, who, header: announce.header })
+			}
+			HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached => {
+				warn!(
+					target: "sync",
+					"💔 Ignored block (#{} -- {}) announcement from {} because all validation slots for this peer are occupied.",
+					number,
+					hash,
+					who,
+				);
+				return Some(BlockAnnounceResult::Nothing { is_best, who, header: announce.header })
+			}
+		}
+
 		// Let external validator check the block announcement.
 		let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice());
 		let future = self.block_announce_validator.validate(&header, assoc_data);
 		let hash = hash.clone();
 
-		self.block_announce_pre_validation.push(async move {
+		self.block_announce_validation.push(async move {
 			match future.await {
 				Ok(Validation::Success { is_new_best }) => PreValidateBlockAnnounce::Process {
 					is_new_best: is_new_best || is_best,
@@ -1291,24 +1370,49 @@ impl<B: BlockT> ChainSync<B> {
 		&mut self,
 		cx: &mut std::task::Context,
 	) -> Poll<BlockAnnounceResult<B::Header>> {
-		match self.block_announce_pre_validation.poll_next_unpin(cx) {
+		match self.block_announce_validation.poll_next_unpin(cx) {
 			Poll::Ready(Some(res)) => Poll::Ready(self.finish_block_announce_validation(res)),
 			_ => Poll::Pending,
 		}
 	}
 
+	/// Should be called when a block announce validation was finished, to update the stats
+	/// of the given peer.
+	fn peer_block_announce_validation_finished(&mut self, peer: &PeerId) {
+		match self.block_announce_validation_per_peer_stats.entry(peer.clone()) {
+			Entry::Vacant(_) => {
+				error!(
+					target: "sync",
+					"💔 Block announcement validation from peer {} finished for that no slot was allocated!",
+					peer,
+				);
+			},
+			Entry::Occupied(mut entry) => {
+				if entry.get_mut().saturating_sub(1) == 0 {
+					entry.remove();
+				}
+			}
+		}
+	}
+
 	/// This will finish processing of the block announcement.
 	fn finish_block_announce_validation(
 		&mut self,
 		pre_validation_result: PreValidateBlockAnnounce<B::Header>,
 	) -> BlockAnnounceResult<B::Header> {
 		let (announce, is_best, who) = match pre_validation_result {
-			PreValidateBlockAnnounce::Nothing { is_best, who, announce } =>
-				return BlockAnnounceResult::Nothing { is_best, who, header: announce.header },
-			PreValidateBlockAnnounce::Failure { who } =>
-				return BlockAnnounceResult::Failure { who },
-			PreValidateBlockAnnounce::Process { announce, is_new_best, who } =>
-				(announce, is_new_best, who),
+			PreValidateBlockAnnounce::Nothing { is_best, who, announce } => {
+				self.peer_block_announce_validation_finished(&who);
+				return BlockAnnounceResult::Nothing { is_best, who, header: announce.header }
+			},
+			PreValidateBlockAnnounce::Failure { who } => {
+				self.peer_block_announce_validation_finished(&who);
+				return BlockAnnounceResult::Failure { who }
+			},
+			PreValidateBlockAnnounce::Process { announce, is_new_best, who } => {
+				self.peer_block_announce_validation_finished(&who);
+				(announce, is_new_best, who)
+			},
 		};
 
 		let header = announce.header;

From 3f977abe57ebecb858684907f9676d1dcdec23b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Fri, 25 Sep 2020 22:29:36 +0200
Subject: [PATCH 115/122] Remove accidentally added file

---
 frame/generic-asset/src/lib.rs | 1369 --------------------------------
 1 file changed, 1369 deletions(-)
 delete mode 100644 frame/generic-asset/src/lib.rs

diff --git a/frame/generic-asset/src/lib.rs b/frame/generic-asset/src/lib.rs
deleted file mode 100644
index 6c3683312d0a8..0000000000000
--- a/frame/generic-asset/src/lib.rs
+++ /dev/null
@@ -1,1369 +0,0 @@
-// Copyright 2019-2020
-//     by  Centrality Investments Ltd.
-//     and Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
-
-//! # Generic Asset Module
-//!
-//! The Generic Asset module provides functionality for handling accounts and asset balances.
-//!
-//! ## Overview
-//!
-//! The Generic Asset module provides functions for:
-//!
-//! - Creating a new kind of asset.
-//! - Setting permissions of an asset.
-//! - Getting and setting free balances.
-//! - Retrieving total, reserved and unreserved balances.
-//! - Repatriating a reserved balance to a beneficiary account.
-//! - Transferring a balance between accounts (when not reserved).
-//! - Slashing an account balance.
-//! - Managing total issuance.
-//! - Setting and managing locks.
-//!
-//! ### Terminology
-//!
-//! - **Staking Asset:** The asset for staking, to participate as Validators in the network.
-//! - **Spending Asset:** The asset for payment, such as paying transfer fees, gas fees, etc.
-//! - **Permissions:** A set of rules for a kind of asset, defining the allowed operations to the asset, and which
-//! accounts are allowed to possess it.
-//! - **Total Issuance:** The total number of units in existence in a system.
-//! - **Free Balance:** The portion of a balance that is not reserved. The free balance is the only balance that matters
-//! for most operations. When this balance falls below the existential deposit, most functionality of the account is
-//! removed. When both it and the reserved balance are deleted, then the account is said to be dead.
-//! - **Reserved Balance:** Reserved balance still belongs to the account holder, but is suspended. Reserved balance
-//! can still be slashed, but only after all the free balance has been slashed. If the reserved balance falls below the
-//! existential deposit then it and any related functionality will be deleted. When both it and the free balance are
-//! deleted, then the account is said to be dead.
-//! - **Imbalance:** A condition when some assets were credited or debited without equal and opposite accounting
-//! (i.e. a difference between total issuance and account balances). Functions that result in an imbalance will
-//! return an object of the `Imbalance` trait that can be managed within your runtime logic. (If an imbalance is
-//! simply dropped, it should automatically maintain any book-keeping such as total issuance.)
-//! - **Lock:** A freeze on a specified amount of an account's free balance until a specified block number. Multiple
-//! locks always operate over the same funds, so they "overlay" rather than "stack".
-//!
-//! ### Implementations
-//!
-//! The Generic Asset module provides `AssetCurrency`, which implements the following traits. If these traits provide
-//! the functionality that you need, you can avoid coupling with the Generic Asset module.
-//!
-//! - `Currency`: Functions for dealing with a fungible assets system.
-//! - `ReservableCurrency`: Functions for dealing with assets that can be reserved from an account.
-//! - `LockableCurrency`: Functions for dealing with accounts that allow liquidity restrictions.
-//! - `Imbalance`: Functions for handling imbalances between total issuance in the system and account balances.
-//! Must be used when a function creates new assets (e.g. a reward) or destroys some assets (e.g. a system fee).
-//!
-//! The Generic Asset module provides two types of `AssetCurrency` as follows.
-//!
-//! - `StakingAssetCurrency`: Currency for staking.
-//! - `SpendingAssetCurrency`: Currency for payments such as transfer fee, gas fee.
-//!
-//! ## Interface
-//!
-//! ### Dispatchable Functions
-//!
-//! - `create`: Create a new kind of asset.
-//! - `transfer`: Transfer some liquid free balance to another account.
-//! - `update_permission`: Updates permission for a given `asset_id` and an account. The origin of this call
-//! must have update permissions.
-//! - `mint`: Mint an asset, increases its total issuance. The origin of this call must have mint permissions.
-//! - `burn`: Burn an asset, decreases its total issuance. The origin of this call must have burn permissions.
-//! - `create_reserved`: Create a new kind of reserved asset. The origin of this call must be root.
-//!
-//! ### Public Functions
-//!
-//! - `total_balance`: Get an account's total balance of an asset kind.
-//! - `free_balance`: Get an account's free balance of an asset kind.
-//! - `reserved_balance`: Get an account's reserved balance of an asset kind.
-//! - `create_asset`: Creates an asset.
-//! - `make_transfer`: Transfer some liquid free balance from one account to another.
-//! This will not emit the `Transferred` event.
-//! - `make_transfer_with_event`: Transfer some liquid free balance from one account to another.
-//! This will emit the `Transferred` event.
-//! - `reserve`: Moves an amount from free balance to reserved balance.
-//! - `unreserve`: Move up to an amount from reserved balance to free balance. This function cannot fail.
-//! - `mint_free`: Mint to an account's free balance.
-//! - `burn_free`: Burn an account's free balance.
-//! - `slash`: Deduct up to an amount from the combined balance of `who`, preferring to deduct from the
-//!	free balance. This function cannot fail.
-//! - `slash_reserved`: Deduct up to an amount from reserved balance of an account. This function cannot fail.
-//! - `repatriate_reserved`: Move up to an amount from reserved balance of an account to free balance of another
-//! account.
-//! - `check_permission`: Check permission to perform burn, mint or update.
-//! - `ensure_can_withdraw`: Check if the account is able to make a withdrawal of the given amount
-//!	for the given reason.
-//!
-//! ### Usage
-//!
-//! The following examples show how to use the Generic Asset Pallet in your custom pallet.
-//!
-//! ### Examples from the FRAME pallet
-//!
-//! The Fees Pallet uses the `Currency` trait to handle fee charge/refund, and its types inherit from `Currency`:
-//!
-//! ```
-//! use frame_support::{
-//! 	dispatch,
-//! 	traits::{Currency, ExistenceRequirement, WithdrawReason},
-//! };
-//! # pub trait Trait: frame_system::Trait {
-//! # 	type Currency: Currency<Self::AccountId>;
-//! # }
-//! type AssetOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
-//!
-//! fn charge_fee<T: Trait>(transactor: &T::AccountId, amount: AssetOf<T>) -> dispatch::DispatchResult {
-//! 	// ...
-//! 	T::Currency::withdraw(
-//! 		transactor,
-//! 		amount,
-//! 		WithdrawReason::TransactionPayment.into(),
-//! 		ExistenceRequirement::KeepAlive,
-//! 	)?;
-//! 	// ...
-//! 	Ok(())
-//! }
-//!
-//! fn refund_fee<T: Trait>(transactor: &T::AccountId, amount: AssetOf<T>) -> dispatch::DispatchResult {
-//! 	// ...
-//! 	T::Currency::deposit_into_existing(transactor, amount)?;
-//! 	// ...
-//! 	Ok(())
-//! }
-//!
-//! # fn main() {}
-//! ```
-//!
-//! ## Genesis config
-//!
-//! The Generic Asset Pallet depends on the [`GenesisConfig`](./struct.GenesisConfig.html).
-
-#![cfg_attr(not(feature = "std"), no_std)]
-
-use codec::{Decode, Encode, HasCompact, Input, Output, Error as CodecError};
-
-use sp_runtime::{RuntimeDebug, DispatchResult, DispatchError};
-use sp_runtime::traits::{
-	CheckedAdd, CheckedSub, MaybeSerializeDeserialize, Member, One, Saturating, AtLeast32Bit,
-	Zero, Bounded, AtLeast32BitUnsigned
-};
-
-use sp_std::prelude::*;
-use sp_std::{cmp, result, fmt::Debug};
-use frame_support::{
-	decl_event, decl_module, decl_storage, ensure, decl_error,
-	traits::{
-		Currency, ExistenceRequirement, Imbalance, LockIdentifier, LockableCurrency,
-		ReservableCurrency, SignedImbalance, WithdrawReason, WithdrawReasons, TryDrop,
-		BalanceStatus,
-	},
-	Parameter, StorageMap,
-};
-use frame_system::{ensure_signed, ensure_root};
-
-mod mock;
-mod tests;
-
-pub use self::imbalances::{NegativeImbalance, PositiveImbalance};
-
-pub trait Trait: frame_system::Trait {
-	type Balance: Parameter + Member + AtLeast32BitUnsigned + Default + Copy + Debug +
-		MaybeSerializeDeserialize;
-	type AssetId: Parameter + Member + AtLeast32Bit + Default + Copy;
-	type Event: From<Event<Self>> + Into<<Self as frame_system::Trait>::Event>;
-}
-
-pub trait Subtrait: frame_system::Trait {
-	type Balance: Parameter + Member + AtLeast32BitUnsigned + Default + Copy + Debug +
-		MaybeSerializeDeserialize;
-	type AssetId: Parameter + Member + AtLeast32Bit + Default + Copy;
-}
-
-impl<T: Trait> Subtrait for T {
-	type Balance = T::Balance;
-	type AssetId = T::AssetId;
-}
-
-/// Asset creation options.
-#[derive(Clone, Encode, Decode, PartialEq, Eq, RuntimeDebug)]
-pub struct AssetOptions<Balance: HasCompact, AccountId> {
-	/// Initial issuance of this asset. All deposit to the creator of the asset.
-	#[codec(compact)]
-	pub initial_issuance: Balance,
-	/// Which accounts are allowed to possess this asset.
-	pub permissions: PermissionLatest<AccountId>,
-}
-
-/// Owner of an asset.
-#[derive(Clone, Encode, Decode, PartialEq, Eq, RuntimeDebug)]
-pub enum Owner<AccountId> {
-	/// No owner.
-	None,
-	/// Owned by an AccountId
-	Address(AccountId),
-}
-
-impl<AccountId> Default for Owner<AccountId> {
-	fn default() -> Self {
-		Owner::None
-	}
-}
-
-/// Asset permissions
-#[derive(Clone, Encode, Decode, PartialEq, Eq, RuntimeDebug)]
-pub struct PermissionsV1<AccountId> {
-	/// Who have permission to update asset permission
-	pub update: Owner<AccountId>,
-	/// Who have permission to mint new asset
-	pub mint: Owner<AccountId>,
-	/// Who have permission to burn asset
-	pub burn: Owner<AccountId>,
-}
-
-#[derive(Clone, Encode, Decode, PartialEq, Eq, RuntimeDebug)]
-#[repr(u8)]
-enum PermissionVersionNumber {
-	V1 = 0,
-}
-
-/// Versioned asset permission
-#[derive(Clone, PartialEq, Eq, RuntimeDebug)]
-pub enum PermissionVersions<AccountId> {
-	V1(PermissionsV1<AccountId>),
-}
-
-/// Asset permission types
-pub enum PermissionType {
-	/// Permission to burn asset permission
-	Burn,
-	/// Permission to mint new asset
-	Mint,
-	/// Permission to update asset
-	Update,
-}
-
-/// Alias to latest asset permissions
-pub type PermissionLatest<AccountId> = PermissionsV1<AccountId>;
-
-impl<AccountId> Default for PermissionVersions<AccountId> {
-	fn default() -> Self {
-		PermissionVersions::V1(Default::default())
-	}
-}
-
-impl<AccountId: Encode> Encode for PermissionVersions<AccountId> {
-	fn encode_to<T: Output>(&self, dest: &mut T) {
-		match self {
-			PermissionVersions::V1(payload) => {
-				dest.push(&PermissionVersionNumber::V1);
-				dest.push(payload);
-			},
-		}
-	}
-}
-
-impl<AccountId: Encode> codec::EncodeLike for PermissionVersions<AccountId> {}
-
-impl<AccountId: Decode> Decode for PermissionVersions<AccountId> {
-	fn decode<I: Input>(input: &mut I) -> core::result::Result<Self, CodecError> {
-		let version = PermissionVersionNumber::decode(input)?;
-		Ok(
-			match version {
-				PermissionVersionNumber::V1 => PermissionVersions::V1(Decode::decode(input)?)
-			}
-		)
-	}
-}
-
-impl<AccountId> Default for PermissionsV1<AccountId> {
-	fn default() -> Self {
-		PermissionsV1 {
-			update: Owner::None,
-			mint: Owner::None,
-			burn: Owner::None,
-		}
-	}
-}
-
-impl<AccountId> Into<PermissionLatest<AccountId>> for PermissionVersions<AccountId> {
-	fn into(self) -> PermissionLatest<AccountId> {
-		match self {
-			PermissionVersions::V1(v1) => v1,
-		}
-	}
-}
-
-/// Converts the latest permission to other version.
-impl<AccountId> Into<PermissionVersions<AccountId>> for PermissionLatest<AccountId> {
-	fn into(self) -> PermissionVersions<AccountId> {
-		PermissionVersions::V1(self)
-	}
-}
-
-decl_error! {
-	/// Error for the generic-asset module.
-	pub enum Error for Module<T: Trait> {
-		/// No new assets id available.
-		NoIdAvailable,
-		/// Cannot transfer zero amount.
-		ZeroAmount,
-		/// The origin does not have enough permission to update permissions.
-		NoUpdatePermission,
-		/// The origin does not have permission to mint an asset.
-		NoMintPermission,
-		/// The origin does not have permission to burn an asset.
-		NoBurnPermission,
-		/// Total issuance got overflowed after minting.
-		TotalMintingOverflow,
-		/// Free balance got overflowed after minting.
-		FreeMintingOverflow,
-		/// Total issuance got underflowed after burning.
-		TotalBurningUnderflow,
-		/// Free balance got underflowed after burning.
-		FreeBurningUnderflow,
-		/// Asset id is already taken.
-		IdAlreadyTaken,
-		/// Asset id not available.
-		IdUnavailable,
-		/// The balance is too low to send amount.
-		InsufficientBalance,
-		/// The account liquidity restrictions prevent withdrawal.
-		LiquidityRestrictions,
-	}
-}
-
-decl_module! {
-	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
-		type Error = Error<T>;
-
-		fn deposit_event() = default;
-
-		/// Create a new kind of asset.
-		#[weight = 0]
-		fn create(origin, options: AssetOptions<T::Balance, T::AccountId>) -> DispatchResult {
-			let origin = ensure_signed(origin)?;
-			Self::create_asset(None, Some(origin), options)
-		}
-
-		/// Transfer some liquid free balance to another account.
-		#[weight = 0]
-		pub fn transfer(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, #[compact] amount: T::Balance) {
-			let origin = ensure_signed(origin)?;
-			ensure!(!amount.is_zero(), Error::<T>::ZeroAmount);
-			Self::make_transfer_with_event(&asset_id, &origin, &to, amount)?;
-		}
-
-		/// Updates permission for a given `asset_id` and an account.
-		///
-		/// The `origin` must have `update` permission.
-		#[weight = 0]
-		fn update_permission(
-			origin,
-			#[compact] asset_id: T::AssetId,
-			new_permission: PermissionLatest<T::AccountId>
-		) -> DispatchResult {
-			let origin = ensure_signed(origin)?;
-
-			let permissions: PermissionVersions<T::AccountId> = new_permission.into();
-
-			if Self::check_permission(&asset_id, &origin, &PermissionType::Update) {
-				<Permissions<T>>::insert(asset_id, &permissions);
-
-				Self::deposit_event(RawEvent::PermissionUpdated(asset_id, permissions.into()));
-
-				Ok(())
-			} else {
-				Err(Error::<T>::NoUpdatePermission)?
-			}
-		}
-
-		/// Mints an asset, increases its total issuance.
-		/// The origin must have `mint` permissions.
-		#[weight = 0]
-		fn mint(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, amount: T::Balance) -> DispatchResult {
-			let who = ensure_signed(origin)?;
-			Self::mint_free(&asset_id, &who, &to, &amount)?;
-			Self::deposit_event(RawEvent::Minted(asset_id, to, amount));
-			Ok(())
-		}
-
-		/// Burns an asset, decreases its total issuance.
-		/// The `origin` must have `burn` permissions.
-		#[weight = 0]
-		fn burn(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, amount: T::Balance) -> DispatchResult {
-			let who = ensure_signed(origin)?;
-			Self::burn_free(&asset_id, &who, &to, &amount)?;
-			Self::deposit_event(RawEvent::Burned(asset_id, to, amount));
-			Ok(())
-		}
-
-		/// Can be used to create reserved tokens.
-		/// Requires Root call.
-		#[weight = 0]
-		fn create_reserved(
-			origin,
-			asset_id: T::AssetId,
-			options: AssetOptions<T::Balance, T::AccountId>
-		) -> DispatchResult {
-			ensure_root(origin)?;
-			Self::create_asset(Some(asset_id), None, options)
-		}
-	}
-}
-
-#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)]
-pub struct BalanceLock<Balance> {
-	pub id: LockIdentifier,
-	pub amount: Balance,
-	pub reasons: WithdrawReasons,
-}
-
-decl_storage! {
-	trait Store for Module<T: Trait> as GenericAsset {
-		/// Total issuance of a given asset.
-		///
-		/// TWOX-NOTE: `AssetId` is trusted.
-		pub TotalIssuance get(fn total_issuance) build(|config: &GenesisConfig<T>| {
-			let issuance = config.initial_balance * (config.endowed_accounts.len() as u32).into();
-			config.assets.iter().map(|id| (id.clone(), issuance)).collect::<Vec<_>>()
-		}): map hasher(twox_64_concat) T::AssetId => T::Balance;
-
-		/// The free balance of a given asset under an account.
-		///
-		/// TWOX-NOTE: `AssetId` is trusted.
-		pub FreeBalance:
-			double_map hasher(twox_64_concat) T::AssetId, hasher(blake2_128_concat) T::AccountId => T::Balance;
-
-		/// The reserved balance of a given asset under an account.
-		///
-		/// TWOX-NOTE: `AssetId` is trusted.
-		pub ReservedBalance:
-			double_map hasher(twox_64_concat) T::AssetId, hasher(blake2_128_concat) T::AccountId => T::Balance;
-
-		/// Next available ID for user-created asset.
-		pub NextAssetId get(fn next_asset_id) config(): T::AssetId;
-
-		/// Permission options for a given asset.
-		///
-		/// TWOX-NOTE: `AssetId` is trusted.
-		pub Permissions get(fn get_permission):
-			map hasher(twox_64_concat) T::AssetId => PermissionVersions<T::AccountId>;
-
-		/// Any liquidity locks on some account balances.
-		pub Locks get(fn locks):
-			map hasher(blake2_128_concat) T::AccountId => Vec<BalanceLock<T::Balance>>;
-
-		/// The identity of the asset which is the one that is designated for the chain's staking system.
-		pub StakingAssetId get(fn staking_asset_id) config(): T::AssetId;
-
-		/// The identity of the asset which is the one that is designated for paying the chain's transaction fee.
-		pub SpendingAssetId get(fn spending_asset_id) config(): T::AssetId;
-	}
-	add_extra_genesis {
-		config(assets): Vec<T::AssetId>;
-		config(initial_balance): T::Balance;
-		config(endowed_accounts): Vec<T::AccountId>;
-
-		build(|config: &GenesisConfig<T>| {
-			config.assets.iter().for_each(|asset_id| {
-				config.endowed_accounts.iter().for_each(|account_id| {
-					<FreeBalance<T>>::insert(asset_id, account_id, &config.initial_balance);
-				});
-			});
-		});
-	}
-}
-
-decl_event!(
-	pub enum Event<T> where
-		<T as frame_system::Trait>::AccountId,
-		<T as Trait>::Balance,
-		<T as Trait>::AssetId,
-		AssetOptions = AssetOptions<<T as Trait>::Balance, <T as frame_system::Trait>::AccountId>
-	{
-		/// Asset created. \[asset_id, creator, asset_options\]
-		Created(AssetId, AccountId, AssetOptions),
-		/// Asset transfer succeeded. \[asset_id, from, to, amount\]
-		Transferred(AssetId, AccountId, AccountId, Balance),
-		/// Asset permission updated. \[asset_id, new_permissions\]
-		PermissionUpdated(AssetId, PermissionLatest<AccountId>),
-		/// New asset minted. \[asset_id, account, amount\]
-		Minted(AssetId, AccountId, Balance),
-		/// Asset burned. \[asset_id, account, amount\]
-		Burned(AssetId, AccountId, Balance),
-	}
-);
-
-impl<T: Trait> Module<T> {
-	// PUBLIC IMMUTABLES
-
-	/// Get an account's total balance of an asset kind.
-	pub fn total_balance(asset_id: &T::AssetId, who: &T::AccountId) -> T::Balance {
-		Self::free_balance(asset_id, who) + Self::reserved_balance(asset_id, who)
-	}
-
-	/// Get an account's free balance of an asset kind.
-	pub fn free_balance(asset_id: &T::AssetId, who: &T::AccountId) -> T::Balance {
-		<FreeBalance<T>>::get(asset_id, who)
-	}
-
-	/// Get an account's reserved balance of an asset kind.
-	pub fn reserved_balance(asset_id: &T::AssetId, who: &T::AccountId) -> T::Balance {
-		<ReservedBalance<T>>::get(asset_id, who)
-	}
-
-	/// Mint to an account's free balance, without event
-	pub fn mint_free(
-		asset_id: &T::AssetId,
-		who: &T::AccountId,
-		to: &T::AccountId,
-		amount: &T::Balance,
-	) -> DispatchResult {
-		if Self::check_permission(asset_id, who, &PermissionType::Mint) {
-			let original_free_balance = Self::free_balance(&asset_id, &to);
-			let current_total_issuance = <TotalIssuance<T>>::get(asset_id);
-			let new_total_issuance = current_total_issuance.checked_add(&amount)
-				.ok_or(Error::<T>::TotalMintingOverflow)?;
-			let value = original_free_balance.checked_add(&amount)
-				.ok_or(Error::<T>::FreeMintingOverflow)?;
-
-			<TotalIssuance<T>>::insert(asset_id, new_total_issuance);
-			Self::set_free_balance(&asset_id, &to, value);
-			Ok(())
-		} else {
-			Err(Error::<T>::NoMintPermission)?
-		}
-	}
-
-	/// Burn an account's free balance, without event
-	pub fn burn_free(
-		asset_id: &T::AssetId,
-		who: &T::AccountId,
-		to: &T::AccountId,
-		amount: &T::Balance,
-	) -> DispatchResult {
-		if Self::check_permission(asset_id, who, &PermissionType::Burn) {
-			let original_free_balance = Self::free_balance(asset_id, to);
-
-			let current_total_issuance = <TotalIssuance<T>>::get(asset_id);
-			let new_total_issuance = current_total_issuance.checked_sub(amount)
-				.ok_or(Error::<T>::TotalBurningUnderflow)?;
-			let value = original_free_balance.checked_sub(amount)
-				.ok_or(Error::<T>::FreeBurningUnderflow)?;
-
-			<TotalIssuance<T>>::insert(asset_id, new_total_issuance);
-			Self::set_free_balance(asset_id, to, value);
-			Ok(())
-		} else {
-			Err(Error::<T>::NoBurnPermission)?
-		}
-	}
-
-	/// Creates an asset.
-	///
-	/// # Arguments
-	/// * `asset_id`: An ID of a reserved asset.
-	/// If not provided, a user-generated asset will be created with the next available ID.
-	/// * `from_account`: The initiator account of this call
-	/// * `asset_options`: Asset creation options.
-	///
-	pub fn create_asset(
-		asset_id: Option<T::AssetId>,
-		from_account: Option<T::AccountId>,
-		options: AssetOptions<T::Balance, T::AccountId>,
-	) -> DispatchResult {
-		let asset_id = if let Some(asset_id) = asset_id {
-			ensure!(!<TotalIssuance<T>>::contains_key(&asset_id), Error::<T>::IdAlreadyTaken);
-			ensure!(asset_id < Self::next_asset_id(), Error::<T>::IdUnavailable);
-			asset_id
-		} else {
-			let asset_id = Self::next_asset_id();
-			let next_id = asset_id
-				.checked_add(&One::one())
-				.ok_or(Error::<T>::NoIdAvailable)?;
-			<NextAssetId<T>>::put(next_id);
-			asset_id
-		};
-
-		let account_id = from_account.unwrap_or_default();
-		let permissions: PermissionVersions<T::AccountId> = options.permissions.clone().into();
-
-		<TotalIssuance<T>>::insert(asset_id, &options.initial_issuance);
-		<FreeBalance<T>>::insert(&asset_id, &account_id, &options.initial_issuance);
-		<Permissions<T>>::insert(&asset_id, permissions);
-
-		Self::deposit_event(RawEvent::Created(asset_id, account_id, options));
-
-		Ok(())
-	}
-
-	/// Transfer some liquid free balance from one account to another.
-	/// This will not emit the `Transferred` event.
-	pub fn make_transfer(
-		asset_id: &T::AssetId,
-		from: &T::AccountId,
-		to: &T::AccountId,
-		amount: T::Balance
-	) -> DispatchResult {
-		let new_balance = Self::free_balance(asset_id, from)
-			.checked_sub(&amount)
-			.ok_or(Error::<T>::InsufficientBalance)?;
-		Self::ensure_can_withdraw(asset_id, from, amount, WithdrawReason::Transfer.into(), new_balance)?;
-
-		if from != to {
-			<FreeBalance<T>>::mutate(asset_id, from, |balance| *balance -= amount);
-			<FreeBalance<T>>::mutate(asset_id, to, |balance| *balance += amount);
-		}
-
-		Ok(())
-	}
-
-	/// Transfer some liquid free balance from one account to another.
-	/// This will emit the `Transferred` event.
-	pub fn make_transfer_with_event(
-		asset_id: &T::AssetId,
-		from: &T::AccountId,
-		to: &T::AccountId,
-		amount: T::Balance,
-	) -> DispatchResult {
-		Self::make_transfer(asset_id, from, to, amount)?;
-
-		if from != to {
-			Self::deposit_event(RawEvent::Transferred(*asset_id, from.clone(), to.clone(), amount));
-		}
-
-		Ok(())
-	}
-
-	/// Move `amount` from free balance to reserved balance.
-	///
-	/// If the free balance is lower than `amount`, then no funds will be moved and an `Err` will
-	/// be returned. This is different behavior than `unreserve`.
-	pub fn reserve(asset_id: &T::AssetId, who: &T::AccountId, amount: T::Balance)
-		-> DispatchResult
-	{
-		// Do we need to consider that this is an atomic transaction?
-		let original_reserve_balance = Self::reserved_balance(asset_id, who);
-		let original_free_balance = Self::free_balance(asset_id, who);
-		if original_free_balance < amount {
-			Err(Error::<T>::InsufficientBalance)?
-		}
-		let new_reserve_balance = original_reserve_balance + amount;
-		Self::set_reserved_balance(asset_id, who, new_reserve_balance);
-		let new_free_balance = original_free_balance - amount;
-		Self::set_free_balance(asset_id, who, new_free_balance);
-		Ok(())
-	}
-
-	/// Moves up to `amount` from reserved balance to free balance. This function cannot fail.
-	///
-	/// As many assets up to `amount` will be moved as possible. If the reserve balance of `who`
-	/// is less than `amount`, then the remaining amount will be returned.
-	/// NOTE: This is different behavior than `reserve`.
-	pub fn unreserve(asset_id: &T::AssetId, who: &T::AccountId, amount: T::Balance) -> T::Balance {
-		let b = Self::reserved_balance(asset_id, who);
-		let actual = sp_std::cmp::min(b, amount);
-		let original_free_balance = Self::free_balance(asset_id, who);
-		let new_free_balance = original_free_balance + actual;
-		Self::set_free_balance(asset_id, who, new_free_balance);
-		Self::set_reserved_balance(asset_id, who, b - actual);
-		amount - actual
-	}
-
-	/// Deduct up to `amount` from the combined balance of `who`, preferring to deduct from the
-	/// free balance. This function cannot fail.
-	///
-	/// As much funds up to `amount` will be deducted as possible. If this is less than `amount`
-	/// then `Some(remaining)` will be returned. Full completion is given by `None`.
-	/// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that
-	/// the caller will do this.
-	pub fn slash(asset_id: &T::AssetId, who: &T::AccountId, amount: T::Balance) -> Option<T::Balance> {
-		let free_balance = Self::free_balance(asset_id, who);
-		let free_slash = sp_std::cmp::min(free_balance, amount);
-		let new_free_balance = free_balance - free_slash;
-		Self::set_free_balance(asset_id, who, new_free_balance);
-		if free_slash < amount {
-			Self::slash_reserved(asset_id, who, amount - free_slash)
-		} else {
-			None
-		}
-	}
-
-	/// Deducts up to `amount` from reserved balance of `who`. This function cannot fail.
-	///
-	/// As much funds up to `amount` will be deducted as possible. If the reserve balance of `who`
-	/// is less than `amount`, then a non-zero second item will be returned.
-	/// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that
-	/// the caller will do this.
-	pub fn slash_reserved(asset_id: &T::AssetId, who: &T::AccountId, amount: T::Balance) -> Option<T::Balance> {
-		let original_reserve_balance = Self::reserved_balance(asset_id, who);
-		let slash = sp_std::cmp::min(original_reserve_balance, amount);
-		let new_reserve_balance = original_reserve_balance - slash;
-		Self::set_reserved_balance(asset_id, who, new_reserve_balance);
-		if amount == slash {
-			None
-		} else {
-			Some(amount - slash)
-		}
-	}
-
-	/// Move up to `amount` from reserved balance of account `who` to balance of account
-	/// `beneficiary`, either free or reserved depending on `status`.
-	///
-	/// As much funds up to `amount` will be moved as possible. If this is less than `amount`, then
-	/// the `remaining` would be returned, else `Zero::zero()`.
-	/// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that
-	/// the caller will do this.
-	pub fn repatriate_reserved(
-		asset_id: &T::AssetId,
-		who: &T::AccountId,
-		beneficiary: &T::AccountId,
-		amount: T::Balance,
-		status: BalanceStatus,
-	) -> T::Balance {
-		let b = Self::reserved_balance(asset_id, who);
-		let slash = sp_std::cmp::min(b, amount);
-
-		match status {
-			BalanceStatus::Free => {
-				let original_free_balance = Self::free_balance(asset_id, beneficiary);
-				let new_free_balance = original_free_balance + slash;
-				Self::set_free_balance(asset_id, beneficiary, new_free_balance);
-			}
-			BalanceStatus::Reserved => {
-				let original_reserved_balance = Self::reserved_balance(asset_id, beneficiary);
-				let new_reserved_balance = original_reserved_balance + slash;
-				Self::set_reserved_balance(asset_id, beneficiary, new_reserved_balance);
-			}
-		}
-
-		let new_reserve_balance = b - slash;
-		Self::set_reserved_balance(asset_id, who, new_reserve_balance);
-		amount - slash
-	}
-
-	/// Check permission to perform burn, mint or update.
-	///
-	/// # Arguments
-	/// * `asset_id`:  A `T::AssetId` type that contains the `asset_id`, which has the permission embedded.
-	/// * `who`: A `T::AccountId` type that contains the `account_id` for which to check permissions.
-	/// * `what`: The permission to check.
-	///
-	pub fn check_permission(asset_id: &T::AssetId, who: &T::AccountId, what: &PermissionType) -> bool {
-		let permission_versions: PermissionVersions<T::AccountId> = Self::get_permission(asset_id);
-		let permission = permission_versions.into();
-
-		match (what, permission) {
-			(
-				PermissionType::Burn,
-				PermissionLatest {
-					burn: Owner::Address(account),
-					..
-				},
-			) => account == *who,
-			(
-				PermissionType::Mint,
-				PermissionLatest {
-					mint: Owner::Address(account),
-					..
-				},
-			) => account == *who,
-			(
-				PermissionType::Update,
-				PermissionLatest {
-					update: Owner::Address(account),
-					..
-				},
-			) => account == *who,
-			_ => false,
-		}
-	}
-
-	/// Return `Ok` iff the account is able to make a withdrawal of the given amount
-	/// for the given reason.
-	///
-	/// `Err(...)` with the reason why not otherwise.
-	pub fn ensure_can_withdraw(
-		asset_id: &T::AssetId,
-		who: &T::AccountId,
-		_amount: T::Balance,
-		reasons: WithdrawReasons,
-		new_balance: T::Balance,
-	) -> DispatchResult {
-		if asset_id != &Self::staking_asset_id() {
-			return Ok(());
-		}
-
-		let locks = Self::locks(who);
-		if locks.is_empty() {
-			return Ok(());
-		}
-		if Self::locks(who)
-			.into_iter().all(|l| new_balance >= l.amount || !l.reasons.intersects(reasons))
-		{
-			Ok(())
-		} else {
-			Err(Error::<T>::LiquidityRestrictions)?
-		}
-	}
-
-	// PRIVATE MUTABLES
-
-	/// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that
-	/// the caller will do this.
-	fn set_reserved_balance(asset_id: &T::AssetId, who: &T::AccountId, balance: T::Balance) {
-		<ReservedBalance<T>>::insert(asset_id, who, &balance);
-	}
-
-	/// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that
-	/// the caller will do this.
-	fn set_free_balance(asset_id: &T::AssetId, who: &T::AccountId, balance: T::Balance) {
-		<FreeBalance<T>>::insert(asset_id, who, &balance);
-	}
-
-	fn set_lock(
-		id: LockIdentifier,
-		who: &T::AccountId,
-		amount: T::Balance,
-		reasons: WithdrawReasons,
-	) {
-		let mut new_lock = Some(BalanceLock {
-			id,
-			amount,
-			reasons,
-		});
-		let mut locks = <Module<T>>::locks(who)
-			.into_iter()
-			.filter_map(|l| {
-				if l.id == id {
-					new_lock.take()
-				} else {
-					Some(l)
-				}
-			})
-			.collect::<Vec<_>>();
-		if let Some(lock) = new_lock {
-			locks.push(lock)
-		}
-		<Locks<T>>::insert(who, locks);
-	}
-
-	fn extend_lock(
-		id: LockIdentifier,
-		who: &T::AccountId,
-		amount: T::Balance,
-		reasons: WithdrawReasons,
-	) {
-		let mut new_lock = Some(BalanceLock {
-			id,
-			amount,
-			reasons,
-		});
-		let mut locks = <Module<T>>::locks(who)
-			.into_iter()
-			.filter_map(|l| {
-				if l.id == id {
-					new_lock.take().map(|nl| BalanceLock {
-						id: l.id,
-						amount: l.amount.max(nl.amount),
-						reasons: l.reasons | nl.reasons,
-					})
-				} else {
-					Some(l)
-				}
-			})
-			.collect::<Vec<_>>();
-		if let Some(lock) = new_lock {
-			locks.push(lock)
-		}
-		<Locks<T>>::insert(who, locks);
-	}
-
-	fn remove_lock(id: LockIdentifier, who: &T::AccountId) {
-		let mut locks = <Module<T>>::locks(who);
-		locks.retain(|l| l.id != id);
-		<Locks<T>>::insert(who, locks);
-	}
-}
-
-pub trait AssetIdProvider {
-	type AssetId;
-	fn asset_id() -> Self::AssetId;
-}
-
-// wrapping these imbalances in a private module is necessary to ensure absolute privacy
-// of the inner member.
-mod imbalances {
-	use super::{
-		result, AssetIdProvider, Imbalance, Saturating, StorageMap, Subtrait, Zero, TryDrop
-	};
-	use sp_std::mem;
-
-	/// Opaque, move-only struct with private fields that serves as a token denoting that
-	/// funds have been created without any equal and opposite accounting.
-	#[must_use]
-	pub struct PositiveImbalance<T: Subtrait, U: AssetIdProvider<AssetId = T::AssetId>>(
-		T::Balance,
-		sp_std::marker::PhantomData<U>,
-	);
-	impl<T, U> PositiveImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		pub fn new(amount: T::Balance) -> Self {
-			PositiveImbalance(amount, Default::default())
-		}
-	}
-
-	/// Opaque, move-only struct with private fields that serves as a token denoting that
-	/// funds have been destroyed without any equal and opposite accounting.
-	#[must_use]
-	pub struct NegativeImbalance<T: Subtrait, U: AssetIdProvider<AssetId = T::AssetId>>(
-		T::Balance,
-		sp_std::marker::PhantomData<U>,
-	);
-	impl<T, U> NegativeImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		pub fn new(amount: T::Balance) -> Self {
-			NegativeImbalance(amount, Default::default())
-		}
-	}
-
-	impl<T, U> TryDrop for PositiveImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		fn try_drop(self) -> result::Result<(), Self> {
-			self.drop_zero()
-		}
-	}
-
-	impl<T, U> Imbalance<T::Balance> for PositiveImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		type Opposite = NegativeImbalance<T, U>;
-
-		fn zero() -> Self {
-			Self::new(Zero::zero())
-		}
-		fn drop_zero(self) -> result::Result<(), Self> {
-			if self.0.is_zero() {
-				Ok(())
-			} else {
-				Err(self)
-			}
-		}
-		fn split(self, amount: T::Balance) -> (Self, Self) {
-			let first = self.0.min(amount);
-			let second = self.0 - first;
-
-			mem::forget(self);
-			(Self::new(first), Self::new(second))
-		}
-		fn merge(mut self, other: Self) -> Self {
-			self.0 = self.0.saturating_add(other.0);
-			mem::forget(other);
-
-			self
-		}
-		fn subsume(&mut self, other: Self) {
-			self.0 = self.0.saturating_add(other.0);
-			mem::forget(other);
-		}
-		fn offset(self, other: Self::Opposite) -> result::Result<Self, Self::Opposite> {
-			let (a, b) = (self.0, other.0);
-			mem::forget((self, other));
-
-			if a >= b {
-				Ok(Self::new(a - b))
-			} else {
-				Err(NegativeImbalance::new(b - a))
-			}
-		}
-		fn peek(&self) -> T::Balance {
-			self.0.clone()
-		}
-	}
-
-	impl<T, U> TryDrop for NegativeImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		fn try_drop(self) -> result::Result<(), Self> {
-			self.drop_zero()
-		}
-	}
-
-	impl<T, U> Imbalance<T::Balance> for NegativeImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		type Opposite = PositiveImbalance<T, U>;
-
-		fn zero() -> Self {
-			Self::new(Zero::zero())
-		}
-		fn drop_zero(self) -> result::Result<(), Self> {
-			if self.0.is_zero() {
-				Ok(())
-			} else {
-				Err(self)
-			}
-		}
-		fn split(self, amount: T::Balance) -> (Self, Self) {
-			let first = self.0.min(amount);
-			let second = self.0 - first;
-
-			mem::forget(self);
-			(Self::new(first), Self::new(second))
-		}
-		fn merge(mut self, other: Self) -> Self {
-			self.0 = self.0.saturating_add(other.0);
-			mem::forget(other);
-
-			self
-		}
-		fn subsume(&mut self, other: Self) {
-			self.0 = self.0.saturating_add(other.0);
-			mem::forget(other);
-		}
-		fn offset(self, other: Self::Opposite) -> result::Result<Self, Self::Opposite> {
-			let (a, b) = (self.0, other.0);
-			mem::forget((self, other));
-
-			if a >= b {
-				Ok(Self::new(a - b))
-			} else {
-				Err(PositiveImbalance::new(b - a))
-			}
-		}
-		fn peek(&self) -> T::Balance {
-			self.0.clone()
-		}
-	}
-
-	impl<T, U> Drop for PositiveImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		/// Basic drop handler will just square up the total issuance.
-		fn drop(&mut self) {
-			<super::TotalIssuance<super::ElevatedTrait<T>>>::mutate(&U::asset_id(), |v| *v = v.saturating_add(self.0));
-		}
-	}
-
-	impl<T, U> Drop for NegativeImbalance<T, U>
-	where
-		T: Subtrait,
-		U: AssetIdProvider<AssetId = T::AssetId>,
-	{
-		/// Basic drop handler will just square up the total issuance.
-		fn drop(&mut self) {
-			<super::TotalIssuance<super::ElevatedTrait<T>>>::mutate(&U::asset_id(), |v| *v = v.saturating_sub(self.0));
-		}
-	}
-}
-
-// TODO: #2052
-// Somewhat ugly hack in order to gain access to module's `increase_total_issuance_by`
-// using only the Subtrait (which defines only the types that are not dependent
-// on Positive/NegativeImbalance). Subtrait must be used otherwise we end up with a
-// circular dependency with Trait having some types be dependent on PositiveImbalance<Trait>
-// and PositiveImbalance itself depending back on Trait for its Drop impl (and thus
-// its type declaration).
-// This works as long as `increase_total_issuance_by` doesn't use the Imbalance
-// types (basically for charging fees).
-// This should eventually be refactored so that the two type items that do
-// depend on the Imbalance type (TransactionPayment, DustRemoval)
-// are placed in their own pallet.
-struct ElevatedTrait<T: Subtrait>(T);
-impl<T: Subtrait> Clone for ElevatedTrait<T> {
-	fn clone(&self) -> Self {
-		unimplemented!()
-	}
-}
-impl<T: Subtrait> PartialEq for ElevatedTrait<T> {
-	fn eq(&self, _: &Self) -> bool {
-		unimplemented!()
-	}
-}
-impl<T: Subtrait> Eq for ElevatedTrait<T> {}
-impl<T: Subtrait> frame_system::Trait for ElevatedTrait<T> {
-	type BaseCallFilter = T::BaseCallFilter;
-	type Origin = T::Origin;
-	type Call = T::Call;
-	type Index = T::Index;
-	type BlockNumber = T::BlockNumber;
-	type Hash = T::Hash;
-	type Hashing = T::Hashing;
-	type AccountId = T::AccountId;
-	type Lookup = T::Lookup;
-	type Header = T::Header;
-	type Event = ();
-	type BlockHashCount = T::BlockHashCount;
-	type MaximumBlockWeight = T::MaximumBlockWeight;
-	type DbWeight = ();
-	type BlockExecutionWeight = ();
-	type ExtrinsicBaseWeight = ();
-	type MaximumExtrinsicWeight = T::MaximumBlockWeight;
-	type MaximumBlockLength = T::MaximumBlockLength;
-	type AvailableBlockRatio = T::AvailableBlockRatio;
-	type Version = T::Version;
-	type ModuleToIndex = ();
-	type AccountData = ();
-	type OnNewAccount = ();
-	type OnKilledAccount = ();
-	type SystemWeightInfo = ();
-}
-impl<T: Subtrait> Trait for ElevatedTrait<T> {
-	type Balance = T::Balance;
-	type AssetId = T::AssetId;
-	type Event = ();
-}
-
-#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)]
-pub struct AssetCurrency<T, U>(sp_std::marker::PhantomData<T>, sp_std::marker::PhantomData<U>);
-
-impl<T, U> Currency<T::AccountId> for AssetCurrency<T, U>
-where
-	T: Trait,
-	U: AssetIdProvider<AssetId = T::AssetId>,
-{
-	type Balance = T::Balance;
-	type PositiveImbalance = PositiveImbalance<T, U>;
-	type NegativeImbalance = NegativeImbalance<T, U>;
-
-	fn total_balance(who: &T::AccountId) -> Self::Balance {
-		Self::free_balance(&who) + Self::reserved_balance(&who)
-	}
-
-	fn free_balance(who: &T::AccountId) -> Self::Balance {
-		<Module<T>>::free_balance(&U::asset_id(), &who)
-	}
-
-	/// Returns the total staking asset issuance
-	fn total_issuance() -> Self::Balance {
-		<Module<T>>::total_issuance(U::asset_id())
-	}
-
-	fn minimum_balance() -> Self::Balance {
-		Zero::zero()
-	}
-
-	fn transfer(
-		transactor: &T::AccountId,
-		dest: &T::AccountId,
-		value: Self::Balance,
-		_: ExistenceRequirement, // no existential deposit policy for generic asset
-	) -> DispatchResult {
-		<Module<T>>::make_transfer(&U::asset_id(), transactor, dest, value)
-	}
-
-	fn ensure_can_withdraw(
-		who: &T::AccountId,
-		amount: Self::Balance,
-		reasons: WithdrawReasons,
-		new_balance: Self::Balance,
-	) -> DispatchResult {
-		<Module<T>>::ensure_can_withdraw(&U::asset_id(), who, amount, reasons, new_balance)
-	}
-
-	fn withdraw(
-		who: &T::AccountId,
-		value: Self::Balance,
-		reasons: WithdrawReasons,
-		_: ExistenceRequirement, // no existential deposit policy for generic asset
-	) -> result::Result<Self::NegativeImbalance, DispatchError> {
-		let new_balance = Self::free_balance(who)
-			.checked_sub(&value)
-			.ok_or(Error::<T>::InsufficientBalance)?;
-		Self::ensure_can_withdraw(who, value, reasons, new_balance)?;
-		<Module<T>>::set_free_balance(&U::asset_id(), who, new_balance);
-		Ok(NegativeImbalance::new(value))
-	}
-
-	fn deposit_into_existing(
-		who: &T::AccountId,
-		value: Self::Balance,
-	) -> result::Result<Self::PositiveImbalance, DispatchError> {
-		// No existential deposit rule and creation fee in GA. `deposit_into_existing` is same with `deposit_creating`.
-		Ok(Self::deposit_creating(who, value))
-	}
-
-	fn deposit_creating(who: &T::AccountId, value: Self::Balance) -> Self::PositiveImbalance {
-		let imbalance = Self::make_free_balance_be(who, Self::free_balance(who) + value);
-		if let SignedImbalance::Positive(p) = imbalance {
-			p
-		} else {
-			// Impossible, but be defensive.
-			Self::PositiveImbalance::zero()
-		}
-	}
-
-	fn make_free_balance_be(
-		who: &T::AccountId,
-		balance: Self::Balance,
-	) -> SignedImbalance<Self::Balance, Self::PositiveImbalance> {
-		let original = <Module<T>>::free_balance(&U::asset_id(), who);
-		let imbalance = if original <= balance {
-			SignedImbalance::Positive(PositiveImbalance::new(balance - original))
-		} else {
-			SignedImbalance::Negative(NegativeImbalance::new(original - balance))
-		};
-		<Module<T>>::set_free_balance(&U::asset_id(), who, balance);
-		imbalance
-	}
-
-	fn can_slash(who: &T::AccountId, value: Self::Balance) -> bool {
-		<Module<T>>::free_balance(&U::asset_id(), &who) >= value
-	}
-
-	fn slash(who: &T::AccountId, value: Self::Balance) -> (Self::NegativeImbalance, Self::Balance) {
-		let remaining = <Module<T>>::slash(&U::asset_id(), who, value);
-		if let Some(r) = remaining {
-			(NegativeImbalance::new(value - r), r)
-		} else {
-			(NegativeImbalance::new(value), Zero::zero())
-		}
-	}
-
-	fn burn(mut amount: Self::Balance) -> Self::PositiveImbalance {
-		<TotalIssuance<T>>::mutate(&U::asset_id(), |issued|
-			issued.checked_sub(&amount).unwrap_or_else(|| {
-				amount = *issued;
-				Zero::zero()
-			})
-		);
-		PositiveImbalance::new(amount)
-	}
-
-	fn issue(mut amount: Self::Balance) -> Self::NegativeImbalance {
-		<TotalIssuance<T>>::mutate(&U::asset_id(), |issued|
-			*issued = issued.checked_add(&amount).unwrap_or_else(|| {
-				amount = Self::Balance::max_value() - *issued;
-				Self::Balance::max_value()
-			})
-		);
-		NegativeImbalance::new(amount)
-	}
-}
-
-impl<T, U> ReservableCurrency<T::AccountId> for AssetCurrency<T, U>
-where
-	T: Trait,
-	U: AssetIdProvider<AssetId = T::AssetId>,
-{
-	fn can_reserve(who: &T::AccountId, value: Self::Balance) -> bool {
-		Self::free_balance(who)
-			.checked_sub(&value)
-			.map_or(false, |new_balance|
-				<Module<T>>::ensure_can_withdraw(
-					&U::asset_id(), who, value, WithdrawReason::Reserve.into(), new_balance
-				).is_ok()
-			)
-	}
-
-	fn reserved_balance(who: &T::AccountId) -> Self::Balance {
-		<Module<T>>::reserved_balance(&U::asset_id(), &who)
-	}
-
-	fn reserve(who: &T::AccountId, value: Self::Balance) -> DispatchResult {
-		<Module<T>>::reserve(&U::asset_id(), who, value)
-	}
-
-	fn unreserve(who: &T::AccountId, value: Self::Balance) -> Self::Balance {
-		<Module<T>>::unreserve(&U::asset_id(), who, value)
-	}
-
-	fn slash_reserved(who: &T::AccountId, value: Self::Balance) -> (Self::NegativeImbalance, Self::Balance) {
-		let b = Self::reserved_balance(&who.clone());
-		let slash = cmp::min(b, value);
-
-		<Module<T>>::set_reserved_balance(&U::asset_id(), who, b - slash);
-		(NegativeImbalance::new(slash), value - slash)
-	}
-
-	fn repatriate_reserved(
-		slashed: &T::AccountId,
-		beneficiary: &T::AccountId,
-		value: Self::Balance,
-		status: BalanceStatus,
-	) -> result::Result<Self::Balance, DispatchError> {
-		Ok(<Module<T>>::repatriate_reserved(&U::asset_id(), slashed, beneficiary, value, status))
-	}
-}
-
-pub struct StakingAssetIdProvider<T>(sp_std::marker::PhantomData<T>);
-
-impl<T: Trait> AssetIdProvider for StakingAssetIdProvider<T> {
-	type AssetId = T::AssetId;
-	fn asset_id() -> Self::AssetId {
-		<Module<T>>::staking_asset_id()
-	}
-}
-
-pub struct SpendingAssetIdProvider<T>(sp_std::marker::PhantomData<T>);
-
-impl<T: Trait> AssetIdProvider for SpendingAssetIdProvider<T> {
-	type AssetId = T::AssetId;
-	fn asset_id() -> Self::AssetId {
-		<Module<T>>::spending_asset_id()
-	}
-}
-
-impl<T> LockableCurrency<T::AccountId> for AssetCurrency<T, StakingAssetIdProvider<T>>
-where
-	T: Trait,
-	T::Balance: MaybeSerializeDeserialize + Debug,
-{
-	type Moment = T::BlockNumber;
-
-	type MaxLocks = ();
-
-	fn set_lock(
-		id: LockIdentifier,
-		who: &T::AccountId,
-		amount: T::Balance,
-		reasons: WithdrawReasons,
-	) {
-		<Module<T>>::set_lock(id, who, amount, reasons)
-	}
-
-	fn extend_lock(
-		id: LockIdentifier,
-		who: &T::AccountId,
-		amount: T::Balance,
-		reasons: WithdrawReasons,
-	) {
-		<Module<T>>::extend_lock(id, who, amount, reasons)
-	}
-
-	fn remove_lock(id: LockIdentifier, who: &T::AccountId) {
-		<Module<T>>::remove_lock(id, who)
-	}
-}
-
-pub type StakingAssetCurrency<T> = AssetCurrency<T, StakingAssetIdProvider<T>>;
-pub type SpendingAssetCurrency<T> = AssetCurrency<T, SpendingAssetIdProvider<T>>;

From f7f08738593575e1b957e7ba501186414f384281 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Tue, 29 Sep 2020 10:30:31 +0200
Subject: [PATCH 116/122] Apply suggestions from code review

Co-authored-by: Max Inden <mail@max-inden.de>
---
 client/network/src/protocol.rs                      | 2 +-
 client/network/src/protocol/sync.rs                 | 2 +-
 primitives/consensus/common/src/block_validation.rs | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 27d5706c10c07..23bb20a68254e 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -1168,7 +1168,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 	/// It is required that [`ChainSync::poll_block_announce_validation`] is
 	/// called later to check for finished validations. The result of the validation
 	/// needs to be passed to [`Protocol::process_block_announce_validation_result`]
-	/// finish the processing.
+	/// to finish the processing.
 	fn push_block_announce_validation(
 		&mut self,
 		who: PeerId,
diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 119a7070b1ef2..908bf27e25a10 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -1244,7 +1244,7 @@ impl<B: BlockT> ChainSync<B> {
 	/// Checks if there is a slot for a block announce validation.
 	///
 	/// The total number and the number per peer of concurrent block announce validations
-	/// is caped.
+	/// is capped.
 	///
 	/// It will return `true` when there is still a slot for a block announce validation.
 	fn has_slot_for_block_announce_validation(&mut self, peer: &PeerId) -> HasSlotForBlockAnnounceValidation {
diff --git a/primitives/consensus/common/src/block_validation.rs b/primitives/consensus/common/src/block_validation.rs
index c45d076caad5e..f8255130e6416 100644
--- a/primitives/consensus/common/src/block_validation.rs
+++ b/primitives/consensus/common/src/block_validation.rs
@@ -51,8 +51,8 @@ pub trait BlockAnnounceValidator<B: Block> {
 	///
 	/// # Note
 	///
-	/// Returning [`Validation::Failure`] will led to a decrease of the
-	/// peers reputation as it send us invalid data.
+	/// Returning [`Validation::Failure`] will lead to a decrease of the
+	/// peers reputation as it sent us invalid data.
 	fn validate(
 		&mut self,
 		header: &B::Header,

From 4f252be290da4ec3221321186d8748930c1c599b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Tue, 29 Sep 2020 10:45:51 +0200
Subject: [PATCH 117/122] Rename `BlockAnnounceResult` to
 `PollBlockAnnounceValidation`

---
 client/network/src/protocol.rs      |  8 +++----
 client/network/src/protocol/sync.rs | 34 ++++++++++++++---------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 23bb20a68254e..80b91c24b0dd1 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -1195,10 +1195,10 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 	/// Process the result of the block announce validation.
 	fn process_block_announce_validation_result(
 		&mut self,
-		validation_result: sync::BlockAnnounceResult<B::Header>,
+		validation_result: sync::PollBlockAnnounceValidation<B::Header>,
 	) -> CustomMessageOutcome<B> {
 		let (header, is_best, who) = match validation_result {
-			sync::BlockAnnounceResult::Nothing { is_best, who, header } => {
+			sync::PollBlockAnnounceValidation::Nothing { is_best, who, header } => {
 				self.update_peer_info(&who);
 
 				// `on_block_announce` returns `OnBlockAnnounce::ImportHeader`
@@ -1213,11 +1213,11 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 					return CustomMessageOutcome::None
 				}
 			}
-			sync::BlockAnnounceResult::ImportHeader { header, is_best, who } => {
+			sync::PollBlockAnnounceValidation::ImportHeader { header, is_best, who } => {
 				self.update_peer_info(&who);
 				(header, is_best, who)
 			}
-			sync::BlockAnnounceResult::Failure { who } => {
+			sync::PollBlockAnnounceValidation::Failure { who } => {
 				self.report_peer(who, rep::BAD_BLOCK_ANNOUNCEMENT);
 				return CustomMessageOutcome::None
 			}
diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 908bf27e25a10..41b424ac0cd8c 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -327,9 +327,9 @@ pub enum OnBlockData<B: BlockT> {
 	Request(PeerId, BlockRequest<B>)
 }
 
-/// Result of [`ChainSync::process_block_announce_pre_validation`].
+/// Result of [`ChainSync::poll_block_announce_validation`].
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum BlockAnnounceResult<H> {
+pub enum PollBlockAnnounceValidation<H> {
 	/// The announcement failed at validation.
 	///
 	/// The peer reputation should be decreased.
@@ -1280,7 +1280,7 @@ impl<B: BlockT> ChainSync<B> {
 		hash: &B::Hash,
 		announce: BlockAnnounce<B::Header>,
 		is_best: bool,
-	) -> Option<BlockAnnounceResult<B::Header>> {
+	) -> Option<PollBlockAnnounceValidation<B::Header>> {
 		let header = &announce.header;
 		let number = *header.number();
 		debug!(
@@ -1298,7 +1298,7 @@ impl<B: BlockT> ChainSync<B> {
 				who,
 				hash,
 			);
-			return Some(BlockAnnounceResult::Nothing { is_best, who, header: announce.header })
+			return Some(PollBlockAnnounceValidation::Nothing { is_best, who, header: announce.header })
 		}
 
 		// Check if there is a slot for this block announce validation.
@@ -1312,7 +1312,7 @@ impl<B: BlockT> ChainSync<B> {
 					hash,
 					who,
 				);
-				return Some(BlockAnnounceResult::Nothing { is_best, who, header: announce.header })
+				return Some(PollBlockAnnounceValidation::Nothing { is_best, who, header: announce.header })
 			}
 			HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached => {
 				warn!(
@@ -1322,7 +1322,7 @@ impl<B: BlockT> ChainSync<B> {
 					hash,
 					who,
 				);
-				return Some(BlockAnnounceResult::Nothing { is_best, who, header: announce.header })
+				return Some(PollBlockAnnounceValidation::Nothing { is_best, who, header: announce.header })
 			}
 		}
 
@@ -1364,12 +1364,12 @@ impl<B: BlockT> ChainSync<B> {
 	///
 	/// This should be polled until it returns [`Poll::Pending`].
 	///
-	/// If [`BlockAnnounceResult::ImportHeader`] is returned, then the caller MUST try to import passed
+	/// If [`PollBlockAnnounceValidation::ImportHeader`] is returned, then the caller MUST try to import passed
 	/// header (call `on_block_data`). The network request isn't sent in this case.
 	pub fn poll_block_announce_validation(
 		&mut self,
 		cx: &mut std::task::Context,
-	) -> Poll<BlockAnnounceResult<B::Header>> {
+	) -> Poll<PollBlockAnnounceValidation<B::Header>> {
 		match self.block_announce_validation.poll_next_unpin(cx) {
 			Poll::Ready(Some(res)) => Poll::Ready(self.finish_block_announce_validation(res)),
 			_ => Poll::Pending,
@@ -1399,15 +1399,15 @@ impl<B: BlockT> ChainSync<B> {
 	fn finish_block_announce_validation(
 		&mut self,
 		pre_validation_result: PreValidateBlockAnnounce<B::Header>,
-	) -> BlockAnnounceResult<B::Header> {
+	) -> PollBlockAnnounceValidation<B::Header> {
 		let (announce, is_best, who) = match pre_validation_result {
 			PreValidateBlockAnnounce::Nothing { is_best, who, announce } => {
 				self.peer_block_announce_validation_finished(&who);
-				return BlockAnnounceResult::Nothing { is_best, who, header: announce.header }
+				return PollBlockAnnounceValidation::Nothing { is_best, who, header: announce.header }
 			},
 			PreValidateBlockAnnounce::Failure { who } => {
 				self.peer_block_announce_validation_finished(&who);
-				return BlockAnnounceResult::Failure { who }
+				return PollBlockAnnounceValidation::Failure { who }
 			},
 			PreValidateBlockAnnounce::Process { announce, is_new_best, who } => {
 				self.peer_block_announce_validation_finished(&who);
@@ -1427,7 +1427,7 @@ impl<B: BlockT> ChainSync<B> {
 			peer
 		} else {
 			error!(target: "sync", "💔 Called on_block_announce with a bad peer ID");
-			return BlockAnnounceResult::Nothing { is_best, who, header }
+			return PollBlockAnnounceValidation::Nothing { is_best, who, header }
 		};
 
 		while peer.recently_announced.len() >= ANNOUNCE_HISTORY_SIZE {
@@ -1442,7 +1442,7 @@ impl<B: BlockT> ChainSync<B> {
 		}
 
 		if let PeerSyncState::AncestorSearch {..} = peer.state {
-			return BlockAnnounceResult::Nothing { is_best, who, header }
+			return PollBlockAnnounceValidation::Nothing { is_best, who, header }
 		}
 
 		// If the announced block is the best they have and is not ahead of us, our common number
@@ -1464,18 +1464,18 @@ impl<B: BlockT> ChainSync<B> {
 			if let Some(target) = self.fork_targets.get_mut(&hash) {
 				target.peers.insert(who.clone());
 			}
-			return BlockAnnounceResult::Nothing { is_best, who, header }
+			return PollBlockAnnounceValidation::Nothing { is_best, who, header }
 		}
 
 		if ancient_parent {
 			trace!(target: "sync", "Ignored ancient block announced from {}: {} {:?}", who, hash, header);
-			return BlockAnnounceResult::Nothing { is_best, who, header }
+			return PollBlockAnnounceValidation::Nothing { is_best, who, header }
 		}
 
 		let requires_additional_data = !self.role.is_light() || !known_parent;
 		if !requires_additional_data {
 			trace!(target: "sync", "Importing new header announced from {}: {} {:?}", who, hash, header);
-			return BlockAnnounceResult::ImportHeader { is_best, header, who }
+			return PollBlockAnnounceValidation::ImportHeader { is_best, header, who }
 		}
 
 		if number <= self.best_queued_number {
@@ -1493,7 +1493,7 @@ impl<B: BlockT> ChainSync<B> {
 				.peers.insert(who.clone());
 		}
 
-		BlockAnnounceResult::Nothing { is_best, who, header }
+		PollBlockAnnounceValidation::Nothing { is_best, who, header }
 	}
 
 	/// Call when a peer has disconnected.

From e08cbcf458c7185ed49f0a3c50578a8dc248ca42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Tue, 29 Sep 2020 11:07:57 +0200
Subject: [PATCH 118/122] Always return result using the internal future

---
 client/network/src/protocol.rs      | 18 ++++-----
 client/network/src/protocol/sync.rs | 63 ++++++++++++++++-------------
 2 files changed, 41 insertions(+), 40 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 80b91c24b0dd1..f7fafee8e9833 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -599,9 +599,8 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		match message {
 			GenericMessage::Status(_) =>
 				debug!(target: "sub-libp2p", "Received unexpected Status"),
-			GenericMessage::BlockAnnounce(announce) => {
-				return self.push_block_announce_validation(who.clone(), announce)
-			},
+			GenericMessage::BlockAnnounce(announce) =>
+				self.push_block_announce_validation(who.clone(), announce),
 			GenericMessage::Transactions(m) =>
 				self.on_transactions(who, m),
 			GenericMessage::BlockResponse(_) =>
@@ -1173,7 +1172,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		&mut self,
 		who: PeerId,
 		announce: BlockAnnounce<B::Header>,
-	) -> CustomMessageOutcome<B> {
+	) {
 		let hash = announce.header.hash();
 
 		if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) {
@@ -1185,11 +1184,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			message::BlockState::Normal => false,
 		};
 
-		if let Some(res) = self.sync.push_block_announce_validation(who, &hash, announce, is_best) {
-			self.process_block_announce_validation_result(res)
-		} else {
-			CustomMessageOutcome::None
-		}
+		self.sync.push_block_announce_validation(who, hash, announce, is_best);
 	}
 
 	/// Process the result of the block announce validation.
@@ -1680,11 +1675,12 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 					}
 					Some(Fallback::BlockAnnounce) => {
 						if let Ok(announce) = message::BlockAnnounce::decode(&mut message.as_ref()) {
-							self.push_block_announce_validation(peer_id, announce)
+							self.push_block_announce_validation(peer_id, announce);
 						} else {
 							warn!(target: "sub-libp2p", "Failed to decode block announce");
-							CustomMessageOutcome::None
 						}
+
+						CustomMessageOutcome::None
 					}
 					None => {
 						debug!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name);
diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 41b424ac0cd8c..7714d5730b352 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -1272,15 +1272,13 @@ impl<B: BlockT> ChainSync<B> {
 	///
 	/// It is required that [`ChainSync::poll_block_announce_validation`] is called
 	/// to check for finished block announce validations.
-	///
-	/// Returns `Some(_)` if the block announce was validated immediately.
 	pub fn push_block_announce_validation(
 		&mut self,
 		who: PeerId,
-		hash: &B::Hash,
+		hash: B::Hash,
 		announce: BlockAnnounce<B::Header>,
 		is_best: bool,
-	) -> Option<PollBlockAnnounceValidation<B::Header>> {
+	) {
 		let header = &announce.header;
 		let number = *header.number();
 		debug!(
@@ -1292,37 +1290,46 @@ impl<B: BlockT> ChainSync<B> {
 		);
 
 		if number.is_zero() {
-			warn!(
-				target: "sync",
-				"💔 Ignored genesis block (#0) announcement from {}: {}",
-				who,
-				hash,
-			);
-			return Some(PollBlockAnnounceValidation::Nothing { is_best, who, header: announce.header })
+			self.block_announce_validation.push(async move {
+				warn!(
+					target: "sync",
+					"💔 Ignored genesis block (#0) announcement from {}: {}",
+					who,
+					hash,
+				);
+				PreValidateBlockAnnounce::Nothing { is_best, who, announce }
+			}.boxed());
+			return
 		}
 
 		// Check if there is a slot for this block announce validation.
 		match self.has_slot_for_block_announce_validation(&who) {
 			HasSlotForBlockAnnounceValidation::Yes => {},
 			HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached => {
-				warn!(
-					target: "sync",
-					"💔 Ignored block (#{} -- {}) announcement from {} because all validation slots are occupied.",
-					number,
-					hash,
-					who,
-				);
-				return Some(PollBlockAnnounceValidation::Nothing { is_best, who, header: announce.header })
+				self.block_announce_validation.push(async move {
+					warn!(
+						target: "sync",
+						"💔 Ignored block (#{} -- {}) announcement from {} because all validation slots are occupied.",
+						number,
+						hash,
+						who,
+					);
+					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
+				}.boxed());
+				return
 			}
 			HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached => {
-				warn!(
-					target: "sync",
-					"💔 Ignored block (#{} -- {}) announcement from {} because all validation slots for this peer are occupied.",
-					number,
-					hash,
-					who,
-				);
-				return Some(PollBlockAnnounceValidation::Nothing { is_best, who, header: announce.header })
+				self.block_announce_validation.push(async move {
+					warn!(
+						target: "sync",
+						"💔 Ignored block (#{} -- {}) announcement from {} because all validation slots for this peer are occupied.",
+						number,
+						hash,
+						who,
+					);
+					PreValidateBlockAnnounce::Nothing { is_best, who, announce }
+				}.boxed());
+				return
 			}
 		}
 
@@ -1353,8 +1360,6 @@ impl<B: BlockT> ChainSync<B> {
 				}
 			}
 		}.boxed());
-
-		None
 	}
 
 	/// Poll block announce validation.

From 2b1823497a11e57840950d73564c6c6b39edf978 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Wed, 30 Sep 2020 10:46:02 +0200
Subject: [PATCH 119/122] Move the polling and make sure all validation futures
 are registered

---
 client/network/src/protocol.rs | 46 +++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index f7fafee8e9833..7ebeea76d8450 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -572,10 +572,11 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		self.context_data.peers.iter().map(|(id, peer)| (id, &peer.info))
 	}
 
-	pub fn on_custom_message(
+	fn on_custom_message(
 		&mut self,
 		who: PeerId,
 		data: BytesMut,
+		cx: &mut std::task::Context,
 	) -> CustomMessageOutcome<B> {
 		let message = match <Message<B> as Decode>::decode(&mut &data[..]) {
 			Ok(message) => message,
@@ -600,7 +601,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			GenericMessage::Status(_) =>
 				debug!(target: "sub-libp2p", "Received unexpected Status"),
 			GenericMessage::BlockAnnounce(announce) =>
-				self.push_block_announce_validation(who.clone(), announce),
+				return self.push_block_announce_validation(who.clone(), announce, cx),
 			GenericMessage::Transactions(m) =>
 				self.on_transactions(who, m),
 			GenericMessage::BlockResponse(_) =>
@@ -1168,11 +1169,18 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 	/// called later to check for finished validations. The result of the validation
 	/// needs to be passed to [`Protocol::process_block_announce_validation_result`]
 	/// to finish the processing.
+	///
+	/// This function ensures that the block announce validation future is polled
+	/// at least once so that the task is woken up when the future is ready.
+	///
+	/// Returns the message outcome from polling and processing a potential finished
+	/// block announce validation.
 	fn push_block_announce_validation(
 		&mut self,
 		who: PeerId,
 		announce: BlockAnnounce<B::Header>,
-	) {
+		cx: &mut std::task::Context,
+	) -> CustomMessageOutcome<B> {
 		let hash = announce.header.hash();
 
 		if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) {
@@ -1185,6 +1193,14 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		};
 
 		self.sync.push_block_announce_validation(who, hash, announce, is_best);
+
+		// Make sure that the newly added block announce validation future was
+		// polled once to be registered in the task.
+		if let Poll::Ready(res) = self.sync.poll_block_announce_validation(cx) {
+			self.process_block_announce_validation_result(res)
+		} else {
+			CustomMessageOutcome::None
+		}
 	}
 
 	/// Process the result of the block announce validation.
@@ -1581,6 +1597,15 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 				warn!(target: "sub-libp2p", "Inconsistent state, no peers for pending transaction!");
 			}
 		}
+
+		// Check if there is any block announcement validation finished.
+		while let Poll::Ready(result) = self.sync.poll_block_announce_validation(cx) {
+			match self.process_block_announce_validation_result(result) {
+				CustomMessageOutcome::None => {},
+				outcome => self.pending_messages.push_back(outcome),
+			}
+		}
+
 		if let Some(message) = self.pending_messages.pop_front() {
 			return Poll::Ready(NetworkBehaviourAction::GenerateEvent(message));
 		}
@@ -1656,7 +1681,7 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 				self.on_peer_disconnected(peer_id)
 			},
 			GenericProtoOut::LegacyMessage { peer_id, message } =>
-				self.on_custom_message(peer_id, message),
+				self.on_custom_message(peer_id, message, cx),
 			GenericProtoOut::Notification { peer_id, protocol_name, message } =>
 				match self.legacy_equiv_by_name.get(&protocol_name) {
 					Some(Fallback::Consensus(engine_id)) => {
@@ -1675,12 +1700,11 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 					}
 					Some(Fallback::BlockAnnounce) => {
 						if let Ok(announce) = message::BlockAnnounce::decode(&mut message.as_ref()) {
-							self.push_block_announce_validation(peer_id, announce);
+							self.push_block_announce_validation(peer_id, announce, cx)
 						} else {
 							warn!(target: "sub-libp2p", "Failed to decode block announce");
+							CustomMessageOutcome::None
 						}
-
-						CustomMessageOutcome::None
 					}
 					None => {
 						debug!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name);
@@ -1690,14 +1714,6 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 		};
 
 		if let CustomMessageOutcome::None = outcome {
-			// Check if there is any block announcement validation finished.
-			while let Poll::Ready(result) = self.sync.poll_block_announce_validation(cx) {
-				match self.process_block_announce_validation_result(result) {
-					CustomMessageOutcome::None => continue,
-					outcome => return Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome))
-				}
-			}
-
 			Poll::Pending
 		} else {
 			Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome))

From 047f80c02164e14878bbd5c49b17952a51e99051 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Wed, 30 Sep 2020 12:24:54 +0200
Subject: [PATCH 120/122] Ignore the future in the legacy stuff

---
 client/network/src/protocol.rs | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index 7ebeea76d8450..e529e4a557fbf 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -601,7 +601,7 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 			GenericMessage::Status(_) =>
 				debug!(target: "sub-libp2p", "Received unexpected Status"),
 			GenericMessage::BlockAnnounce(announce) =>
-				return self.push_block_announce_validation(who.clone(), announce, cx),
+				self.push_block_announce_validation(who.clone(), announce),
 			GenericMessage::Transactions(m) =>
 				self.on_transactions(who, m),
 			GenericMessage::BlockResponse(_) =>
@@ -1170,17 +1170,17 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 	/// needs to be passed to [`Protocol::process_block_announce_validation_result`]
 	/// to finish the processing.
 	///
-	/// This function ensures that the block announce validation future is polled
-	/// at least once so that the task is woken up when the future is ready.
+	/// # Note
 	///
-	/// Returns the message outcome from polling and processing a potential finished
-	/// block announce validation.
+	/// This will internally create a future, but this future will not be registered
+	/// in the task before being polled once. So, it is required to call
+	/// [`ChainSync::poll_block_announce_validation`] to ensure that the future is
+	/// registered properly and will wake up the task when being ready.
 	fn push_block_announce_validation(
 		&mut self,
 		who: PeerId,
 		announce: BlockAnnounce<B::Header>,
-		cx: &mut std::task::Context,
-	) -> CustomMessageOutcome<B> {
+	) {
 		let hash = announce.header.hash();
 
 		if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) {
@@ -1193,14 +1193,6 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		};
 
 		self.sync.push_block_announce_validation(who, hash, announce, is_best);
-
-		// Make sure that the newly added block announce validation future was
-		// polled once to be registered in the task.
-		if let Poll::Ready(res) = self.sync.poll_block_announce_validation(cx) {
-			self.process_block_announce_validation_result(res)
-		} else {
-			CustomMessageOutcome::None
-		}
 	}
 
 	/// Process the result of the block announce validation.
@@ -1700,7 +1692,15 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 					}
 					Some(Fallback::BlockAnnounce) => {
 						if let Ok(announce) = message::BlockAnnounce::decode(&mut message.as_ref()) {
-							self.push_block_announce_validation(peer_id, announce, cx)
+							self.push_block_announce_validation(peer_id, announce);
+
+							// Make sure that the newly added block announce validation future was
+							// polled once to be registered in the task.
+							if let Poll::Ready(res) = self.sync.poll_block_announce_validation(cx) {
+								self.process_block_announce_validation_result(res)
+							} else {
+								CustomMessageOutcome::None
+							}
 						} else {
 							warn!(target: "sub-libp2p", "Failed to decode block announce");
 							CustomMessageOutcome::None

From 8ea6c88f4086827ef12d0b0ebe5711059d8c8f0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <git@kchr.de>
Date: Wed, 30 Sep 2020 12:45:42 +0200
Subject: [PATCH 121/122] Remove leftover stuff

---
 client/network/src/protocol.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs
index e529e4a557fbf..ac74af0f5ca94 100644
--- a/client/network/src/protocol.rs
+++ b/client/network/src/protocol.rs
@@ -576,7 +576,6 @@ impl<B: BlockT, H: ExHashT> Protocol<B, H> {
 		&mut self,
 		who: PeerId,
 		data: BytesMut,
-		cx: &mut std::task::Context,
 	) -> CustomMessageOutcome<B> {
 		let message = match <Message<B> as Decode>::decode(&mut &data[..]) {
 			Ok(message) => message,
@@ -1673,7 +1672,7 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviour for Protocol<B, H> {
 				self.on_peer_disconnected(peer_id)
 			},
 			GenericProtoOut::LegacyMessage { peer_id, message } =>
-				self.on_custom_message(peer_id, message, cx),
+				self.on_custom_message(peer_id, message),
 			GenericProtoOut::Notification { peer_id, protocol_name, message } =>
 				match self.legacy_equiv_by_name.get(&protocol_name) {
 					Some(Fallback::Consensus(engine_id)) => {

From 8e5a00ace7759817eda05e8bf0878a6050525881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <bkchr@users.noreply.github.com>
Date: Wed, 30 Sep 2020 17:20:03 +0200
Subject: [PATCH 122/122] Update client/network/src/protocol/sync.rs

---
 client/network/src/protocol/sync.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs
index 7714d5730b352..9866dc0568a8e 100644
--- a/client/network/src/protocol/sync.rs
+++ b/client/network/src/protocol/sync.rs
@@ -1246,7 +1246,7 @@ impl<B: BlockT> ChainSync<B> {
 	/// The total number and the number per peer of concurrent block announce validations
 	/// is capped.
 	///
-	/// It will return `true` when there is still a slot for a block announce validation.
+	/// Returns [`HasSlotForBlockAnnounceValidation`] to inform about the result.
 	fn has_slot_for_block_announce_validation(&mut self, peer: &PeerId) -> HasSlotForBlockAnnounceValidation {
 		if self.block_announce_validation.len() >= MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS {
 			return HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached