From a549691a57604ee311997a66c81583a0fa9dc771 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 14:15:19 +0200 Subject: [PATCH 001/241] start sketching out how voter bags might be implemented --- frame/staking/src/lib.rs | 31 +++++++++- frame/staking/src/voter_bags.rs | 101 ++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 frame/staking/src/voter_bags.rs diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 888601e307f35..42dc062569e27 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -276,8 +276,9 @@ pub mod testing_utils; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod benchmarking; -pub mod slashing; pub mod inflation; +pub mod slashing; +pub mod voter_bags; pub mod weights; use sp_std::{ @@ -353,6 +354,9 @@ type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; +type AccountIdOf = ::AccountId; +type VotingDataOf = (AccountIdOf, VoteWeight, Vec>); + /// Information regarding the active era (era in used in session). #[derive(Encode, Decode, RuntimeDebug)] pub struct ActiveEraInfo { @@ -994,6 +998,25 @@ decl_storage! { /// /// This is set to v6.0.0 for new networks. StorageVersion build(|_: &GenesisConfig| Releases::V6_0_0): Releases; + + // The next storage items collectively comprise the voter bags: a composite data structure + // designed to allow efficient iteration of the top N voters by stake, mostly. See + // `mod voter_bags` for details. + + /// Which bag currently contains a particular voter. + /// + /// This may not be the appropriate bag for the voter's weight if they have been rewarded or + /// slashed. + VoterBagFor: map hasher(twox_64_concat) T::AccountId => voter_bags::BagIdx; + + /// The head and tail of each bag of voters. + VoterBags: map hasher(twox_64_concat) voter_bags::BagIdx => voter_bags::Bag; + + /// The nodes comprising each bag. + VoterNodes: double_map + hasher(twox_64_concat) voter_bags::BagIdx, + hasher(twox_64_concat) T::AccountId => + Option>; } add_extra_genesis { config(stakers): @@ -2495,7 +2518,9 @@ impl Module { /// auto-chilled. /// /// Note that this is VERY expensive. Use with care. - pub fn get_npos_voters() -> Vec<(T::AccountId, VoteWeight, Vec)> { + pub fn get_npos_voters( + maybe_max_len: Option, + ) -> Vec> { let weight_of = Self::slashable_balance_of_fn(); let mut all_voters = Vec::new(); @@ -2561,7 +2586,7 @@ impl frame_election_provider_support::ElectionDataProvider) -> data_provider::Result<(Vec, Weight)> { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs new file mode 100644 index 0000000000000..aa54e61ea5161 --- /dev/null +++ b/frame/staking/src/voter_bags.rs @@ -0,0 +1,101 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implement a data structure designed for the properties that: +//! +//! - It's efficient to insert or remove a voter +//! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of +//! voters doesn't particularly matter. + +use crate::{ + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VotingDataOf, VoteWeight, +}; +use codec::{Encode, Decode}; +use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; +use sp_runtime::SaturatedConversion; +use sp_std::marker::PhantomData; + +/// Index type for a bag. +pub type BagIdx = u8; + +/// Given a certain vote weight, which bag should this voter contain? +fn bag_for(weight: VoteWeight) -> BagIdx { + todo!("geometric series of some description; ask alfonso") +} + +/// Type of voter. +/// +/// Similar to [`crate::StakerStatus`], but somewhat more limited. +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum VoterType { + Validator, + Nominator, +} + +/// Fundamental information about a voter. +#[derive(Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Voter { + /// Account Id of this voter + pub id: AccountId, + /// Whether the voter is a validator or nominator + pub voter_type: VoterType, +} + +pub type VoterOf = Voter>; + +/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. +/// +/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of +/// arbitrary and unbounded length, all having a vote weight within a particular constant range. +/// This structure means that voters can be added and removed in `O(1)` time. +/// +/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. +/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall +/// stake decreases as successive bags are reached. This means that it is valid to truncate +/// iteration at any desired point; only those voters in the lowest bag (who are known to have +/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire +/// for fairness and the requirement for efficiency. +pub struct VoterList(PhantomData); + +pub type VoterListOf = VoterList>; + +/// A Bag contains a singly-linked list of voters. +/// +/// Note that we maintain both head and tail pointers. While it would be possible to get away +/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's +/// more desirable to ensure that there is some element of first-come, first-serve to the list's +/// iteration so that there's no incentive to churn voter positioning to improve the chances of +/// appearing within the voter set. +#[derive(Default, Encode, Decode)] +pub struct Bag { + head: Option, + tail: Option, +} + +pub type BagOf = Bag>; + +/// A Node is the fundamental element comprising the singly-linked lists which for each bag. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Node { + voter: Voter, + next: Option, +} + +pub type NodeOf = Node>; From bdfabd2bd40bba47dad02fef80e44920022be900 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:03:46 +0200 Subject: [PATCH 002/241] storage getters and setters for Bag, Node --- frame/staking/src/lib.rs | 7 +- frame/staking/src/voter_bags.rs | 119 +++++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 34 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 42dc062569e27..7dd8c19cd4487 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1003,6 +1003,9 @@ decl_storage! { // designed to allow efficient iteration of the top N voters by stake, mostly. See // `mod voter_bags` for details. + /// How many voters are registered. + VoterCount: u32; + /// Which bag currently contains a particular voter. /// /// This may not be the appropriate bag for the voter's weight if they have been rewarded or @@ -1010,13 +1013,13 @@ decl_storage! { VoterBagFor: map hasher(twox_64_concat) T::AccountId => voter_bags::BagIdx; /// The head and tail of each bag of voters. - VoterBags: map hasher(twox_64_concat) voter_bags::BagIdx => voter_bags::Bag; + VoterBags: map hasher(twox_64_concat) voter_bags::BagIdx => Option>; /// The nodes comprising each bag. VoterNodes: double_map hasher(twox_64_concat) voter_bags::BagIdx, hasher(twox_64_concat) T::AccountId => - Option>; + Option>; } add_extra_genesis { config(stakers): diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index aa54e61ea5161..8670197398ecf 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,7 +22,7 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VotingDataOf, VoteWeight, + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VoterBagFor, VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; @@ -33,32 +33,15 @@ use sp_std::marker::PhantomData; pub type BagIdx = u8; /// Given a certain vote weight, which bag should this voter contain? -fn bag_for(weight: VoteWeight) -> BagIdx { +fn notional_bag_for(weight: VoteWeight) -> BagIdx { todo!("geometric series of some description; ask alfonso") } -/// Type of voter. -/// -/// Similar to [`crate::StakerStatus`], but somewhat more limited. -#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] -#[cfg_attr(feature = "std", derive(Debug))] -pub enum VoterType { - Validator, - Nominator, -} - -/// Fundamental information about a voter. -#[derive(Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct Voter { - /// Account Id of this voter - pub id: AccountId, - /// Whether the voter is a validator or nominator - pub voter_type: VoterType, +/// Find the actual bag containing the current voter. +fn current_bag_for(id: &AccountIdOf) -> Option { + VoterBagFor::::try_get(id).ok() } -pub type VoterOf = Voter>; - /// Data structure providing efficient mostly-accurate selection of the top N voters by stake. /// /// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of @@ -71,9 +54,13 @@ pub type VoterOf = Voter>; /// iteration at any desired point; only those voters in the lowest bag (who are known to have /// relatively little power to affect the outcome) can be excluded. This satisfies both the desire /// for fairness and the requirement for efficiency. -pub struct VoterList(PhantomData); +pub struct VoterList(PhantomData); -pub type VoterListOf = VoterList>; +impl VoterList { + pub fn decode_len() -> Option { + crate::VoterCount::try_get().ok().map(|n| n.saturated_into()) + } +} /// A Bag contains a singly-linked list of voters. /// @@ -83,19 +70,87 @@ pub type VoterListOf = VoterList>; /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. #[derive(Default, Encode, Decode)] -pub struct Bag { - head: Option, - tail: Option, +pub struct Bag { + head: Option>, + tail: Option>, + + #[codec(skip)] + bag_idx: BagIdx, } -pub type BagOf = Bag>; +impl Bag { + /// Get a bag by idx. + pub fn get(bag_idx: BagIdx) -> Option> { + crate::VoterBags::::try_get(bag_idx).ok().map(|mut bag| { + bag.bag_idx = bag_idx; + bag + }) + } + + /// Put the bag back into storage. + pub fn put(self) { + crate::VoterBags::::insert(self.bag_idx, self); + } +} /// A Node is the fundamental element comprising the singly-linked lists which for each bag. #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug))] -pub struct Node { - voter: Voter, - next: Option, +pub struct Node { + voter: Voter>, + next: Option>, + + #[codec(skip)] + bag_idx: BagIdx, } -pub type NodeOf = Node>; +impl Node { + /// Get a node by bag idx and account id. + pub fn get(bag_idx: BagIdx, account_id: &AccountIdOf) -> Option> { + crate::VoterNodes::::try_get(&bag_idx, account_id).ok().map(|mut node| { + node.bag_idx = bag_idx; + node + }) + } + + /// Get a node by account id. + /// + /// Note that this must perform two storage lookups: one to identify which bag is appropriate, + /// and another to actually fetch the node. + pub fn from_id(account_id: &AccountIdOf) -> Option> { + let bag = current_bag_for::(account_id)?; + Self::get(bag, account_id) + } + + /// Get a node by account id, assuming it's in the same bag as this node. + pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { + Self::get(self.bag_idx, account_id) + } + + /// Put the node back into storage. + pub fn put(self) { + crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); + } +} + +/// Fundamental information about a voter. +#[derive(Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Voter { + /// Account Id of this voter + pub id: AccountId, + /// Whether the voter is a validator or nominator + pub voter_type: VoterType, +} + +pub type VoterOf = Voter>; + +/// Type of voter. +/// +/// Similar to [`crate::StakerStatus`], but somewhat more limited. +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum VoterType { + Validator, + Nominator, +} From 450c3440c4df356c50dda55ef3dfad3d67ab6d6e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:27:23 +0200 Subject: [PATCH 003/241] node getters and iterators for VoterList, Bag --- frame/staking/src/voter_bags.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 8670197398ecf..39d469dee8055 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -60,6 +60,14 @@ impl VoterList { pub fn decode_len() -> Option { crate::VoterCount::try_get().ok().map(|n| n.saturated_into()) } + + /// Iterate over all nodes in all bags in the voter list. + /// + /// Note that this exhaustively attempts to try all possible bag indices. Full iteration can be + /// expensive; it's recommended to limit the number of items with `.take(n)`. + pub fn iter() -> impl Iterator> { + (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) + } } /// A Bag contains a singly-linked list of voters. @@ -91,6 +99,21 @@ impl Bag { pub fn put(self) { crate::VoterBags::::insert(self.bag_idx, self); } + + /// Get the head node in this bag. + pub fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + } + + /// Get the tail node in this bag. + pub fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + } + + /// Iterate over the nodes in this bag. + pub fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } } /// A Node is the fundamental element comprising the singly-linked lists which for each bag. @@ -131,6 +154,11 @@ impl Node { pub fn put(self) { crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); } + + /// Get the next node in the bag. + pub fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| self.in_bag(id)) + } } /// Fundamental information about a voter. From 21e79eece2afb6c37400f94bc2371082e8a465dd Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:37:11 +0200 Subject: [PATCH 004/241] simplify get_npos_voters --- frame/staking/src/lib.rs | 37 +++++++-------------------------- frame/staking/src/voter_bags.rs | 34 ++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 7dd8c19cd4487..0c7d7ce543f6c 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2521,39 +2521,18 @@ impl Module { /// auto-chilled. /// /// Note that this is VERY expensive. Use with care. - pub fn get_npos_voters( - maybe_max_len: Option, - ) -> Vec> { - let weight_of = Self::slashable_balance_of_fn(); - let mut all_voters = Vec::new(); - - for (validator, _) in >::iter() { - // append self vote - let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]); - all_voters.push(self_vote); - } + pub fn get_npos_voters(maybe_max_len: Option) -> Vec> { + let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); + let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); + let weight_of = Self::slashable_balance_of_fn(); // collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - for (nominator, nominations) in >::iter() { - let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; - - // Filter out nomination targets which were nominated before the most recent - // slashing span. - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); - - if !targets.is_empty() { - let vote_weight = weight_of(&nominator); - all_voters.push((nominator, vote_weight, targets)) - } - } - - all_voters + voter_bags::VoterList::::iter() + .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) + .take(wanted_voters) + .collect() } pub fn get_npos_targets() -> Vec { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 39d469dee8055..39d34d4d3f749 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,12 +22,13 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VoterBagFor, VotingDataOf, VoteWeight, + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, Pallet, VoterBagFor, + VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; use sp_runtime::SaturatedConversion; -use sp_std::marker::PhantomData; +use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; /// Index type for a bag. pub type BagIdx = u8; @@ -159,6 +160,35 @@ impl Node { pub fn next(&self) -> Option> { self.next.as_ref().and_then(|id| self.in_bag(id)) } + + /// Get this voter's voting data. + pub fn voting_data( + &self, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + slashing_spans: &BTreeMap, SlashingSpans>, + ) -> Option> { + match self.voter.voter_type { + VoterType::Validator => Some(( + self.voter.id.clone(), + weight_of(&self.voter.id), + vec![self.voter.id.clone()], + )), + VoterType::Nominator => { + let Nominations { submitted_in, mut targets, .. } = + Nominators::::get(self.voter.id.clone())?; + // Filter out nomination targets which were nominated before the most recent + // slashing span. + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + + (!targets.is_empty()) + .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) + } + } + } } /// Fundamental information about a voter. From f4998e2926e6ab1f40880c08bad2050e72137c07 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:43:39 +0200 Subject: [PATCH 005/241] simplify fn voters --- frame/staking/src/lib.rs | 24 +++++++++--------------- frame/staking/src/voter_bags.rs | 6 +++--- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 0c7d7ce543f6c..476e33602053f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2521,8 +2521,10 @@ impl Module { /// auto-chilled. /// /// Note that this is VERY expensive. Use with care. - pub fn get_npos_voters(maybe_max_len: Option) -> Vec> { - let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); + pub fn get_npos_voters( + maybe_max_len: Option, + voter_count: usize, + ) -> Vec> { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); let weight_of = Self::slashable_balance_of_fn(); @@ -2551,24 +2553,16 @@ impl frame_election_provider_support::ElectionDataProvider, ) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec)>, Weight)> { - // NOTE: reading these counts already needs to iterate a lot of storage keys, but they get - // cached. This is okay for the case of `Ok(_)`, but bad for `Err(_)`, as the trait does not - // report weight in failures. - let nominator_count = >::iter().count(); - let validator_count = >::iter().count(); - let voter_count = nominator_count.saturating_add(validator_count); - - if maybe_max_len.map_or(false, |max_len| voter_count > max_len) { - return Err("Voter snapshot too big"); - } + let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( - nominator_count as u32, - validator_count as u32, + // TODO: fix the weight calculation here + 0 as u32, + voter_count as u32, slashing_span_count as u32, ); - Ok((Self::get_npos_voters(maybe_max_len), weight)) + Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) } fn targets(maybe_max_len: Option) -> data_provider::Result<(Vec, Weight)> { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 39d34d4d3f749..f4a48de8a0afa 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,11 +22,11 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, Pallet, VoterBagFor, + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, VoterBagFor, VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; -use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; +use frame_support::{StorageMap, StorageValue, StorageDoubleMap}; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; @@ -34,7 +34,7 @@ use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; pub type BagIdx = u8; /// Given a certain vote weight, which bag should this voter contain? -fn notional_bag_for(weight: VoteWeight) -> BagIdx { +fn notional_bag_for(_weight: VoteWeight) -> BagIdx { todo!("geometric series of some description; ask alfonso") } From 8960daac0e6c54016f39adb1560058a09c8d3725 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 16:24:17 +0200 Subject: [PATCH 006/241] VoterList::insert --- frame/staking/src/voter_bags.rs | 61 +++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index f4a48de8a0afa..53dd68a0f5b14 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -26,7 +26,7 @@ use crate::{ VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; -use frame_support::{StorageMap, StorageValue, StorageDoubleMap}; +use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; @@ -69,6 +69,31 @@ impl VoterList { pub fn iter() -> impl Iterator> { (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) } + + /// Insert a new voter into the appropriate bag in the voter list. + pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { + Self::insert_many(sp_std::iter::once(voter), weight_of) + } + + /// Insert several voters into the appropriate bags in the voter list. + /// + /// This is more efficient than repeated calls to `Self::insert`. + pub fn insert_many( + voters: impl IntoIterator>, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + ) { + let mut bags = BTreeMap::new(); + + for voter in voters.into_iter() { + let weight = weight_of(&voter.id); + let bag = notional_bag_for(weight); + bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); + } + + for (_, bag) in bags { + bag.put(); + } + } } /// A Bag contains a singly-linked list of voters. @@ -78,7 +103,7 @@ impl VoterList { /// more desirable to ensure that there is some element of first-come, first-serve to the list's /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. -#[derive(Default, Encode, Decode)] +#[derive(DefaultNoBound, Encode, Decode)] pub struct Bag { head: Option>, tail: Option>, @@ -96,6 +121,11 @@ impl Bag { }) } + /// Get a bag by idx or make it, appropriately initialized. + pub fn get_or_make(bag_idx: BagIdx) -> Bag { + Self::get(bag_idx).unwrap_or(Bag { bag_idx, ..Default::default() }) + } + /// Put the bag back into storage. pub fn put(self) { crate::VoterBags::::insert(self.bag_idx, self); @@ -115,6 +145,33 @@ impl Bag { pub fn iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } + + /// Insert a new voter into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + fn insert(&mut self, voter: VoterOf) { + let id = voter.id.clone(); + + // insert the actual voter + let voter_node = Node:: { voter, next: None, bag_idx: self.bag_idx }; + voter_node.put(); + + // update the previous tail + if let Some(mut tail) = self.tail() { + tail.next = Some(id.clone()); + tail.put(); + } + + // update the internal bag links + if self.head.is_none() { + self.head = Some(id.clone()); + } + self.tail = Some(id); + } } /// A Node is the fundamental element comprising the singly-linked lists which for each bag. From f9d061d55f896b0a7f185d930150f869a98dda0e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 16:51:29 +0200 Subject: [PATCH 007/241] VoterList::remove --- frame/staking/src/voter_bags.rs | 63 +++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 53dd68a0f5b14..556c7893b9a13 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -94,9 +94,54 @@ impl VoterList { bag.put(); } } + + /// Remove a voter (by id) from the voter list. + pub fn remove(voter: &AccountIdOf) { + Self::remove_many(sp_std::iter::once(voter)) + } + + /// Remove many voters (by id) from the voter list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + pub fn remove_many<'a>(voters: impl IntoIterator>) { + let mut bags = BTreeMap::new(); + + for voter_id in voters.into_iter() { + let node = match Node::::from_id(voter_id) { + Some(node) => node, + None => continue, + }; + + // modify the surrounding nodes + if let Some(mut prev) = node.prev() { + prev.next = node.next.clone(); + prev.put(); + } + if let Some(mut next) = node.next() { + next.prev = node.prev.clone(); + next.put(); + } + + // clear the bag head/tail pointers as necessary + let mut bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); + if bag.head.as_ref() == Some(voter_id) { + bag.head = node.next; + } + if bag.tail.as_ref() == Some(voter_id) { + bag.tail = node.prev; + } + + // now get rid of the node itself + crate::VoterNodes::::remove(node.bag_idx, voter_id); + } + + for (_, bag) in bags { + bag.put(); + } + } } -/// A Bag contains a singly-linked list of voters. +/// A Bag is a doubly-linked list of voters. /// /// Note that we maintain both head and tail pointers. While it would be possible to get away /// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's @@ -155,13 +200,19 @@ impl Bag { /// `self.put()` after use. fn insert(&mut self, voter: VoterOf) { let id = voter.id.clone(); + let tail = self.tail(); // insert the actual voter - let voter_node = Node:: { voter, next: None, bag_idx: self.bag_idx }; + let voter_node = Node:: { + voter, + prev: tail.as_ref().map(|prev| prev.voter.id.clone()), + next: None, + bag_idx: self.bag_idx, + }; voter_node.put(); // update the previous tail - if let Some(mut tail) = self.tail() { + if let Some(mut tail) = tail { tail.next = Some(id.clone()); tail.put(); } @@ -179,6 +230,7 @@ impl Bag { #[cfg_attr(feature = "std", derive(Debug))] pub struct Node { voter: Voter>, + prev: Option>, next: Option>, #[codec(skip)] @@ -213,6 +265,11 @@ impl Node { crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); } + /// Get the previous node in the bag. + pub fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| self.in_bag(id)) + } + /// Get the next node in the bag. pub fn next(&self) -> Option> { self.next.as_ref().and_then(|id| self.in_bag(id)) From 1e7dfd96a898eb2e01e430625e0cc635f1539682 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 14 Jun 2021 12:58:30 +0200 Subject: [PATCH 008/241] nodes are doubly-linked --- frame/staking/src/voter_bags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 556c7893b9a13..77becfb5f78c6 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -225,7 +225,7 @@ impl Bag { } } -/// A Node is the fundamental element comprising the singly-linked lists which for each bag. +/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug))] pub struct Node { From c6bcc760bf4fcad12d8f06225532b9262b98c345 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 14 Jun 2021 16:53:16 +0200 Subject: [PATCH 009/241] impl fn update_position_for, update voter counts appropriately --- frame/staking/src/voter_bags.rs | 125 +++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 26 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index b430494677168..fb44797bcd2ef 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -58,6 +58,7 @@ fn current_bag_for(id: &AccountIdOf) -> Option { pub struct VoterList(PhantomData); impl VoterList { + /// Decode the length of the voter list. pub fn decode_len() -> Option { crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) } @@ -83,16 +84,20 @@ impl VoterList { weight_of: impl Fn(&T::AccountId) -> VoteWeight, ) { let mut bags = BTreeMap::new(); + let mut count = 0; for voter in voters.into_iter() { let weight = weight_of(&voter.id); let bag = notional_bag_for(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); + count += 1; } for (_, bag) in bags { bag.put(); } + + crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); } /// Remove a voter (by id) from the voter list. @@ -105,31 +110,18 @@ impl VoterList { /// This is more efficient than repeated calls to `Self::remove`. pub fn remove_many<'a>(voters: impl IntoIterator>) { let mut bags = BTreeMap::new(); + let mut count = 0; for voter_id in voters.into_iter() { let node = match Node::::from_id(voter_id) { Some(node) => node, None => continue, }; - - // modify the surrounding nodes - if let Some(mut prev) = node.prev() { - prev.next = node.next.clone(); - prev.put(); - } - if let Some(mut next) = node.next() { - next.prev = node.prev.clone(); - next.put(); - } + count += 1; // clear the bag head/tail pointers as necessary - let mut bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); - if bag.head.as_ref() == Some(voter_id) { - bag.head = node.next; - } - if bag.tail.as_ref() == Some(voter_id) { - bag.tail = node.prev; - } + let bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); + bag.remove_node(&node); // now get rid of the node itself crate::VoterNodes::::remove(node.bag_idx, voter_id); @@ -138,6 +130,39 @@ impl VoterList { for (_, bag) in bags { bag.put(); } + + crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_sub(count)); + } + + /// Update a voter's position in the voter list. + /// + /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `true` if the voter moved. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub fn update_position_for( + mut node: Node, + weight_of: impl Fn(&AccountIdOf) -> VoteWeight, + ) -> bool { + let was_misplaced = node.is_misplaced(&weight_of); + if was_misplaced { + // clear the old bag head/tail pointers as necessary + if let Some(mut bag) = Bag::::get(node.bag_idx) { + bag.remove_node(&node); + bag.put(); + } + + // put the voter into the appropriate new bag + node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); + let mut bag = Bag::::get_or_make(node.bag_idx); + bag.insert_node(node); + bag.put(); + } + was_misplaced } } @@ -199,20 +224,30 @@ impl Bag { /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. fn insert(&mut self, voter: VoterOf) { - let id = voter.id.clone(); - let tail = self.tail(); - - // insert the actual voter - let voter_node = Node:: { + self.insert_node(Node:: { voter, - prev: tail.as_ref().map(|prev| prev.voter.id.clone()), + prev: None, next: None, bag_idx: self.bag_idx, - }; - voter_node.put(); + }); + } + + /// Insert a voter node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + fn insert_node(&mut self, mut node: Node) { + let id = node.voter.id.clone(); + + node.prev = self.tail.clone(); + node.next = None; + node.put(); // update the previous tail - if let Some(mut tail) = tail { + if let Some(mut tail) = self.tail() { tail.next = Some(id.clone()); tail.put(); } @@ -223,6 +258,25 @@ impl Bag { } self.tail = Some(id); } + + /// Remove a voter node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the voter in + /// the first place. Generally, use [`VoterList::remove`] instead. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()` and `node.put()` after use. + fn remove_node(&mut self, node: &Node) { + node.excise(); + + // clear the bag head/tail pointers as necessary + if self.head.as_ref() == Some(&node.voter.id) { + self.head = node.next.clone(); + } + if self.tail.as_ref() == Some(&node.voter.id) { + self.tail = node.prev.clone(); + } + } } /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. @@ -303,6 +357,25 @@ impl Node { } } } + + /// Remove this node from the linked list. + /// + /// Modifies storage, but only modifies the adjacent nodes. Does not modify `self` or any bag. + fn excise(&self) { + if let Some(mut prev) = self.prev() { + prev.next = self.next.clone(); + prev.put(); + } + if let Some(mut next) = self.next() { + next.prev = self.prev.clone(); + next.put(); + } + } + + /// `true` when this voter is in the wrong bag. + pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { + notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx + } } /// Fundamental information about a voter. From 82154dd8cd7543a1c9802b96db163426361f455d Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 14 Jun 2021 16:58:42 +0200 Subject: [PATCH 010/241] keep VoterBagFor updated --- frame/staking/src/voter_bags.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index fb44797bcd2ef..3a6495c49b745 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -125,6 +125,7 @@ impl VoterList { // now get rid of the node itself crate::VoterNodes::::remove(node.bag_idx, voter_id); + crate::VoterBagFor::::remove(voter_id); } for (_, bag) in bags { @@ -256,7 +257,9 @@ impl Bag { if self.head.is_none() { self.head = Some(id.clone()); } - self.tail = Some(id); + self.tail = Some(id.clone()); + + crate::VoterBagFor::::insert(id, self.bag_idx); } /// Remove a voter node from this bag. From 294cd99a0b82dd9a8ee6d7c182842721ca41f32e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 15 Jun 2021 13:53:42 +0200 Subject: [PATCH 011/241] manipulate VoterList everywhere which seems relevant --- frame/staking/src/lib.rs | 21 ++++++++++++++------- frame/staking/src/voter_bags.rs | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 8bc9cd20dd38a..a2d32aa3f23b8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -312,11 +312,9 @@ use sp_staking::{ SessionIndex, offence::{OnOffenceHandler, OffenceDetails, Offence, ReportOffence, OffenceError}, }; -use frame_system::{ - ensure_signed, ensure_root, pallet_prelude::*, - offchain::SendTransactionTypes, -}; +use frame_system::{ensure_signed, ensure_root, pallet_prelude::*, offchain::SendTransactionTypes}; use frame_election_provider_support::{ElectionProvider, VoteWeight, Supports, data_provider}; +use voter_bags::{VoterList, VoterType}; pub use weights::WeightInfo; pub use pallet::*; @@ -1646,6 +1644,7 @@ pub mod pallet { let stash = &ledger.stash; >::remove(stash); >::insert(stash, prefs); + VoterList::::insert_as(stash, VoterType::Validator); Ok(()) } @@ -1699,6 +1698,7 @@ pub mod pallet { >::remove(stash); >::insert(stash, &nominations); + VoterList::::insert_as(stash, VoterType::Nominator); Ok(()) } @@ -1724,6 +1724,7 @@ pub mod pallet { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; Self::chill_stash(&ledger.stash); + VoterList::::remove(&ledger.stash); Ok(()) } @@ -2308,6 +2309,7 @@ impl Pallet { fn chill_stash(stash: &AccountIdOf) { >::remove(stash); >::remove(stash); + VoterList::::remove(stash); } /// Actually make a payment to a staker. This uses the currency's reward function @@ -2627,6 +2629,8 @@ impl Pallet { >::remove(stash); >::remove(stash); + VoterList::::remove(stash); + frame_system::Pallet::::dec_consumers(stash); Ok(()) @@ -2725,7 +2729,7 @@ impl Pallet { // collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - voter_bags::VoterList::::iter() + VoterList::::iter() .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) .take(wanted_voters) .collect() @@ -2736,7 +2740,8 @@ impl Pallet { } } -impl frame_election_provider_support::ElectionDataProvider, BlockNumberFor> +impl + frame_election_provider_support::ElectionDataProvider, BlockNumberFor> for Pallet { const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; @@ -2747,7 +2752,7 @@ impl frame_election_provider_support::ElectionDataProvider, ) -> data_provider::Result<(Vec<(AccountIdOf, VoteWeight, Vec>)>, Weight)> { - let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); + let voter_count = VoterList::::decode_len().unwrap_or_default(); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( @@ -2819,6 +2824,7 @@ impl frame_election_provider_support::ElectionDataProvider::insert_as(&v, VoterType::Validator); >::insert( v, ValidatorPrefs { commission: Perbill::zero(), blocked: false }, @@ -2840,6 +2846,7 @@ impl frame_election_provider_support::ElectionDataProvider::insert_as(&v, VoterType::Nominator); >::insert( v, Nominations { targets: t, submitted_in: 0, suppressed: false }, diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 3a6495c49b745..f37cff1c99e8c 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,8 +22,8 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, VoterBagFor, - VotingDataOf, VoteWeight, + AccountIdOf, Config, Nominations, Nominators, Pallet, VoteWeight, VoterBagFor, VotingDataOf, + slashing::SlashingSpans, }; use codec::{Encode, Decode}; use frame_support::DefaultNoBound; @@ -71,6 +71,20 @@ impl VoterList { (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) } + /// Insert a new voter into the appropriate bag in the voter list. + /// + /// If the voter is already present in the list, their type will be updated. + /// That case is cheaper than inserting a new voter. + pub fn insert_as(account_id: &AccountIdOf, voter_type: VoterType) { + // if this is an update operation we can complete this easily and cheaply + if !Node::::update_voter_type_for(account_id, voter_type) { + // otherwise, we need to insert from scratch + let weight_of = Pallet::::slashable_balance_of_fn(); + let voter = Voter { id: account_id.clone(), voter_type }; + Self::insert(voter, weight_of); + } + } + /// Insert a new voter into the appropriate bag in the voter list. pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { Self::insert_many(sp_std::iter::once(voter), weight_of) @@ -379,6 +393,21 @@ impl Node { pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx } + + /// Update the voter type associated with a particular node by id. + /// + /// This updates storage immediately. + /// + /// Returns whether the voter existed and was successfully updated. + pub fn update_voter_type_for(account_id: &AccountIdOf, voter_type: VoterType) -> bool { + let node = Self::from_id(account_id); + let existed = node.is_some(); + if let Some(mut node) = node { + node.voter.voter_type = voter_type; + node.put(); + } + existed + } } /// Fundamental information about a voter. From 9787daf1372742473d6098696a3bfd7c20017975 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 15 Jun 2021 16:59:43 +0200 Subject: [PATCH 012/241] start sketching out `notional_bag_for` --- frame/staking/src/voter_bags.rs | 69 ++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index f37cff1c99e8c..80d30fc51249a 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -33,9 +33,68 @@ use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; /// Index type for a bag. pub type BagIdx = u8; +/// How many bags there are +const N_BAGS: BagIdx = 200; + /// Given a certain vote weight, which bag should this voter contain? -fn notional_bag_for(_weight: VoteWeight) -> BagIdx { - todo!("geometric series of some description; ask alfonso") +/// +/// Bags are separated by fixed thresholds. The formula for a threshold is, for `t` in `0..=N_BAGS`: +/// +/// 10 ^ ((t / 10) - 10) +/// +/// Given `N_BAGS == 200`, this means that the lowest threshold is `10^-10`, and the highest is +/// `10^10`. A given vote weight always fits into the bag `t` such that `threshold(t-1) <= weight < +/// threshold(t)`. We can determine an appropriate value for `t` by binary search. +/// +/// It is important for the correctness of the iteration algorithm that the bags of highest value +/// have the lowest threshold. Therefore, the appropriate `BagIdx` for a given value `T` is +/// `N_BAGS - t`. +fn notional_bag_for(weight: VoteWeight) -> BagIdx { + // the input to this threshold function is _not_ reversed; `threshold(0) == 0` + let threshold = |bag: BagIdx| -> u64 { + // The goal is to segment the full range of `u64` into `N_BAGS`, such that `threshold(0) != 0`, + // `threshold(N_BAGS) == u64::MAX`, and for all `t` in `0..N_BAGS`, + // `threshold(t + 1) as f64 / threshold(t) as f64 == CONSTANT_RATIO`. For `N_BAGS == 200`, + // `CONSTANT_RATIO ~= 1.25`. + // + // The natural, simple implementation here is + // + // ```rust + // // float exp and ln are not constant functions, unfortunately + // let CONSTANT_RATIO = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); + // CONSTANT_RATIO.powi(bag.into()).into() + // ``` + // + // Unfortunately, that doesn't quite work, for two reasons: + // + // - floats are nondeterministic and not allowed on the blockchain + // - f64 has insufficient capacity to completely and accurately compute f64 + // + // Perhaps the answer is going to end up being to bring in a fixed-point bignum implementation. + // See: https://docs.rs/fixed/1.9.0/fixed/struct.FixedU128.html + todo!() + }; + + // `want_bag` is the highest bag for which `threshold(want_bag) >= weight` + let want_bag = { + // TODO: use a binary search instead + let mut t = N_BAGS; + while t > 0 { + if threshold(t) >= weight { + break; + } + t -= 1; + } + t + }; + + debug_assert!( + (0..=N_BAGS).contains(&want_bag), + "must have computed a valid want_bag" + ); + + // reverse the index so that iteration works properly + N_BAGS - want_bag } /// Find the actual bag containing the current voter. @@ -102,7 +161,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); - let bag = notional_bag_for(weight); + let bag = notional_bag_for::(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -172,7 +231,7 @@ impl VoterList { } // put the voter into the appropriate new bag - node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); + node.bag_idx = notional_bag_for::(weight_of(&node.voter.id)); let mut bag = Bag::::get_or_make(node.bag_idx); bag.insert_node(node); bag.put(); @@ -391,7 +450,7 @@ impl Node { /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx + notional_bag_for::(weight_of(&self.voter.id)) != self.bag_idx } /// Update the voter type associated with a particular node by id. From a857f596348359761acff8ea4f3ba171ce787965 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 15 Jun 2021 17:01:11 +0200 Subject: [PATCH 013/241] `notional_bag_for` doesn't need a type parameter --- frame/staking/src/voter_bags.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 80d30fc51249a..009a943b644a8 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -49,7 +49,7 @@ const N_BAGS: BagIdx = 200; /// It is important for the correctness of the iteration algorithm that the bags of highest value /// have the lowest threshold. Therefore, the appropriate `BagIdx` for a given value `T` is /// `N_BAGS - t`. -fn notional_bag_for(weight: VoteWeight) -> BagIdx { +fn notional_bag_for(weight: VoteWeight) -> BagIdx { // the input to this threshold function is _not_ reversed; `threshold(0) == 0` let threshold = |bag: BagIdx| -> u64 { // The goal is to segment the full range of `u64` into `N_BAGS`, such that `threshold(0) != 0`, @@ -161,7 +161,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); - let bag = notional_bag_for::(weight); + let bag = notional_bag_for(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -231,7 +231,7 @@ impl VoterList { } // put the voter into the appropriate new bag - node.bag_idx = notional_bag_for::(weight_of(&node.voter.id)); + node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); let mut bag = Bag::::get_or_make(node.bag_idx); bag.insert_node(node); bag.put(); @@ -450,7 +450,7 @@ impl Node { /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for::(weight_of(&self.voter.id)) != self.bag_idx + notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx } /// Update the voter type associated with a particular node by id. From 1a50bbafa958bc8315c9e0dd9b89b2a08b071123 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 17 Jun 2021 17:04:32 +0200 Subject: [PATCH 014/241] implement voter bag thresholds in terms of a precomputed list --- frame/staking/Cargo.toml | 5 + frame/staking/src/bin/make_bags.rs | 40 +++ .../src/{voter_bags.rs => voter_bags/mod.rs} | 70 ++---- frame/staking/src/voter_bags/thresholds.rs | 229 ++++++++++++++++++ 4 files changed, 288 insertions(+), 56 deletions(-) create mode 100644 frame/staking/src/bin/make_bags.rs rename frame/staking/src/{voter_bags.rs => voter_bags/mod.rs} (86%) create mode 100644 frame/staking/src/voter_bags/thresholds.rs diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 908e361e667e3..ce0a394df4167 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -70,3 +70,8 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] +make-bags = [] + +[[bin]] +name = "make_bags" +required-features = ["make-bags", "std"] diff --git a/frame/staking/src/bin/make_bags.rs b/frame/staking/src/bin/make_bags.rs new file mode 100644 index 0000000000000..664c7b5470ba0 --- /dev/null +++ b/frame/staking/src/bin/make_bags.rs @@ -0,0 +1,40 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Make the set of voting bag thresholds to be used in `voter_bags.rs`. + +use pallet_staking::voter_bags::N_BAGS; + +fn main() { + let ratio = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); + println!("pub const CONSTANT_RATIO: f64 = {};", ratio); + + let mut thresholds = Vec::with_capacity(N_BAGS as usize); + + while thresholds.len() < N_BAGS as usize { + let prev_item: u64 = thresholds.last().copied().unwrap_or_default(); + let item = (prev_item as f64 * ratio).max(prev_item as f64 + 1.0); + thresholds.push(item as u64); + } + + *thresholds.last_mut().unwrap() = u64::MAX; + + println!("pub const THRESHOLDS: [u64; {}] = {:#?};", N_BAGS, thresholds); + + debug_assert_eq!(thresholds.len(), N_BAGS as usize); + debug_assert_eq!(*thresholds.last().unwrap(), u64::MAX); +} diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags/mod.rs similarity index 86% rename from frame/staking/src/voter_bags.rs rename to frame/staking/src/voter_bags/mod.rs index 009a943b644a8..cb8909c74475a 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -21,6 +21,9 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. +mod thresholds; + +use thresholds::THRESHOLDS; use crate::{ AccountIdOf, Config, Nominations, Nominators, Pallet, VoteWeight, VoterBagFor, VotingDataOf, slashing::SlashingSpans, @@ -34,67 +37,22 @@ use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; pub type BagIdx = u8; /// How many bags there are -const N_BAGS: BagIdx = 200; +pub const N_BAGS: BagIdx = 200; /// Given a certain vote weight, which bag should this voter contain? /// -/// Bags are separated by fixed thresholds. The formula for a threshold is, for `t` in `0..=N_BAGS`: -/// -/// 10 ^ ((t / 10) - 10) +/// Bags are separated by fixed thresholds. To the extent possible, each threshold is a constant +/// small multiple of the one before it. That ratio is [`thresholds::CONSTANT_RATIO`]. The exception +/// are the smallest bags, which are each at least 1 greater than the previous, and the largest bag, +/// which is defined as `u64::MAX`. /// -/// Given `N_BAGS == 200`, this means that the lowest threshold is `10^-10`, and the highest is -/// `10^10`. A given vote weight always fits into the bag `t` such that `threshold(t-1) <= weight < -/// threshold(t)`. We can determine an appropriate value for `t` by binary search. -/// -/// It is important for the correctness of the iteration algorithm that the bags of highest value -/// have the lowest threshold. Therefore, the appropriate `BagIdx` for a given value `T` is -/// `N_BAGS - t`. +/// Bags are arranged such that `bags[0]` is the largest bag, and `bags[N_BAGS-1]` is the smallest. fn notional_bag_for(weight: VoteWeight) -> BagIdx { - // the input to this threshold function is _not_ reversed; `threshold(0) == 0` - let threshold = |bag: BagIdx| -> u64 { - // The goal is to segment the full range of `u64` into `N_BAGS`, such that `threshold(0) != 0`, - // `threshold(N_BAGS) == u64::MAX`, and for all `t` in `0..N_BAGS`, - // `threshold(t + 1) as f64 / threshold(t) as f64 == CONSTANT_RATIO`. For `N_BAGS == 200`, - // `CONSTANT_RATIO ~= 1.25`. - // - // The natural, simple implementation here is - // - // ```rust - // // float exp and ln are not constant functions, unfortunately - // let CONSTANT_RATIO = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); - // CONSTANT_RATIO.powi(bag.into()).into() - // ``` - // - // Unfortunately, that doesn't quite work, for two reasons: - // - // - floats are nondeterministic and not allowed on the blockchain - // - f64 has insufficient capacity to completely and accurately compute f64 - // - // Perhaps the answer is going to end up being to bring in a fixed-point bignum implementation. - // See: https://docs.rs/fixed/1.9.0/fixed/struct.FixedU128.html - todo!() - }; - - // `want_bag` is the highest bag for which `threshold(want_bag) >= weight` - let want_bag = { - // TODO: use a binary search instead - let mut t = N_BAGS; - while t > 0 { - if threshold(t) >= weight { - break; - } - t -= 1; - } - t - }; - - debug_assert!( - (0..=N_BAGS).contains(&want_bag), - "must have computed a valid want_bag" - ); - - // reverse the index so that iteration works properly - N_BAGS - want_bag + let raw_bag = match THRESHOLDS.binary_search(&weight) { + Ok(bag) => bag, + Err(bag) => bag, + } as BagIdx; + N_BAGS - raw_bag } /// Find the actual bag containing the current voter. diff --git a/frame/staking/src/voter_bags/thresholds.rs b/frame/staking/src/voter_bags/thresholds.rs new file mode 100644 index 0000000000000..710403d75e64f --- /dev/null +++ b/frame/staking/src/voter_bags/thresholds.rs @@ -0,0 +1,229 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Generated voter bag thresholds. + +use super::N_BAGS; + +/// Ratio between adjacent bags; +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 1.2483305489016119; + +/// Upper thresholds for each bag. +pub const THRESHOLDS: [u64; N_BAGS as usize] = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 11, + 13, + 16, + 19, + 23, + 28, + 34, + 42, + 52, + 64, + 79, + 98, + 122, + 152, + 189, + 235, + 293, + 365, + 455, + 567, + 707, + 882, + 1101, + 1374, + 1715, + 2140, + 2671, + 3334, + 4161, + 5194, + 6483, + 8092, + 10101, + 12609, + 15740, + 19648, + 24527, + 30617, + 38220, + 47711, + 59559, + 74349, + 92812, + 115860, + 144631, + 180547, + 225382, + 281351, + 351219, + 438437, + 547314, + 683228, + 852894, + 1064693, + 1329088, + 1659141, + 2071156, + 2585487, + 3227542, + 4029039, + 5029572, + 6278568, + 7837728, + 9784075, + 12213759, + 15246808, + 19033056, + 23759545, + 29659765, + 37025190, + 46219675, + 57697432, + 72025466, + 89911589, + 112239383, + 140111850, + 174905902, + 218340380, + 272560966, + 340246180, + 424739700, + 530215542, + 661884258, + 826250339, + 1031433539, + 1287569995, + 1607312958, + 2006457867, + 2504722650, + 3126721800, + 3903182340, + 4872461752, + 6082442853, + 7592899225, + 9478448057, + 11832236265, + 14770541991, + 18438518791, + 23017366283, + 28733281486, + 35868633049, + 44775910382, + 55895136784, + 69775606782, + 87103021514, + 108733362657, + 135735178289, + 169442369618, + 211520086272, + 264046985399, + 329617918218, + 411472116776, + 513653213392, + 641208997818, + 800440780206, + 999214678517, + 1247350208103, + 1557105369953, + 1943782201171, + 2426482702132, + 3029052483452, + 3781258749319, + 4720260810076, + 5892445768000, + 7355720059940, + 9182370059991, + 11462633057206, + 14309155016159, + 17862555335640, + 22298373506924, + 27835740839511, + 34748205641269, + 43377246621511, + 54149142084871, + 67596028261358, + 84382187063069, + 105336861893959, + 131495222627659, + 164149503440725, + 204912839732087, + 255798957699744, + 319321653273781, + 398618974707429, + 497608243499122, + 621179571745225, + 775437435763184, + 968002239825113, + 1208386767378873, + 1508466116607513, + 1883064335344139, + 2350686735357198, + 2934434062644189, + 3663143684136207, + 4572814165923224, + 5708383617772005, + 7125949654914296, + 8895540644164415, + 11104575135106362, + 13862180373726516, + 17304583234907174, + 21601839888145304, + 26966236644853160, + 33662776992680304, + 42022272880825152, + 52457686971413784, + 65484533171133904, + 81746343238087392, + 102046457525101200, + 127387710335774608, + 159021970366777056, + 198511983555374656, + 247808573395228608, + 309347012448991104, + 386167325851522816, + 482064469848099072, + 601775804251442048, + 751215120036911616, + 937764783138868096, + 1170640426476344320, + 1461346206149632000, + 1824243111658058240, + 2277258404906088192, + 2842771234587226112, + 3548718175673985024, + 4429973308136232448, + 5530071011365192704, + 6903356581082402816, + 8617670910126150656, + 10757701857491230720, + 13429167864681918464, + 18446744073709551615, +]; From 721c2cb5c665b0d6f9df91b675b876868d757bad Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 18 Jun 2021 12:27:39 +0200 Subject: [PATCH 015/241] fix a failing test --- Cargo.lock | 1 + .../election-provider-support/src/onchain.rs | 7 +--- frame/staking/Cargo.toml | 1 + frame/staking/src/tests.rs | 9 ++-- primitives/npos-elections/src/lib.rs | 42 +++++++++++++++---- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb944b782abd9..034414e675fd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5501,6 +5501,7 @@ dependencies = [ "sp-application-crypto", "sp-core", "sp-io", + "sp-npos-elections", "sp-runtime", "sp-staking", "sp-std", diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index e034a9c36a8ac..89358fce12af8 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -77,11 +77,8 @@ impl ElectionProvider for OnChainSequen let (desired_targets, _) = Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; - let mut stake_map: BTreeMap = BTreeMap::new(); - - voters.iter().for_each(|(v, s, _)| { - stake_map.insert(v.clone(), *s); - }); + let stake_map: BTreeMap = + voters.iter().map(|(v, s, _)| (v.clone(), *s)).collect(); let stake_of = |w: &T::AccountId| -> VoteWeight { stake_map.get(w).cloned().unwrap_or_default() diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index ce0a394df4167..3e080f2537014 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -37,6 +37,7 @@ rand_chacha = { version = "0.2", default-features = false, optional = true } sp-storage = { version = "3.0.0", path = "../../primitives/storage" } sp-tracing = { version = "3.0.0", path = "../../primitives/tracing" } sp-core = { version = "3.0.0", path = "../../primitives/core" } +sp-npos-elections = { version = "3.0.0", path = "../../primitives/npos-elections", features = ["mocks"] } pallet-balances = { version = "3.0.0", path = "../balances" } pallet-timestamp = { version = "3.0.0", path = "../timestamp" } pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 5d42d866b1336..e7a7fe1178387 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -19,6 +19,7 @@ use super::{*, Event}; use mock::*; +use sp_npos_elections::supports_eq_unordered; use sp_runtime::{ assert_eq_error_rate, traits::{BadOrigin, Dispatchable}, @@ -1905,13 +1906,13 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider_elected() { // winners should be 21 and 11. let supports = ::ElectionProvider::elect().unwrap().0; - assert_eq!( - supports, - vec![ + assert!(supports_eq_unordered( + &supports, + &vec![ (11, Support { total: 1500, voters: vec![(11, 1000), (1, 500)] }), (21, Support { total: 2500, voters: vec![(21, 1000), (3, 1000), (1, 500)] }) ], - ); + )); }); } diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index c1cf41a40f2b5..5563b75e69eff 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -476,6 +476,18 @@ pub struct Support { pub voters: Vec<(AccountId, ExtendedBalance)>, } +#[cfg(feature = "mocks")] +impl Support { + /// `true` when the support is identical except for the ordering of the voters. + pub fn eq_unordered(&self, other: &Self) -> bool { + self.total == other.total && { + let my_voters: BTreeMap<_, _> = self.voters.iter().cloned().collect(); + let other_voters: BTreeMap<_, _> = other.voters.iter().cloned().collect(); + my_voters == other_voters + } + } +} + /// A target-major representation of the the election outcome. /// /// Essentially a flat variant of [`SupportMap`]. @@ -483,6 +495,18 @@ pub struct Support { /// The main advantage of this is that it is encodable. pub type Supports = Vec<(A, Support)>; +#[cfg(feature = "mocks")] +pub fn supports_eq_unordered(a: &Supports, b: &Supports) -> bool { + let map: BTreeMap<_, _> = a.iter().cloned().collect(); + b.iter().all(|(id, b_support)| { + let a_support = match map.get(id) { + Some(support) => support, + None => return false, + }; + a_support.eq_unordered(b_support) + }) +} + /// Linkage from a winner to their [`Support`]. /// /// This is more helpful than a normal [`Supports`] as it allows faster error checking. @@ -504,12 +528,12 @@ impl FlattenSupportMap for SupportMap { /// /// The list of winners is basically a redundancy for error checking only; It ensures that all the /// targets pointed to by the [`Assignment`] are present in the `winners`. -pub fn to_support_map( - winners: &[A], - assignments: &[StakedAssignment], -) -> Result, Error> { +pub fn to_support_map( + winners: &[AccountId], + assignments: &[StakedAssignment], +) -> Result, Error> { // Initialize the support of each candidate. - let mut supports = >::new(); + let mut supports = >::new(); winners.iter().for_each(|e| { supports.insert(e.clone(), Default::default()); }); @@ -532,10 +556,10 @@ pub fn to_support_map( /// flat vector. /// /// Similar to [`to_support_map`], `winners` is used for error checking. -pub fn to_supports( - winners: &[A], - assignments: &[StakedAssignment], -) -> Result, Error> { +pub fn to_supports( + winners: &[AccountId], + assignments: &[StakedAssignment], +) -> Result, Error> { to_support_map(winners, assignments).map(FlattenSupportMap::flatten) } From 92220ec6213f609dee9c528f501e8d7f73e4e4bd Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 18 Jun 2021 13:29:50 +0200 Subject: [PATCH 016/241] fix the rest of the non-benchmark tests --- frame/staking/src/tests.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index e7a7fe1178387..767192bd131ce 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -514,8 +514,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 800, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 400 }, IndividualExposure { who: 1, value: 400 }, + IndividualExposure { who: 3, value: 400 }, ] }, ); @@ -525,8 +525,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 1200, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 600 }, IndividualExposure { who: 1, value: 600 }, + IndividualExposure { who: 3, value: 600 }, ] }, ); @@ -1860,13 +1860,13 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider() { // winners should be 21 and 31. Otherwise this election is taking duplicates into // account. let supports = ::ElectionProvider::elect().unwrap().0; - assert_eq!( - supports, - vec![ + assert!(supports_eq_unordered( + &supports, + &vec![ (21, Support { total: 1800, voters: vec![(21, 1000), (3, 400), (1, 400)] }), (31, Support { total: 2200, voters: vec![(31, 1000), (3, 600), (1, 600)] }) ], - ); + )); }); } @@ -3958,7 +3958,7 @@ mod election_data_provider { #[test] fn respects_len_limits() { ExtBuilder::default().build_and_execute(|| { - assert_eq!(Staking::voters(Some(1)).unwrap_err(), "Voter snapshot too big"); + assert_eq!(Staking::voters(Some(1)).unwrap().0.len(), 1); assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); }); } From 033d87faec30339328f87d2ec7abeb67bb3a3e42 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 10:14:31 +0200 Subject: [PATCH 017/241] revert an accidental change --- frame/staking/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index fed1e2df6598f..a1b3e948303cc 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2987,8 +2987,9 @@ impl Pallet { .collect() } - pub fn get_npos_targets() -> Vec> { - >::iter().map(|(v, _)| v).collect::>() + /// This is a very expensive function and result should be cached versus being called multiple times. + pub fn get_npos_targets() -> Vec { + Validators::::iter().map(|(v, _)| v).collect::>() } /// This function will add a nominator to the `Nominators` storage map, From 18c8eefdaaa75a52e9580032ee77eba0872fc5ec Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 16:43:44 +0200 Subject: [PATCH 018/241] adapt tests from benchmarking.rs for easier debugging --- frame/staking/src/tests.rs | 95 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 767192bd131ce..9746b059df105 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -17,6 +17,8 @@ //! Tests for the module. +use crate::voter_bags::Node; + use super::{*, Event}; use mock::*; use sp_npos_elections::supports_eq_unordered; @@ -30,6 +32,7 @@ use frame_support::{ traits::{Currency, ReservableCurrency, OnInitialize}, weights::{extract_actual_weight, GetDispatchInfo}, }; +use frame_system::RawOrigin; use pallet_balances::Error as BalancesError; use substrate_test_utils::assert_eq_uvec; use frame_election_provider_support::Support; @@ -3860,6 +3863,98 @@ fn on_finalize_weight_is_nonzero() { }) } + + +/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. +#[test] +fn payout_stakers_dead_controller() { + ExtBuilder::default() + .build_and_execute(|| { + // setup + let n = ::MaxNominatorRewardedPerValidator::get() as u32; + let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( + n, + ::MaxNominatorRewardedPerValidator::get() as u32, + true, + RewardDestination::Controller, + ).unwrap(); + + let current_era = CurrentEra::::get().unwrap(); + // set the commission for this particular era as well. + >::insert(current_era, validator.clone(), >::validators(&validator)); + + let caller = crate::benchmarking::whitelisted_caller(); + let validator_controller = >::get(&validator).unwrap(); + let balance_before = ::Currency::free_balance(&validator_controller); + for (_, controller) in &nominators { + let balance = ::Currency::free_balance(controller); + assert!(balance.is_zero(), "Controller has balance, but should be dead."); + } + + // benchmark + Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); + + // verify + let balance_after = ::Currency::free_balance(&validator_controller); + assert!( + balance_before < balance_after, + "Balance of validator controller should have increased after payout.", + ); + for (_, controller) in &nominators { + let balance = ::Currency::free_balance(controller); + if balance.is_zero() { + dbg!(controller, Node::::from_id(controller)); + } + assert!(!balance.is_zero(), "Payout not given to controller."); + } + }); +} + +/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. +#[test] +fn payout_stakers_alive_staked() { + ExtBuilder::default() + .build_and_execute(|| { + // setup + let n = ::MaxNominatorRewardedPerValidator::get() as u32; + let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( + n, + ::MaxNominatorRewardedPerValidator::get() as u32, + false, + RewardDestination::Staked, + ).unwrap(); + + let current_era = CurrentEra::::get().unwrap(); + // set the commission for this particular era as well. + >::insert(current_era, validator.clone(), >::validators(&validator)); + + let caller = crate::benchmarking::whitelisted_caller(); + let balance_before = ::Currency::free_balance(validator); + let mut nominator_balances_before = Vec::new(); + for (stash, _) in &nominators { + let balance = ::Currency::free_balance(stash); + nominator_balances_before.push(balance); + } + + // benchmark + Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); + + // verify + let balance_after = ::Currency::free_balance(validator); + assert!( + balance_before < balance_after, + "Balance of validator stash should have increased after payout.", + ); + for ((stash, _), balance_before) in nominators.iter().zip(nominator_balances_before.iter()) { + let balance_after = ::Currency::free_balance(stash); + assert!( + balance_before < &balance_after, + "Balance of nominator stash should have increased after payout.", + ); + } + }); +} + mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; From 9bd6682231e263a821399793deac59c12599c87c Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 16:43:50 +0200 Subject: [PATCH 019/241] investigate a theory as to why the payout_stakers tests are failing The thought was that we shouldn't be tracking the stash ID as the VoterList::Nominator; instead, we should track its controller. Unfortunately, that theory was wrong, and this just makes things worse. --- frame/staking/src/benchmarking.rs | 1 + frame/staking/src/lib.rs | 43 +++++++++++++++++++----------- frame/staking/src/slashing.rs | 4 +-- frame/staking/src/testing_utils.rs | 1 + 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 189158cc5643b..1633544861752 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -85,6 +85,7 @@ pub fn create_validator_with_nominators( }; if i < n { Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), vec![stash_lookup.clone()])?; + assert!(crate::voter_bags::Node::::from_id(&n_controller).is_some()); nominators.push((n_stash, n_controller)); } } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index a1b3e948303cc..651bdf597530a 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1761,7 +1761,7 @@ pub mod pallet { ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); let stash = &ledger.stash; - Self::do_remove_nominator(stash); + Self::do_remove_nominator(stash, &controller); Self::do_add_validator(stash, prefs); Ok(()) } @@ -1824,7 +1824,7 @@ pub mod pallet { }; Self::do_remove_validator(stash); - Self::do_add_nominator(stash, nominations); + Self::do_add_nominator(stash, &controller, nominations); Ok(()) } @@ -1849,8 +1849,7 @@ pub mod pallet { pub fn chill(origin: OriginFor) -> DispatchResult { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - Self::chill_stash(&ledger.stash); - VoterList::::remove(&ledger.stash); + Self::chill_stash(&ledger.stash, controller.into()); Ok(()) } @@ -2361,7 +2360,7 @@ pub mod pallet { ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); } - Self::chill_stash(&stash); + Self::chill_stash(&stash, controller.into()); Ok(()) } } @@ -2524,9 +2523,15 @@ impl Pallet { } /// Chill a stash account. - fn chill_stash(stash: &T::AccountId) { + /// + /// If `controller.is_none()`, looks it up in the ledger. However, if the controller is avilable + /// in the calling scope, it's more efficient to pass it directly. + fn chill_stash(stash: &T::AccountId, controller: Option) { Self::do_remove_validator(stash); - Self::do_remove_nominator(stash); + let controller = controller.unwrap_or_else(|| { + Bonded::::get(stash).expect("TODO: fix this error handling") + }); + Self::do_remove_nominator(stash, &controller); } /// Actually make a payment to a staker. This uses the currency's reward function @@ -2875,7 +2880,7 @@ impl Pallet { >::remove(stash); Self::do_remove_validator(stash); - Self::do_remove_nominator(stash); + Self::do_remove_nominator(stash, &controller); VoterList::::remove(stash); @@ -2996,22 +3001,26 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. - pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { - if !Nominators::::contains_key(who) { + pub fn do_add_nominator( + stash: &T::AccountId, + controller: &T::AccountId, + nominations: Nominations, + ) { + if !Nominators::::contains_key(stash) { CounterForNominators::::mutate(|x| x.saturating_inc()) } - Nominators::::insert(who, nominations); - VoterList::::insert_as(who, VoterType::Nominator); + Nominators::::insert(stash, nominations); + VoterList::::insert_as(controller, VoterType::Nominator); } /// This function will remove a nominator from the `Nominators` storage map, /// and keep track of the `CounterForNominators`. - pub fn do_remove_nominator(who: &T::AccountId) { - if Nominators::::contains_key(who) { - Nominators::::remove(who); + pub fn do_remove_nominator(stash: &T::AccountId, controller: &T::AccountId) { + if Nominators::::contains_key(stash) { + Nominators::::remove(stash); CounterForNominators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); } + VoterList::::remove(controller); } /// This function will add a validator to the `Validators` storage map, @@ -3151,8 +3160,10 @@ impl claimed_rewards: vec![], }, ); + let controller = Bonded::::get(&v).unwrap(); Self::do_add_nominator( &v, + &controller, Nominations { targets: t, submitted_in: 0, suppressed: false }, ); }); diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index 1e959e9341add..ef2ad15aa724f 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -285,7 +285,7 @@ pub(crate) fn compute_slash(params: SlashParams) // chill the validator - it misbehaved in the current span and should // not continue in the next election. also end the slashing span. spans.end_span(now); - >::chill_stash(stash); + >::chill_stash(stash, None); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(stash).unwrap_or(false) { @@ -325,7 +325,7 @@ fn kick_out_if_recent( if spans.era_span(params.slash_era).map(|s| s.index) == Some(spans.span_index()) { spans.end_span(params.now); - >::chill_stash(params.stash); + >::chill_stash(params.stash, None); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(params.stash).unwrap_or(false) { diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index c643cb283373b..f85ae93e2416f 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -33,6 +33,7 @@ pub fn clear_validators_and_nominators() { CounterForValidators::::kill(); Nominators::::remove_all(None); CounterForNominators::::kill(); + VoterBags::::drain(); } /// Grab a funded user. From 7a74fcf0270563e2a54ddcbc47dc384484765c4e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 16:45:52 +0200 Subject: [PATCH 020/241] Revert "investigate a theory as to why the payout_stakers tests are failing" This reverts commit 9bd6682231e263a821399793deac59c12599c87c. --- frame/staking/src/benchmarking.rs | 1 - frame/staking/src/lib.rs | 43 +++++++++++------------------- frame/staking/src/slashing.rs | 4 +-- frame/staking/src/testing_utils.rs | 1 - 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 1633544861752..189158cc5643b 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -85,7 +85,6 @@ pub fn create_validator_with_nominators( }; if i < n { Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), vec![stash_lookup.clone()])?; - assert!(crate::voter_bags::Node::::from_id(&n_controller).is_some()); nominators.push((n_stash, n_controller)); } } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 651bdf597530a..a1b3e948303cc 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1761,7 +1761,7 @@ pub mod pallet { ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); let stash = &ledger.stash; - Self::do_remove_nominator(stash, &controller); + Self::do_remove_nominator(stash); Self::do_add_validator(stash, prefs); Ok(()) } @@ -1824,7 +1824,7 @@ pub mod pallet { }; Self::do_remove_validator(stash); - Self::do_add_nominator(stash, &controller, nominations); + Self::do_add_nominator(stash, nominations); Ok(()) } @@ -1849,7 +1849,8 @@ pub mod pallet { pub fn chill(origin: OriginFor) -> DispatchResult { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - Self::chill_stash(&ledger.stash, controller.into()); + Self::chill_stash(&ledger.stash); + VoterList::::remove(&ledger.stash); Ok(()) } @@ -2360,7 +2361,7 @@ pub mod pallet { ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); } - Self::chill_stash(&stash, controller.into()); + Self::chill_stash(&stash); Ok(()) } } @@ -2523,15 +2524,9 @@ impl Pallet { } /// Chill a stash account. - /// - /// If `controller.is_none()`, looks it up in the ledger. However, if the controller is avilable - /// in the calling scope, it's more efficient to pass it directly. - fn chill_stash(stash: &T::AccountId, controller: Option) { + fn chill_stash(stash: &T::AccountId) { Self::do_remove_validator(stash); - let controller = controller.unwrap_or_else(|| { - Bonded::::get(stash).expect("TODO: fix this error handling") - }); - Self::do_remove_nominator(stash, &controller); + Self::do_remove_nominator(stash); } /// Actually make a payment to a staker. This uses the currency's reward function @@ -2880,7 +2875,7 @@ impl Pallet { >::remove(stash); Self::do_remove_validator(stash); - Self::do_remove_nominator(stash, &controller); + Self::do_remove_nominator(stash); VoterList::::remove(stash); @@ -3001,26 +2996,22 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. - pub fn do_add_nominator( - stash: &T::AccountId, - controller: &T::AccountId, - nominations: Nominations, - ) { - if !Nominators::::contains_key(stash) { + pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { + if !Nominators::::contains_key(who) { CounterForNominators::::mutate(|x| x.saturating_inc()) } - Nominators::::insert(stash, nominations); - VoterList::::insert_as(controller, VoterType::Nominator); + Nominators::::insert(who, nominations); + VoterList::::insert_as(who, VoterType::Nominator); } /// This function will remove a nominator from the `Nominators` storage map, /// and keep track of the `CounterForNominators`. - pub fn do_remove_nominator(stash: &T::AccountId, controller: &T::AccountId) { - if Nominators::::contains_key(stash) { - Nominators::::remove(stash); + pub fn do_remove_nominator(who: &T::AccountId) { + if Nominators::::contains_key(who) { + Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); + VoterList::::remove(who); } - VoterList::::remove(controller); } /// This function will add a validator to the `Validators` storage map, @@ -3160,10 +3151,8 @@ impl claimed_rewards: vec![], }, ); - let controller = Bonded::::get(&v).unwrap(); Self::do_add_nominator( &v, - &controller, Nominations { targets: t, submitted_in: 0, suppressed: false }, ); }); diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index ef2ad15aa724f..1e959e9341add 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -285,7 +285,7 @@ pub(crate) fn compute_slash(params: SlashParams) // chill the validator - it misbehaved in the current span and should // not continue in the next election. also end the slashing span. spans.end_span(now); - >::chill_stash(stash, None); + >::chill_stash(stash); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(stash).unwrap_or(false) { @@ -325,7 +325,7 @@ fn kick_out_if_recent( if spans.era_span(params.slash_era).map(|s| s.index) == Some(spans.span_index()) { spans.end_span(params.now); - >::chill_stash(params.stash, None); + >::chill_stash(params.stash); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(params.stash).unwrap_or(false) { diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index f85ae93e2416f..c643cb283373b 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -33,7 +33,6 @@ pub fn clear_validators_and_nominators() { CounterForValidators::::kill(); Nominators::::remove_all(None); CounterForNominators::::kill(); - VoterBags::::drain(); } /// Grab a funded user. From e0ae1c05651f5f05df27074e06d4cf3163f8940d Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 14:09:26 +0200 Subject: [PATCH 021/241] find threshold more directly This method is known to return values in the range `(0..N_BAGS)`; see https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=b49ec461d667184c6856f8da6a059f85 for demonstration. --- frame/staking/src/voter_bags/mod.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index cb8909c74475a..8046db211f2d2 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -48,11 +48,9 @@ pub const N_BAGS: BagIdx = 200; /// /// Bags are arranged such that `bags[0]` is the largest bag, and `bags[N_BAGS-1]` is the smallest. fn notional_bag_for(weight: VoteWeight) -> BagIdx { - let raw_bag = match THRESHOLDS.binary_search(&weight) { - Ok(bag) => bag, - Err(bag) => bag, - } as BagIdx; - N_BAGS - raw_bag + let raw_bag = + THRESHOLDS.partition_point(|&threshold| weight > threshold) as BagIdx; + N_BAGS - raw_bag - 1 } /// Find the actual bag containing the current voter. From 66996549360742657bc3f72a710aae0144f0d9c2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 14:48:59 +0200 Subject: [PATCH 022/241] fix failing benchmark tests --- frame/staking/src/benchmarking.rs | 8 +++ frame/staking/src/testing_utils.rs | 1 + frame/staking/src/tests.rs | 95 ----------------------------- frame/staking/src/voter_bags/mod.rs | 8 +++ 4 files changed, 17 insertions(+), 95 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 189158cc5643b..0e258491133e2 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -96,6 +96,14 @@ pub fn create_validator_with_nominators( assert_eq!(new_validators.len(), 1); assert_eq!(new_validators[0], v_stash, "Our validator was not selected!"); + assert_eq!( + VoterList::::decode_len().unwrap_or_default() as u32, + CounterForNominators::::get() + CounterForValidators::::get(), + "ensure storage has been mutated coherently", + ); + assert_ne!(CounterForValidators::::get(), 0); + assert_ne!(CounterForNominators::::get(), 0); + assert_ne!(VoterList::::decode_len().unwrap_or_default(), 0); // Give Era Points let reward = EraRewardPoints:: { diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index c643cb283373b..482fcad8fb73f 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -33,6 +33,7 @@ pub fn clear_validators_and_nominators() { CounterForValidators::::kill(); Nominators::::remove_all(None); CounterForNominators::::kill(); + VoterList::::clear(); } /// Grab a funded user. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 9746b059df105..767192bd131ce 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -17,8 +17,6 @@ //! Tests for the module. -use crate::voter_bags::Node; - use super::{*, Event}; use mock::*; use sp_npos_elections::supports_eq_unordered; @@ -32,7 +30,6 @@ use frame_support::{ traits::{Currency, ReservableCurrency, OnInitialize}, weights::{extract_actual_weight, GetDispatchInfo}, }; -use frame_system::RawOrigin; use pallet_balances::Error as BalancesError; use substrate_test_utils::assert_eq_uvec; use frame_election_provider_support::Support; @@ -3863,98 +3860,6 @@ fn on_finalize_weight_is_nonzero() { }) } - - -/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. -#[test] -fn payout_stakers_dead_controller() { - ExtBuilder::default() - .build_and_execute(|| { - // setup - let n = ::MaxNominatorRewardedPerValidator::get() as u32; - let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( - n, - ::MaxNominatorRewardedPerValidator::get() as u32, - true, - RewardDestination::Controller, - ).unwrap(); - - let current_era = CurrentEra::::get().unwrap(); - // set the commission for this particular era as well. - >::insert(current_era, validator.clone(), >::validators(&validator)); - - let caller = crate::benchmarking::whitelisted_caller(); - let validator_controller = >::get(&validator).unwrap(); - let balance_before = ::Currency::free_balance(&validator_controller); - for (_, controller) in &nominators { - let balance = ::Currency::free_balance(controller); - assert!(balance.is_zero(), "Controller has balance, but should be dead."); - } - - // benchmark - Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); - - // verify - let balance_after = ::Currency::free_balance(&validator_controller); - assert!( - balance_before < balance_after, - "Balance of validator controller should have increased after payout.", - ); - for (_, controller) in &nominators { - let balance = ::Currency::free_balance(controller); - if balance.is_zero() { - dbg!(controller, Node::::from_id(controller)); - } - assert!(!balance.is_zero(), "Payout not given to controller."); - } - }); -} - -/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. -#[test] -fn payout_stakers_alive_staked() { - ExtBuilder::default() - .build_and_execute(|| { - // setup - let n = ::MaxNominatorRewardedPerValidator::get() as u32; - let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( - n, - ::MaxNominatorRewardedPerValidator::get() as u32, - false, - RewardDestination::Staked, - ).unwrap(); - - let current_era = CurrentEra::::get().unwrap(); - // set the commission for this particular era as well. - >::insert(current_era, validator.clone(), >::validators(&validator)); - - let caller = crate::benchmarking::whitelisted_caller(); - let balance_before = ::Currency::free_balance(validator); - let mut nominator_balances_before = Vec::new(); - for (stash, _) in &nominators { - let balance = ::Currency::free_balance(stash); - nominator_balances_before.push(balance); - } - - // benchmark - Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); - - // verify - let balance_after = ::Currency::free_balance(validator); - assert!( - balance_before < balance_after, - "Balance of validator stash should have increased after payout.", - ); - for ((stash, _), balance_before) in nominators.iter().zip(nominator_balances_before.iter()) { - let balance_after = ::Currency::free_balance(stash); - assert!( - balance_before < &balance_after, - "Balance of nominator stash should have increased after payout.", - ); - } - }); -} - mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 8046db211f2d2..625497a161ea8 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -73,6 +73,14 @@ fn current_bag_for(id: &AccountIdOf) -> Option { pub struct VoterList(PhantomData); impl VoterList { + /// Remove all data associated with the voter list from storage. + pub fn clear() { + crate::VoterCount::::kill(); + crate::VoterBagFor::::remove_all(None); + crate::VoterBags::::remove_all(None); + crate::VoterNodes::::remove_all(None); + } + /// Decode the length of the voter list. pub fn decode_len() -> Option { crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) From 871b5250ce80c3b8c2b125a20f14aa358e1bd561 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 15:04:05 +0200 Subject: [PATCH 023/241] debug_assert that voter count is accurate --- frame/staking/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index a1b3e948303cc..8c90b9db08dca 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2975,6 +2975,12 @@ impl Pallet { maybe_max_len: Option, voter_count: usize, ) -> Vec> { + debug_assert_eq!( + voter_count, + VoterList::::decode_len().unwrap_or_default(), + "voter_count must be accurate", + ); + let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); let weight_of = Self::slashable_balance_of_fn(); From f2214040dc74dbde1008e2087b9c50a78f2d534a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 15:56:26 +0200 Subject: [PATCH 024/241] add extrinsic to rebag a stash --- frame/staking/src/lib.rs | 34 ++++++++++++++++++++++++++++- frame/staking/src/voter_bags/mod.rs | 22 ++++++++++++++----- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 8c90b9db08dca..7318ee28c95a3 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -314,7 +314,7 @@ use sp_staking::{ }; use frame_system::{ensure_signed, ensure_root, pallet_prelude::*, offchain::SendTransactionTypes}; use frame_election_provider_support::{ElectionProvider, VoteWeight, Supports, data_provider}; -use voter_bags::{VoterList, VoterType}; +use voter_bags::{BagIdx, VoterList, VoterType}; pub use weights::WeightInfo; pub use pallet::*; @@ -1360,6 +1360,10 @@ pub mod pallet { Kicked(T::AccountId, T::AccountId), /// The election failed. No new era is planned. StakingElectionFailed, + /// Attempted to rebag an account. + /// + /// If the second parameter is not `None`, it is the `(from, to)` tuple of bag indices. + Rebag(T::AccountId, Option<(BagIdx, BagIdx)>), } #[pallet::error] @@ -2364,6 +2368,34 @@ pub mod pallet { Self::chill_stash(&stash); Ok(()) } + + /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its + /// stake that it should properly fall into a different bag than its current position. + /// + /// This will adjust its position into the appropriate bag. This will affect its position + /// among the nominator/validator set once the snapshot is prepared for the election. + /// + /// Anyone can call this function about any stash. + // + // TODO: benchmark + #[pallet::weight(0)] + pub fn rebag( + origin: OriginFor, + stash: AccountIdOf, + ) -> DispatchResult { + ensure_signed(origin)?; + + let weight_of = Self::slashable_balance_of_fn(); + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let moved = voter_bags::Node::::from_id(&stash).and_then(|node| { + VoterList::update_position_for(node, weight_of) + }); + + Self::deposit_event(Event::::Rebag(stash, moved)); + + Ok(()) + } } } diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 625497a161ea8..f47ff63b8ba4c 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -185,22 +185,32 @@ impl VoterList { pub fn update_position_for( mut node: Node, weight_of: impl Fn(&AccountIdOf) -> VoteWeight, - ) -> bool { - let was_misplaced = node.is_misplaced(&weight_of); - if was_misplaced { + ) -> Option<(BagIdx, BagIdx)> { + node.is_misplaced(&weight_of).then(move || { + let old_idx = node.bag_idx; + // clear the old bag head/tail pointers as necessary if let Some(mut bag) = Bag::::get(node.bag_idx) { bag.remove_node(&node); bag.put(); + } else { + debug_assert!(false, "every node must have an extant bag associated with it"); + crate::log!( + error, + "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", + node.voter.id, + ); } // put the voter into the appropriate new bag - node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); + let new_idx = notional_bag_for(weight_of(&node.voter.id)); + node.bag_idx = new_idx; let mut bag = Bag::::get_or_make(node.bag_idx); bag.insert_node(node); bag.put(); - } - was_misplaced + + (old_idx, new_idx) + }) } } From 976c41b7c47131ce73012431bde3f4a00aba75c9 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 16:54:40 +0200 Subject: [PATCH 025/241] WIP: rebag benchmark --- frame/staking/src/benchmarking.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 0e258491133e2..2f2b5e8437d05 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -637,6 +637,15 @@ benchmarks! { verify { assert!(!Validators::::contains_key(controller)); } + + rebag { + let caller = whitelisted_caller(); + let (stash, _controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + + // TODO: figure out what's the worst case scenario for this call (lots of other users in + // VoterList?) and arrange for that to be the case + + }: _(RawOrigin::Signed(caller), stash.clone()) } #[cfg(test)] From 37e666ac6b956717120e02a8c91907a20b5d3d71 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 25 Jun 2021 12:02:56 +0200 Subject: [PATCH 026/241] WIP: rebag benchmark Note: this currently fails because the deposit into the target stash appears not to have changed its appropriate bag. --- frame/staking/src/benchmarking.rs | 62 +++++++++++++++++++++++++++-- frame/staking/src/lib.rs | 6 +-- frame/staking/src/voter_bags/mod.rs | 17 ++++++-- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 2f2b5e8437d05..f9be1fb2f61bb 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -639,13 +639,67 @@ benchmarks! { } rebag { - let caller = whitelisted_caller(); - let (stash, _controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // The most expensive case for this call: + // + // - It doesn't matter where in the origin bag the stash lies; the number of reads and + // writes is constant. We can use the case that the stash is the only one in the origin + // bag, for simplicity. + // - The destination bag is not empty, because then we need to update the `next` pointer + // of the previous node in addition to the work we do otherwise. + + // Clean up any existing state. + clear_validators_and_nominators::(); + + use crate::voter_bags::{Bag, Node}; + + // stash and controller control the node account, which is a validator for simplicity of setup + let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + + let prefs = ValidatorPrefs::default(); + whitelist_account!(controller); + Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; - // TODO: figure out what's the worst case scenario for this call (lots of other users in - // VoterList?) and arrange for that to be the case + // create another validator with 3x the stake + create_validators::(1, 300)?; + // update the stash account's value/weight + T::Currency::deposit_into_existing(&stash, T::Currency::minimum_balance() * 200_u32.into())?; + + // verify preconditions + let weight_of = Staking::::weight_of_fn(); + let node = Node::::from_id(&stash).ok_or("node not found for stash")?; + assert!( + node.is_misplaced(&weight_of), + "rebagging only makes sense when a node is misplaced", + ); + assert_eq!( + { + let origin_bag = Bag::::get(node.bag_idx).ok_or("origin bag not found")?; + origin_bag.iter().count() + }, + 1, + "stash should be the only node in origin bag", + ); + assert_ne!( + { + let destination_bag = Bag::::get(node.proper_bag_for()).ok_or("destination bag not found")?; + destination_bag.iter().count() + }, + 0, + "destination bag should not be empty", + ); + drop(node); + + // caller will call rebag + let caller = whitelisted_caller(); + // ensure it's distinct from the other accounts + assert_ne!(caller, stash); + assert_ne!(caller, controller); }: _(RawOrigin::Signed(caller), stash.clone()) + verify { + let node = Node::::from_id(&stash).ok_or("node not found for stash")?; + assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); + } } #[cfg(test)] diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 7318ee28c95a3..9004651d01f28 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2385,10 +2385,10 @@ pub mod pallet { ) -> DispatchResult { ensure_signed(origin)?; - let weight_of = Self::slashable_balance_of_fn(); // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. let moved = voter_bags::Node::::from_id(&stash).and_then(|node| { + let weight_of = Self::weight_of_fn(); VoterList::update_position_for(node, weight_of) }); @@ -2418,7 +2418,7 @@ impl Pallet { /// /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is /// important to be only used while the total issuance is not changing. - pub fn slashable_balance_of_fn() -> Box) -> VoteWeight> { + pub fn weight_of_fn() -> Box) -> VoteWeight> { // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still // compile, while some types in mock fail to resolve. let issuance = T::Currency::total_issuance(); @@ -3015,7 +3015,7 @@ impl Pallet { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); - let weight_of = Self::slashable_balance_of_fn(); + let weight_of = Self::weight_of_fn(); // collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index f47ff63b8ba4c..ca65495faafb5 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -50,7 +50,7 @@ pub const N_BAGS: BagIdx = 200; fn notional_bag_for(weight: VoteWeight) -> BagIdx { let raw_bag = THRESHOLDS.partition_point(|&threshold| weight > threshold) as BagIdx; - N_BAGS - raw_bag - 1 + N_BAGS - 1 - raw_bag } /// Find the actual bag containing the current voter. @@ -102,7 +102,7 @@ impl VoterList { // if this is an update operation we can complete this easily and cheaply if !Node::::update_voter_type_for(account_id, voter_type) { // otherwise, we need to insert from scratch - let weight_of = Pallet::::slashable_balance_of_fn(); + let weight_of = Pallet::::weight_of_fn(); let voter = Voter { id: account_id.clone(), voter_type }; Self::insert(voter, weight_of); } @@ -337,8 +337,9 @@ pub struct Node { prev: Option>, next: Option>, + /// The bag index is not stored in storage, but injected during all fetch operations. #[codec(skip)] - bag_idx: BagIdx, + pub(crate) bag_idx: BagIdx, } impl Node { @@ -441,6 +442,16 @@ impl Node { } existed } + + /// Get the index of the bag that this node _should_ be in, given its vote weight. + /// + /// This is a helper intended only for benchmarking and should not be used in production. + #[cfg(feature = "runtime-benchmarks")] + pub fn proper_bag_for(&self) -> BagIdx { + let weight_of = crate::Pallet::::weight_of_fn(); + let current_weight = weight_of(&self.voter.id); + notional_bag_for(current_weight) + } } /// Fundamental information about a voter. From b75b55be5d558ab74275fa700809e2eeb38a8e32 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 25 Jun 2021 16:13:53 +0200 Subject: [PATCH 027/241] make rebag test into a test case for easier debugging --- frame/staking/src/tests.rs | 81 +++++++++++++++++++++++++++++ frame/staking/src/voter_bags/mod.rs | 4 +- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 767192bd131ce..fe3d758d6b25a 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3860,6 +3860,87 @@ fn on_finalize_weight_is_nonzero() { }) } +#[test] +fn test_rebag() { + use crate::{ + testing_utils::create_stash_controller, + voter_bags::{Bag, Node}, + }; + use frame_benchmarking::{whitelisted_caller}; + use frame_system::RawOrigin; + + const USER_SEED: u32 = 999666; + + let whitelist_account = |account_id: &AccountIdOf| { + frame_benchmarking::benchmarking::add_to_whitelist( + frame_system::Account::::hashed_key_for(account_id).into() + ); + }; + + let make_validator = |n: u32, balance_factor: u32| -> Result<(AccountIdOf, AccountIdOf), &'static str> { + let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); + whitelist_account(&controller); + + let prefs = ValidatorPrefs::default(); + // bond the full value of the stash + Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); + Staking::validate(RawOrigin::Signed(controller.clone()).into(), prefs).unwrap(); + + Ok((stash, controller)) + }; + + ExtBuilder::default().build_and_execute(|| { + // stash controls the node account + let (stash, controller) = make_validator(USER_SEED, 100).unwrap(); + + // create another validator with 3x the stake + let (other_stash, _) = make_validator(USER_SEED + 1, 300).unwrap(); + + // update the stash account's value/weight + ::Currency::make_free_balance_be(&stash, ::Currency::free_balance(&other_stash)); + Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); + + // verify preconditions + let weight_of = Staking::weight_of_fn(); + let node = Node::::from_id(&stash).unwrap(); + assert!( + node.is_misplaced(&weight_of), + "rebagging only makes sense when a node is misplaced", + ); + assert_eq!( + { + let origin_bag = Bag::::get(node.bag_idx).unwrap(); + origin_bag.iter().count() + }, + 1, + "stash should be the only node in origin bag", + ); + let other_node = Node::::from_id(&other_stash).unwrap(); + assert!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); + assert_ne!( + { + let destination_bag = Bag::::get(other_node.bag_idx); + destination_bag.iter().count() + }, + 0, + "destination bag should not be empty", + ); + drop(node); + + // caller will call rebag + let caller = whitelisted_caller(); + // ensure it's distinct from the other accounts + assert_ne!(caller, stash); + assert_ne!(caller, controller); + + // call rebag + Pallet::::rebag(RawOrigin::Signed(caller).into(), stash.clone()).unwrap(); + + let node = Node::::from_id(&stash).unwrap(); + assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); + }); +} + mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index ca65495faafb5..ee754dd795e97 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -390,7 +390,7 @@ impl Node { VoterType::Validator => Some(( self.voter.id.clone(), weight_of(&self.voter.id), - vec![self.voter.id.clone()], + sp_std::vec![self.voter.id.clone()], )), VoterType::Nominator => { let Nominations { submitted_in, mut targets, .. } = @@ -446,7 +446,7 @@ impl Node { /// Get the index of the bag that this node _should_ be in, given its vote weight. /// /// This is a helper intended only for benchmarking and should not be used in production. - #[cfg(feature = "runtime-benchmarks")] + #[cfg(any(test, feature = "runtime-benchmarks"))] pub fn proper_bag_for(&self) -> BagIdx { let weight_of = crate::Pallet::::weight_of_fn(); let current_weight = weight_of(&self.voter.id); From 1cb9e26b51be117d7339bed74cb4d2f59c776053 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 25 Jun 2021 16:27:04 +0200 Subject: [PATCH 028/241] WIP: rebag benchmark --- frame/staking/src/benchmarking.rs | 58 +++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index f9be1fb2f61bb..9abef89056ed8 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -647,45 +647,67 @@ benchmarks! { // - The destination bag is not empty, because then we need to update the `next` pointer // of the previous node in addition to the work we do otherwise. + use crate::voter_bags::{Bag, Node}; + + let dbg_weight = |human: &str, account_id: &T::AccountId| { + let weight_of = Staking::::weight_of_fn(); + sp_runtime::print(human); + sp_runtime::print(weight_of(account_id)); + }; + + let make_validator = |n: u32, balance_factor: u32| -> Result<(T::AccountId, T::AccountId), &'static str> { + let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default())?; + whitelist_account!(controller); + + let prefs = ValidatorPrefs::default(); + // bond the full value of the stash + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; + Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; + + Ok((stash, controller)) + }; + // Clean up any existing state. clear_validators_and_nominators::(); - use crate::voter_bags::{Bag, Node}; - // stash and controller control the node account, which is a validator for simplicity of setup - let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // stash controls the node account + let (stash, controller) = make_validator(USER_SEED, 100)?; - let prefs = ValidatorPrefs::default(); - whitelist_account!(controller); - Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; + dbg_weight("stash", &stash); // create another validator with 3x the stake - create_validators::(1, 300)?; + let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; + + dbg_weight("other stash", &other_stash); // update the stash account's value/weight - T::Currency::deposit_into_existing(&stash, T::Currency::minimum_balance() * 200_u32.into())?; + T::Currency::make_free_balance_be(&stash, T::Currency::free_balance(&other_stash)); + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; + + dbg_weight("stash after deposit", &stash); // verify preconditions let weight_of = Staking::::weight_of_fn(); let node = Node::::from_id(&stash).ok_or("node not found for stash")?; - assert!( + ensure!( node.is_misplaced(&weight_of), "rebagging only makes sense when a node is misplaced", ); - assert_eq!( + ensure!( { let origin_bag = Bag::::get(node.bag_idx).ok_or("origin bag not found")?; - origin_bag.iter().count() + origin_bag.iter().count() == 1 }, - 1, "stash should be the only node in origin bag", ); - assert_ne!( + let other_node = Node::::from_id(&other_stash).ok_or("node not found for other_stash")?; + ensure!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); + ensure!( { let destination_bag = Bag::::get(node.proper_bag_for()).ok_or("destination bag not found")?; - destination_bag.iter().count() + destination_bag.iter().count() != 0 }, - 0, "destination bag should not be empty", ); drop(node); @@ -693,12 +715,12 @@ benchmarks! { // caller will call rebag let caller = whitelisted_caller(); // ensure it's distinct from the other accounts - assert_ne!(caller, stash); - assert_ne!(caller, controller); + ensure!(caller != stash, "caller must not be the same as the stash"); + ensure!(caller != controller, "caller must not be the same as the controller"); }: _(RawOrigin::Signed(caller), stash.clone()) verify { let node = Node::::from_id(&stash).ok_or("node not found for stash")?; - assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); + ensure!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); } } From 27cbb79e2c16999fea18863de0f0f47a23f6906f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 14:54:04 +0200 Subject: [PATCH 029/241] fix rebag benchmark After much trouble, it turns out that the reason this benchmark wasn't working properly was that it's not safe to assume that `!0_u32` is more than the actual free balance in an account. --- frame/staking/src/benchmarking.rs | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 4e82b40ab5c9a..98fd4bb50004e 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -649,19 +649,14 @@ benchmarks! { use crate::voter_bags::{Bag, Node}; - let dbg_weight = |human: &str, account_id: &T::AccountId| { - let weight_of = Staking::::weight_of_fn(); - sp_runtime::print(human); - sp_runtime::print(weight_of(account_id)); - }; - let make_validator = |n: u32, balance_factor: u32| -> Result<(T::AccountId, T::AccountId), &'static str> { let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default())?; whitelist_account!(controller); let prefs = ValidatorPrefs::default(); // bond the full value of the stash - Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; + let free_balance = T::Currency::free_balance(&stash); + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), free_balance)?; Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; Ok((stash, controller)) @@ -674,18 +669,13 @@ benchmarks! { // stash controls the node account let (stash, controller) = make_validator(USER_SEED, 100)?; - dbg_weight("stash", &stash); - // create another validator with 3x the stake let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; - dbg_weight("other stash", &other_stash); - // update the stash account's value/weight - T::Currency::make_free_balance_be(&stash, T::Currency::free_balance(&other_stash)); - Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; - - dbg_weight("stash after deposit", &stash); + let other_free_balance = T::Currency::free_balance(&other_stash); + T::Currency::make_free_balance_be(&stash, other_free_balance); + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), other_free_balance)?; // verify preconditions let weight_of = Staking::::weight_of_fn(); From b6508617781bd4efac9d767950dc8ba456bfcc7f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 15:13:49 +0200 Subject: [PATCH 030/241] reduce diff --- frame/staking/src/lib.rs | 126 +++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index fd9ae7b101c8b..ba8e012c8643b 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -964,12 +964,12 @@ pub mod pallet { /// invulnerables) and restricted to testnets. #[pallet::storage] #[pallet::getter(fn invulnerables)] - pub type Invulnerables = StorageValue<_, Vec>, ValueQuery>; + pub type Invulnerables = StorageValue<_, Vec, ValueQuery>; /// Map from all locked "stash" accounts to the controller account. #[pallet::storage] #[pallet::getter(fn bonded)] - pub type Bonded = StorageMap<_, Twox64Concat, AccountIdOf, AccountIdOf>; + pub type Bonded = StorageMap<_, Twox64Concat, T::AccountId, T::AccountId>; /// The minimum active bond to become and maintain the role of a nominator. #[pallet::storage] @@ -984,8 +984,8 @@ pub mod pallet { #[pallet::getter(fn ledger)] pub type Ledger = StorageMap< _, - Blake2_128Concat, AccountIdOf, - StakingLedger, BalanceOf>, + Blake2_128Concat, T::AccountId, + StakingLedger>, >; /// Where the reward payment should be made. Keyed by stash. @@ -993,8 +993,8 @@ pub mod pallet { #[pallet::getter(fn payee)] pub type Payee = StorageMap< _, - Twox64Concat, AccountIdOf, - RewardDestination>, + Twox64Concat, T::AccountId, + RewardDestination, ValueQuery, >; @@ -1003,7 +1003,7 @@ pub mod pallet { /// When updating this storage item, you must also update the `CounterForValidators`. #[pallet::storage] #[pallet::getter(fn validators)] - pub type Validators = StorageMap<_, Twox64Concat, AccountIdOf, ValidatorPrefs, ValueQuery>; + pub type Validators = StorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; /// A tracker to keep count of the number of items in the `Validators` map. #[pallet::storage] @@ -1020,7 +1020,7 @@ pub mod pallet { /// When updating this storage item, you must also update the `CounterForNominators`. #[pallet::storage] #[pallet::getter(fn nominators)] - pub type Nominators = StorageMap<_, Twox64Concat, AccountIdOf, Nominations>>; + pub type Nominators = StorageMap<_, Twox64Concat, T::AccountId, Nominations>; /// A tracker to keep count of the number of items in the `Nominators` map. #[pallet::storage] @@ -1067,8 +1067,8 @@ pub mod pallet { pub type ErasStakers = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, - Exposure, BalanceOf>, + Twox64Concat, T::AccountId, + Exposure>, ValueQuery, >; @@ -1088,8 +1088,8 @@ pub mod pallet { pub type ErasStakersClipped = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, - Exposure, BalanceOf>, + Twox64Concat, T::AccountId, + Exposure>, ValueQuery, >; @@ -1104,7 +1104,7 @@ pub mod pallet { pub type ErasValidatorPrefs = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, + Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery, >; @@ -1123,7 +1123,7 @@ pub mod pallet { pub type ErasRewardPoints = StorageMap< _, Twox64Concat, EraIndex, - EraRewardPoints>, + EraRewardPoints, ValueQuery, >; @@ -1156,7 +1156,7 @@ pub mod pallet { pub type UnappliedSlashes = StorageMap< _, Twox64Concat, EraIndex, - Vec, BalanceOf>>, + Vec>>, ValueQuery, >; @@ -1173,7 +1173,7 @@ pub mod pallet { pub(crate) type ValidatorSlashInEra = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, + Twox64Concat, T::AccountId, (Perbill, BalanceOf), >; @@ -1182,20 +1182,20 @@ pub mod pallet { pub(crate) type NominatorSlashInEra = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, + Twox64Concat, T::AccountId, BalanceOf, >; /// Slashing spans for stash accounts. #[pallet::storage] - pub(crate) type SlashingSpans = StorageMap<_, Twox64Concat, AccountIdOf, slashing::SlashingSpans>; + pub(crate) type SlashingSpans = StorageMap<_, Twox64Concat, T::AccountId, slashing::SlashingSpans>; /// Records information about the maximum slash of a stash within a slashing span, /// as well as how much reward has been paid out. #[pallet::storage] pub(crate) type SpanSlash = StorageMap< _, - Twox64Concat, (AccountIdOf, slashing::SpanIndex), + Twox64Concat, (T::AccountId, slashing::SpanIndex), slashing::SpanRecord>, ValueQuery, >; @@ -1264,7 +1264,7 @@ pub mod pallet { pub history_depth: u32, pub validator_count: u32, pub minimum_validator_count: u32, - pub invulnerables: Vec>, + pub invulnerables: Vec, pub force_era: Forcing, pub slash_reward_fraction: Perbill, pub canceled_payout: BalanceOf, @@ -1336,17 +1336,17 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] - #[pallet::metadata(AccountIdOf = "AccountId", BalanceOf = "Balance")] + #[pallet::metadata(T::AccountId = "AccountId", BalanceOf = "Balance")] pub enum Event { /// The era payout has been set; the first balance is the validator-payout; the second is /// the remainder from the maximum amount of reward. /// \[era_index, validator_payout, remainder\] EraPayout(EraIndex, BalanceOf, BalanceOf), /// The staker has been rewarded by this amount. \[stash, amount\] - Reward(AccountIdOf, BalanceOf), + Reward(T::AccountId, BalanceOf), /// One validator (and its nominators) has been slashed by the given amount. /// \[validator, amount\] - Slash(AccountIdOf, BalanceOf), + Slash(T::AccountId, BalanceOf), /// An old slashing report from a prior era was discarded because it could /// not be processed. \[session_index\] OldSlashingReportDiscarded(SessionIndex), @@ -1356,12 +1356,12 @@ pub mod pallet { /// /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, /// it will not be emitted for staking rewards when they are added to stake. - Bonded(AccountIdOf, BalanceOf), + Bonded(T::AccountId, BalanceOf), /// An account has unbonded this amount. \[stash, amount\] - Unbonded(AccountIdOf, BalanceOf), + Unbonded(T::AccountId, BalanceOf), /// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance` /// from the unlocking queue. \[stash, amount\] - Withdrawn(AccountIdOf, BalanceOf), + Withdrawn(T::AccountId, BalanceOf), /// A nominator has been kicked from a validator. \[nominator, stash\] Kicked(T::AccountId, T::AccountId), /// The election failed. No new era is planned. @@ -1504,7 +1504,7 @@ pub mod pallet { origin: OriginFor, controller: ::Source, #[pallet::compact] value: BalanceOf, - payee: RewardDestination>, + payee: RewardDestination, ) -> DispatchResult { let stash = ensure_signed(origin)?; @@ -1830,7 +1830,7 @@ pub mod pallet { } else { Err(Error::::BadTarget.into()) })) - .collect::>, _>>()?; + .collect::, _>>()?; let nominations = Nominations { targets, @@ -1889,7 +1889,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_payee())] pub fn set_payee( origin: OriginFor, - payee: RewardDestination>, + payee: RewardDestination, ) -> DispatchResult { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; @@ -2039,7 +2039,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_invulnerables(invulnerables.len() as u32))] pub fn set_invulnerables( origin: OriginFor, - invulnerables: Vec>, + invulnerables: Vec, ) -> DispatchResult { ensure_root(origin)?; >::put(invulnerables); @@ -2059,7 +2059,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::force_unstake(*num_slashing_spans))] pub fn force_unstake( origin: OriginFor, - stash: AccountIdOf, + stash: T::AccountId, num_slashing_spans: u32, ) -> DispatchResult { ensure_root(origin)?; @@ -2161,7 +2161,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::payout_stakers_alive_staked(T::MaxNominatorRewardedPerValidator::get()))] pub fn payout_stakers( origin: OriginFor, - validator_stash: AccountIdOf, + validator_stash: T::AccountId, era: EraIndex, ) -> DispatchResultWithPostInfo { ensure_signed(origin)?; @@ -2262,7 +2262,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::reap_stash(*num_slashing_spans))] pub fn reap_stash( _origin: OriginFor, - stash: AccountIdOf, + stash: T::AccountId, num_slashing_spans: u32, ) -> DispatchResult { let at_minimum = T::Currency::total_balance(&stash) == T::Currency::minimum_balance(); @@ -2293,7 +2293,7 @@ pub mod pallet { for nom_stash in who.into_iter() .map(T::Lookup::lookup) - .collect::>, _>>()? + .collect::, _>>()? .into_iter() { Nominators::::mutate(&nom_stash, |maybe_nom| if let Some(ref mut nom) = maybe_nom { @@ -2437,14 +2437,14 @@ pub mod pallet { impl Pallet { /// The total balance that can be slashed from a stash account as of right now. - pub fn slashable_balance_of(stash: &AccountIdOf) -> BalanceOf { + pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { // Weight note: consider making the stake accessible through stash. Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() } /// Internal impl of [`Self::slashable_balance_of`] that returns [`VoteWeight`]. pub fn slashable_balance_of_vote_weight( - stash: &AccountIdOf, + stash: &T::AccountId, issuance: BalanceOf, ) -> VoteWeight { T::CurrencyToVote::to_vote(Self::slashable_balance_of(stash), issuance) @@ -2454,16 +2454,16 @@ impl Pallet { /// /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is /// important to be only used while the total issuance is not changing. - pub fn weight_of_fn() -> Box) -> VoteWeight> { + pub fn weight_of_fn() -> Box VoteWeight> { // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still // compile, while some types in mock fail to resolve. let issuance = T::Currency::total_issuance(); - Box::new(move |who: &AccountIdOf| -> VoteWeight { + Box::new(move |who: &T::AccountId| -> VoteWeight { Self::slashable_balance_of_vote_weight(who, issuance) }) } - fn do_payout_stakers(validator_stash: AccountIdOf, era: EraIndex) -> DispatchResultWithPostInfo { + fn do_payout_stakers(validator_stash: T::AccountId, era: EraIndex) -> DispatchResultWithPostInfo { // Validate input data let current_era = CurrentEra::::get().ok_or( Error::::InvalidEraToReward.with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) @@ -2579,8 +2579,8 @@ impl Pallet { /// /// This will also update the stash lock. fn update_ledger( - controller: &AccountIdOf, - ledger: &StakingLedger, BalanceOf> + controller: &T::AccountId, + ledger: &StakingLedger> ) { T::Currency::set_lock( STAKING_ID, @@ -2599,7 +2599,7 @@ impl Pallet { /// Actually make a payment to a staker. This uses the currency's reward function /// to pay the right payee for the given staker account. - fn make_payout(stash: &AccountIdOf, amount: BalanceOf) -> Option> { + fn make_payout(stash: &T::AccountId, amount: BalanceOf) -> Option> { let dest = Self::payee(stash); match dest { RewardDestination::Controller => Self::bonded(stash) @@ -2893,8 +2893,8 @@ impl Pallet { /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a /// [`Exposure`]. fn collect_exposures( - supports: Supports>, - ) -> Vec<(AccountIdOf, Exposure, BalanceOf>)> { + supports: Supports, + ) -> Vec<(T::AccountId, Exposure>)> { let total_issuance = T::Currency::total_issuance(); let to_currency = |e: frame_election_provider_support::ExtendedBalance| { T::CurrencyToVote::to_currency(e, total_issuance) @@ -2923,7 +2923,7 @@ impl Pallet { let exposure = Exposure { own, others, total }; (validator, exposure) }) - .collect::, Exposure<_, _>)>>() + .collect::)>>() } /// Remove all associated data of a stash account from the staking system. @@ -2933,7 +2933,7 @@ impl Pallet { /// This is called: /// - after a `withdraw_unbonded()` call that frees all of a stash's bonded balance. /// - through `reap_stash()` if the balance has fallen to zero (through slashing). - fn kill_stash(stash: &AccountIdOf, num_slashing_spans: u32) -> DispatchResult { + fn kill_stash(stash: &T::AccountId, num_slashing_spans: u32) -> DispatchResult { let controller = >::get(stash).ok_or(Error::::NotStash)?; slashing::clear_stash_metadata::(stash, num_slashing_spans)?; @@ -2991,7 +2991,7 @@ impl Pallet { /// /// COMPLEXITY: Complexity is `number_of_validator_to_reward x current_elected_len`. pub fn reward_by_ids( - validators_points: impl IntoIterator, u32)> + validators_points: impl IntoIterator ) { if let Some(active_era) = Self::active_era() { >::mutate(active_era.index, |era_rewards| { @@ -3014,8 +3014,8 @@ impl Pallet { #[cfg(feature = "runtime-benchmarks")] pub fn add_era_stakers( current_era: EraIndex, - controller: AccountIdOf, - exposure: Exposure, BalanceOf>, + controller: T::AccountId, + exposure: Exposure>, ) { >::insert(¤t_era, &controller, &exposure); } @@ -3112,7 +3112,7 @@ impl Pallet { } impl - frame_election_provider_support::ElectionDataProvider, BlockNumberFor> + frame_election_provider_support::ElectionDataProvider> for Pallet { const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; @@ -3184,8 +3184,8 @@ impl #[cfg(any(feature = "runtime-benchmarks", test))] fn put_snapshot( - voters: Vec<(AccountIdOf, VoteWeight, Vec>)>, - targets: Vec>, + voters: Vec<(T::AccountId, VoteWeight, Vec)>, + targets: Vec, target_stake: Option, ) { use sp_std::convert::TryFrom; @@ -3259,12 +3259,12 @@ impl pallet_session::SessionManager for Pallet { } } -impl historical::SessionManager, Exposure, BalanceOf>> +impl historical::SessionManager>> for Pallet { fn new_session( new_index: SessionIndex, - ) -> Option, Exposure, BalanceOf>)>> { + ) -> Option>)>> { >::new_session(new_index).map(|validators| { let current_era = Self::current_era() // Must be some as a new era has been created. @@ -3302,14 +3302,14 @@ impl historical::SessionManager, Exposure pallet_authorship::EventHandler, T::BlockNumber> for Pallet +impl pallet_authorship::EventHandler for Pallet where T: Config + pallet_authorship::Config + pallet_session::Config, { - fn note_author(author: AccountIdOf) { + fn note_author(author: T::AccountId) { Self::reward_by_ids(vec![(author, 20)]) } - fn note_uncle(author: AccountIdOf, _age: T::BlockNumber) { + fn note_uncle(author: T::AccountId, _age: T::BlockNumber) { Self::reward_by_ids(vec![ (>::author(), 2), (author, 1) @@ -3321,8 +3321,8 @@ where /// if any. pub struct StashOf(sp_std::marker::PhantomData); -impl Convert, Option>> for StashOf { - fn convert(controller: AccountIdOf) -> Option> { +impl Convert> for StashOf { + fn convert(controller: T::AccountId) -> Option { >::ledger(&controller).map(|l| l.stash) } } @@ -3334,10 +3334,10 @@ impl Convert, Option>> for StashOf { /// `active_era`. It can differ from the latest planned exposure in `current_era`. pub struct ExposureOf(sp_std::marker::PhantomData); -impl Convert, Option, BalanceOf>>> +impl Convert>>> for ExposureOf { - fn convert(validator: AccountIdOf) -> Option, BalanceOf>> { + fn convert(validator: T::AccountId) -> Option>> { >::active_era() .map(|active_era| >::eras_stakers(active_era.index, &validator)) } @@ -3345,7 +3345,7 @@ impl Convert, Option, BalanceO /// This is intended to be used with `FilterHistoricalOffences`. impl - OnOffenceHandler, pallet_session::historical::IdentificationTuple, Weight> + OnOffenceHandler, Weight> for Pallet where T: pallet_session::Config::AccountId>, @@ -3362,7 +3362,7 @@ where { fn on_offence( offenders: &[OffenceDetails< - AccountIdOf, + T::AccountId, pallet_session::historical::IdentificationTuple, >], slash_fraction: &[Perbill], From 9b53d4c5247327c942a00becc4bbf636828ed2d4 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 09:44:45 +0200 Subject: [PATCH 031/241] re-benchmark to add rebag --- frame/staking/src/lib.rs | 4 +- frame/staking/src/weights.rs | 307 ++++++++++++++++++----------------- 2 files changed, 161 insertions(+), 150 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index ba8e012c8643b..1788408030748 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2412,9 +2412,7 @@ pub mod pallet { /// among the nominator/validator set once the snapshot is prepared for the election. /// /// Anyone can call this function about any stash. - // - // TODO: benchmark - #[pallet::weight(0)] + #[pallet::weight(T::WeightInfo::rebag())] pub fn rebag( origin: OriginFor, stash: AccountIdOf, diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index cf14e8b22362f..8f918a731fcd8 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-06-19, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-06-29, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -72,381 +72,394 @@ pub trait WeightInfo { fn get_npos_targets(v: u32, ) -> Weight; fn set_staking_limits() -> Weight; fn chill_other() -> Weight; + fn rebag() -> Weight; } /// Weights for pallet_staking using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn bond() -> Weight { - (72_617_000 as Weight) + (68_190_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (55_590_000 as Weight) + (52_644_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (59_730_000 as Weight) + (56_078_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (52_279_000 as Weight) + (49_356_000 as Weight) // Standard Error: 0 - .saturating_add((68_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (86_629_000 as Weight) + (91_998_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_379_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) + .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (32_393_000 as Weight) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (66_528_000 as Weight) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (36_986_000 as Weight) - // Standard Error: 13_000 - .saturating_add((16_574_000 as Weight).saturating_mul(k as Weight)) + (39_441_000 as Weight) + // Standard Error: 18_000 + .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (43_228_000 as Weight) - // Standard Error: 21_000 - .saturating_add((5_119_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (87_415_000 as Weight) + // Standard Error: 42_000 + .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (17_800_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (27_259_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_612_000 as Weight) + (12_074_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_503_000 as Weight) + (27_101_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_119_000 as Weight) + (2_079_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_320_000 as Weight) + (2_249_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_269_000 as Weight) + (2_232_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_334_000 as Weight) + (2_240_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_354_000 as Weight) + (2_239_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (61_556_000 as Weight) + (72_352_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_377_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) + .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_367_105_000 as Weight) - // Standard Error: 222_000 - .saturating_add((19_817_000 as Weight).saturating_mul(s as Weight)) + (4_702_953_000 as Weight) + // Standard Error: 309_000 + .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (47_229_000 as Weight) - // Standard Error: 53_000 - .saturating_add((48_365_000 as Weight).saturating_mul(n as Weight)) + (93_746_000 as Weight) + // Standard Error: 36_000 + .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (156_788_000 as Weight) - // Standard Error: 20_000 - .saturating_add((61_280_000 as Weight).saturating_mul(n as Weight)) + (109_819_000 as Weight) + // Standard Error: 58_000 + .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (47_815_000 as Weight) - // Standard Error: 1_000 - .saturating_add((65_000 as Weight).saturating_mul(l as Weight)) + (48_053_000 as Weight) + // Standard Error: 2_000 + .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) // Standard Error: 74_000 - .saturating_add((34_945_000 as Weight).saturating_mul(e as Weight)) + .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (73_483_000 as Weight) - // Standard Error: 0 - .saturating_add((2_384_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(8 as Weight)) + (104_449_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 846_000 - .saturating_add((305_234_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((48_280_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(10 as Weight)) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + // Standard Error: 1_056_000 + .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 53_000 + .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(83 as Weight)) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 99_000 - .saturating_add((25_735_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 99_000 - .saturating_add((28_122_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_388_000 - .saturating_add((21_500_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + // Standard Error: 100_000 + .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 100_000 + .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_411_000 + .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(75 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 30_000 - .saturating_add((11_065_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 32_000 + .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_028_000 as Weight) - .saturating_add(T::DbWeight::get().writes(4 as Weight)) + (5_669_000 as Weight) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (35_758_000 as Weight) - .saturating_add(T::DbWeight::get().reads(5 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (72_656_000 as Weight) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + fn rebag() -> Weight { + (83_208_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) } } // For backwards compatibility and tests impl WeightInfo for () { fn bond() -> Weight { - (72_617_000 as Weight) + (68_190_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (55_590_000 as Weight) + (52_644_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (59_730_000 as Weight) + (56_078_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (52_279_000 as Weight) + (49_356_000 as Weight) // Standard Error: 0 - .saturating_add((68_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (86_629_000 as Weight) + (91_998_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_379_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (32_393_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (66_528_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (36_986_000 as Weight) - // Standard Error: 13_000 - .saturating_add((16_574_000 as Weight).saturating_mul(k as Weight)) + (39_441_000 as Weight) + // Standard Error: 18_000 + .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (43_228_000 as Weight) - // Standard Error: 21_000 - .saturating_add((5_119_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (87_415_000 as Weight) + // Standard Error: 42_000 + .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (17_800_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (27_259_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_612_000 as Weight) + (12_074_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_503_000 as Weight) + (27_101_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_119_000 as Weight) + (2_079_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_320_000 as Weight) + (2_249_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_269_000 as Weight) + (2_232_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_334_000 as Weight) + (2_240_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_354_000 as Weight) + (2_239_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (61_556_000 as Weight) + (72_352_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_377_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_367_105_000 as Weight) - // Standard Error: 222_000 - .saturating_add((19_817_000 as Weight).saturating_mul(s as Weight)) + (4_702_953_000 as Weight) + // Standard Error: 309_000 + .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (47_229_000 as Weight) - // Standard Error: 53_000 - .saturating_add((48_365_000 as Weight).saturating_mul(n as Weight)) + (93_746_000 as Weight) + // Standard Error: 36_000 + .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (156_788_000 as Weight) - // Standard Error: 20_000 - .saturating_add((61_280_000 as Weight).saturating_mul(n as Weight)) + (109_819_000 as Weight) + // Standard Error: 58_000 + .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (47_815_000 as Weight) - // Standard Error: 1_000 - .saturating_add((65_000 as Weight).saturating_mul(l as Weight)) + (48_053_000 as Weight) + // Standard Error: 2_000 + .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) // Standard Error: 74_000 - .saturating_add((34_945_000 as Weight).saturating_mul(e as Weight)) + .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (73_483_000 as Weight) - // Standard Error: 0 - .saturating_add((2_384_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(8 as Weight)) + (104_449_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 846_000 - .saturating_add((305_234_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((48_280_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(10 as Weight)) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + // Standard Error: 1_056_000 + .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 53_000 + .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(83 as Weight)) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 99_000 - .saturating_add((25_735_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 99_000 - .saturating_add((28_122_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_388_000 - .saturating_add((21_500_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + // Standard Error: 100_000 + .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 100_000 + .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_411_000 + .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(75 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 30_000 - .saturating_add((11_065_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 32_000 + .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_028_000 as Weight) - .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + (5_669_000 as Weight) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (35_758_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(5 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (72_656_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + fn rebag() -> Weight { + (83_208_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } } From ab048c6ca0b6bc0673f3f11a7d4205c34ba0457a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 15:31:19 +0200 Subject: [PATCH 032/241] add VoterList::regenerate() --- frame/staking/src/voter_bags/mod.rs | 37 +++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index ee754dd795e97..ecacd1ebe85ca 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -25,8 +25,8 @@ mod thresholds; use thresholds::THRESHOLDS; use crate::{ - AccountIdOf, Config, Nominations, Nominators, Pallet, VoteWeight, VoterBagFor, VotingDataOf, - slashing::SlashingSpans, + AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, + VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; use frame_support::DefaultNoBound; @@ -81,6 +81,23 @@ impl VoterList { crate::VoterNodes::::remove_all(None); } + /// Regenerate voter data from the `Nominators` and `Validators` storage items. + /// + /// This is expensive and should only ever be performed during a migration, never during + /// consensus. + pub fn regenerate() { + Self::clear(); + + let nominators_iter = Nominators::::iter().map(|(id, _)| Voter::nominator(id)); + let validators_iter = Validators::::iter().map(|(id, _)| Voter::validator(id)); + let weight_of = Pallet::::weight_of_fn(); + + Self::insert_many( + nominators_iter.chain(validators_iter), + weight_of, + ); + } + /// Decode the length of the voter list. pub fn decode_len() -> Option { crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) @@ -464,6 +481,22 @@ pub struct Voter { pub voter_type: VoterType, } +impl Voter { + pub fn nominator(id: AccountId) -> Self { + Self { + id, + voter_type: VoterType::Nominator, + } + } + + pub fn validator(id: AccountId) -> Self { + Self { + id, + voter_type: VoterType::Validator, + } + } +} + pub type VoterOf = Voter>; /// Type of voter. From 5583ae4fd712a1ddd37a3cb173e11459e92167aa Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 16:23:24 +0200 Subject: [PATCH 033/241] add VoterList migration --- frame/staking/src/lib.rs | 32 ++++++++++++++++++++++++++++- frame/staking/src/voter_bags/mod.rs | 11 ++++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 1788408030748..ba35473385540 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -748,17 +748,47 @@ enum Releases { V5_0_0, // blockable validators. V6_0_0, // removal of all storage associated with offchain phragmen. V7_0_0, // keep track of number of nominators / validators in map + V8_0_0, // VoterList and efficient semi-sorted iteration } impl Default for Releases { fn default() -> Self { - Releases::V7_0_0 + Releases::V8_0_0 } } pub mod migrations { use super::*; + pub mod v8 { + use super::*; + + pub fn pre_migrate() -> Result<(), &'static str> { + ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); + ensure!(VoterList::::decode_len().unwrap_or_default() == 0, "voter list already exists"); + Ok(()) + } + + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V8_0_0"); + + let migrated = VoterList::::regenerate(); + + StorageVersion::::put(Releases::V8_0_0); + log!( + info, + "Completed staking migration to Releases::V8_0_0 with {} voters migrated", + migrated, + ); + + // TODO: this is a pretty rough estimate, improve it + T::DbWeight::get().reads_writes( + migrated.into(), + (migrated * 3).into(), + ) + } + } + pub mod v7 { use super::*; diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index ecacd1ebe85ca..626ff010573f3 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -85,7 +85,9 @@ impl VoterList { /// /// This is expensive and should only ever be performed during a migration, never during /// consensus. - pub fn regenerate() { + /// + /// Returns the number of voters migrated. + pub fn regenerate() -> u32 { Self::clear(); let nominators_iter = Nominators::::iter().map(|(id, _)| Voter::nominator(id)); @@ -95,7 +97,7 @@ impl VoterList { Self::insert_many( nominators_iter.chain(validators_iter), weight_of, - ); + ) } /// Decode the length of the voter list. @@ -127,7 +129,7 @@ impl VoterList { /// Insert a new voter into the appropriate bag in the voter list. pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { - Self::insert_many(sp_std::iter::once(voter), weight_of) + Self::insert_many(sp_std::iter::once(voter), weight_of); } /// Insert several voters into the appropriate bags in the voter list. @@ -136,7 +138,7 @@ impl VoterList { pub fn insert_many( voters: impl IntoIterator>, weight_of: impl Fn(&T::AccountId) -> VoteWeight, - ) { + ) -> u32 { let mut bags = BTreeMap::new(); let mut count = 0; @@ -152,6 +154,7 @@ impl VoterList { } crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); + count } /// Remove a voter (by id) from the voter list. From ae67ae79eb9d494b33c04d95d74fa6ac91f0ad6a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 10:42:38 +0200 Subject: [PATCH 034/241] add benchmark for regenerate --- frame/staking/src/benchmarking.rs | 22 +++ frame/staking/src/lib.rs | 9 +- frame/staking/src/weights.rs | 249 +++++++++++++++++------------- 3 files changed, 164 insertions(+), 116 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 2d23a6c983f0d..6f2cd2e7d07a1 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -719,6 +719,28 @@ benchmarks! { let node = Node::::from_id(&stash).ok_or("node not found for stash")?; ensure!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); } + + regenerate { + // number of validator intention. + let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; + // number of nominator intention. + let n in (MAX_NOMINATORS / 2) .. MAX_NOMINATORS; + + clear_validators_and_nominators::(); + ensure!( + create_validators_with_nominators_for_era::( + v, + n, + T::MAX_NOMINATIONS as usize, + true, + None, + ).is_ok(), + "creating validators and nominators failed", + ); + }: { + let migrated = VoterList::::regenerate(); + ensure!(v + n == migrated, "didn't migrate right amount of voters"); + } } #[cfg(test)] diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index ba35473385540..77b59e977385a 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -781,11 +781,10 @@ pub mod migrations { migrated, ); - // TODO: this is a pretty rough estimate, improve it - T::DbWeight::get().reads_writes( - migrated.into(), - (migrated * 3).into(), - ) + T::WeightInfo::regenerate( + CounterForValidators::::get(), + CounterForNominators::::get(), + ).saturating_add(T::DbWeight::get().reads(2)) } } diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 8f918a731fcd8..1a259e4966082 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-06-29, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-06-30, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -73,161 +73,162 @@ pub trait WeightInfo { fn set_staking_limits() -> Weight; fn chill_other() -> Weight; fn rebag() -> Weight; + fn regenerate(v: u32, n: u32, ) -> Weight; } /// Weights for pallet_staking using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn bond() -> Weight { - (68_190_000 as Weight) + (70_861_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (52_644_000 as Weight) + (54_634_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (56_078_000 as Weight) + (58_152_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (49_356_000 as Weight) + (53_561_000 as Weight) // Standard Error: 0 - .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((38_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (91_998_000 as Weight) + (93_297_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_349_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (66_528_000 as Weight) + (67_912_000 as Weight) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (39_441_000 as Weight) - // Standard Error: 18_000 - .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) + (44_238_000 as Weight) + // Standard Error: 20_000 + .saturating_add((15_852_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (87_415_000 as Weight) - // Standard Error: 42_000 - .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + (91_375_000 as Weight) + // Standard Error: 48_000 + .saturating_add((5_002_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (27_259_000 as Weight) + (27_150_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_074_000 as Weight) + (11_982_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_101_000 as Weight) + (26_858_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_079_000 as Weight) + (2_000_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_249_000 as Weight) + (2_232_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_232_000 as Weight) + (2_256_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_240_000 as Weight) + (2_224_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_239_000 as Weight) + (2_344_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (72_352_000 as Weight) + (70_433_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_352_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (4_702_953_000 as Weight) - // Standard Error: 309_000 - .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) + (3_208_783_000 as Weight) + // Standard Error: 220_000 + .saturating_add((18_751_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (93_746_000 as Weight) - // Standard Error: 36_000 - .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) + (136_667_000 as Weight) + // Standard Error: 31_000 + .saturating_add((47_790_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (109_819_000 as Weight) - // Standard Error: 58_000 - .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) + (157_202_000 as Weight) + // Standard Error: 36_000 + .saturating_add((61_104_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (48_053_000 as Weight) + (48_267_000 as Weight) // Standard Error: 2_000 - .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((28_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 74_000 - .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 71_000 + .saturating_add((34_769_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (104_449_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + (108_325_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_365_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 1_056_000 - .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 53_000 - .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 1_084_000 + .saturating_add((307_737_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 54_000 + .saturating_add((51_644_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(83 as Weight)) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -236,12 +237,12 @@ impl WeightInfo for SubstrateWeight { } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 100_000 - .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 100_000 - .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_411_000 - .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 118_000 + .saturating_add((21_454_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 118_000 + .saturating_add((31_494_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 4_041_000 + .saturating_add((52_930_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(75 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -249,179 +250,192 @@ impl WeightInfo for SubstrateWeight { } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 32_000 - .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 29_000 + .saturating_add((10_880_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_669_000 as Weight) + (5_902_000 as Weight) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (72_656_000 as Weight) + (73_685_000 as Weight) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn rebag() -> Weight { - (83_208_000 as Weight) + (84_501_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } + fn regenerate(v: u32, n: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 128_000 + .saturating_add((40_328_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 128_000 + .saturating_add((42_763_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().writes(14 as Weight)) + .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(v as Weight))) + .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) + } } // For backwards compatibility and tests impl WeightInfo for () { fn bond() -> Weight { - (68_190_000 as Weight) + (70_861_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (52_644_000 as Weight) + (54_634_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (56_078_000 as Weight) + (58_152_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (49_356_000 as Weight) + (53_561_000 as Weight) // Standard Error: 0 - .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((38_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (91_998_000 as Weight) + (93_297_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_349_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (66_528_000 as Weight) + (67_912_000 as Weight) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (39_441_000 as Weight) - // Standard Error: 18_000 - .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) + (44_238_000 as Weight) + // Standard Error: 20_000 + .saturating_add((15_852_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (87_415_000 as Weight) - // Standard Error: 42_000 - .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + (91_375_000 as Weight) + // Standard Error: 48_000 + .saturating_add((5_002_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (27_259_000 as Weight) + (27_150_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_074_000 as Weight) + (11_982_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_101_000 as Weight) + (26_858_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_079_000 as Weight) + (2_000_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_249_000 as Weight) + (2_232_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_232_000 as Weight) + (2_256_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_240_000 as Weight) + (2_224_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_239_000 as Weight) + (2_344_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (72_352_000 as Weight) + (70_433_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_352_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (4_702_953_000 as Weight) - // Standard Error: 309_000 - .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) + (3_208_783_000 as Weight) + // Standard Error: 220_000 + .saturating_add((18_751_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (93_746_000 as Weight) - // Standard Error: 36_000 - .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) + (136_667_000 as Weight) + // Standard Error: 31_000 + .saturating_add((47_790_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (109_819_000 as Weight) - // Standard Error: 58_000 - .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) + (157_202_000 as Weight) + // Standard Error: 36_000 + .saturating_add((61_104_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (48_053_000 as Weight) + (48_267_000 as Weight) // Standard Error: 2_000 - .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((28_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 74_000 - .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 71_000 + .saturating_add((34_769_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (104_449_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + (108_325_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_365_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 1_056_000 - .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 53_000 - .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 1_084_000 + .saturating_add((307_737_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 54_000 + .saturating_add((51_644_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(83 as Weight)) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -430,12 +444,12 @@ impl WeightInfo for () { } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 100_000 - .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 100_000 - .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_411_000 - .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 118_000 + .saturating_add((21_454_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 118_000 + .saturating_add((31_494_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 4_041_000 + .saturating_add((52_930_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(75 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -443,23 +457,36 @@ impl WeightInfo for () { } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 32_000 - .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 29_000 + .saturating_add((10_880_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_669_000 as Weight) + (5_902_000 as Weight) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (72_656_000 as Weight) + (73_685_000 as Weight) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn rebag() -> Weight { - (83_208_000 as Weight) + (84_501_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } + fn regenerate(v: u32, n: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 128_000 + .saturating_add((40_328_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 128_000 + .saturating_add((42_763_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().writes(14 as Weight)) + .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(v as Weight))) + .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) + } } From efe15e22261c440adc265a64b6c87d3677fdb038 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:42:13 +0200 Subject: [PATCH 035/241] remove redundant check --- frame/staking/src/benchmarking.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 6f2cd2e7d07a1..af03448204f1e 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -103,7 +103,6 @@ pub fn create_validator_with_nominators( ); assert_ne!(CounterForValidators::::get(), 0); assert_ne!(CounterForNominators::::get(), 0); - assert_ne!(VoterList::::decode_len().unwrap_or_default(), 0); // Give Era Points let reward = EraRewardPoints:: { From fb5431e1782c60a2c59d0f7fa3a5c752c73bed35 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:45:25 +0200 Subject: [PATCH 036/241] debug assertions that `VoterList` and `CounterFor{Nominators,Validators}` are in sync Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/staking/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 77b59e977385a..6d739411007c8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -3103,6 +3103,7 @@ impl Pallet { } Nominators::::insert(who, nominations); VoterList::::insert_as(who, VoterType::Nominator); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } /// This function will remove a nominator from the `Nominators` storage map, @@ -3112,6 +3113,7 @@ impl Pallet { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } } @@ -3125,6 +3127,7 @@ impl Pallet { } Validators::::insert(who, prefs); VoterList::::insert_as(who, VoterType::Validator); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } /// This function will remove a validator from the `Validators` storage map, @@ -3134,6 +3137,7 @@ impl Pallet { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } } } From ee7959db5ff6bd1c80a81078c99dcd53bd0dc223 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:47:24 +0200 Subject: [PATCH 037/241] remove extra whitespace Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/staking/src/benchmarking.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index af03448204f1e..39338fb518e4d 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -671,7 +671,6 @@ benchmarks! { // Clean up any existing state. clear_validators_and_nominators::(); - // stash controls the node account let (stash, controller) = make_validator(USER_SEED, 100)?; From f58bb3aa006d75a49056219dc7f4123bce75cebc Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:50:53 +0200 Subject: [PATCH 038/241] only emit rebag event on success --- frame/staking/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 6d739411007c8..3b4cff10f8feb 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1397,8 +1397,8 @@ pub mod pallet { StakingElectionFailed, /// Attempted to rebag an account. /// - /// If the second parameter is not `None`, it is the `(from, to)` tuple of bag indices. - Rebag(T::AccountId, Option<(BagIdx, BagIdx)>), + /// Contents: `who, from, to`. + Rebagged(T::AccountId, BagIdx, BagIdx), } #[pallet::error] @@ -2450,12 +2450,12 @@ pub mod pallet { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let moved = voter_bags::Node::::from_id(&stash).and_then(|node| { + if let Some((from, to)) = voter_bags::Node::::from_id(&stash).and_then(|node| { let weight_of = Self::weight_of_fn(); VoterList::update_position_for(node, weight_of) - }); - - Self::deposit_event(Event::::Rebag(stash, moved)); + }) { + Self::deposit_event(Event::::Rebagged(stash, from, to)); + }; Ok(()) } From 3c92edac421433656543d04a7d60e815d41b1112 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:55:44 +0200 Subject: [PATCH 039/241] add doc explaining the term voter --- frame/staking/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 3b4cff10f8feb..c2c8dbbb36489 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -100,6 +100,13 @@ //! //! An account can become a nominator via the [`nominate`](Call::nominate) call. //! +//! #### Voting +//! +//! Staking is closely related to elections; actual validators are chosen from among all potential +//! validators by election by the potential validators and nominators. To reduce use of the phrase +//! "potential validators and nominators", we often use the term **voters**, who are simply +//! the union of potential validators and nominators. +//! //! #### Rewards and Slash //! //! The **reward and slashing** procedure is the core of the Staking pallet, attempting to _embrace From 22bc615a25c1f76ac42c1060061b74f9ab644fda Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 13:23:20 +0200 Subject: [PATCH 040/241] revamp/simplify rebag test --- frame/staking/src/tests.rs | 60 ++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index d76aed9ff04f4..a41886d8b37c8 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3863,42 +3863,42 @@ fn on_finalize_weight_is_nonzero() { #[test] fn test_rebag() { use crate::{ - testing_utils::create_stash_controller, + testing_utils::{create_funded_user, create_stash_controller}, voter_bags::{Bag, Node}, }; - use frame_benchmarking::{whitelisted_caller}; use frame_system::RawOrigin; - const USER_SEED: u32 = 999666; - - let whitelist_account = |account_id: &AccountIdOf| { - frame_benchmarking::benchmarking::add_to_whitelist( - frame_system::Account::::hashed_key_for(account_id).into() - ); - }; - - let make_validator = |n: u32, balance_factor: u32| -> Result<(AccountIdOf, AccountIdOf), &'static str> { + fn make_validator(n: u32, balance_factor: u32) -> Result<(AccountIdOf, AccountIdOf), &'static str> { let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); - whitelist_account(&controller); - let prefs = ValidatorPrefs::default(); - // bond the full value of the stash - Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); - Staking::validate(RawOrigin::Signed(controller.clone()).into(), prefs).unwrap(); + // Bond the full value of the stash + // + // By default, `create_stash_controller` only bonds 10% of the stash. However, we're going + // to want to edit one account's bonded value to match another's, so it's simpler if 100% of + // the balance is bonded. + let balance = ::Currency::free_balance(&stash); + Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), balance).unwrap(); + Staking::validate( + RawOrigin::Signed(controller.clone()).into(), + ValidatorPrefs::default(), + ).unwrap(); Ok((stash, controller)) - }; + } ExtBuilder::default().build_and_execute(|| { - // stash controls the node account - let (stash, controller) = make_validator(USER_SEED, 100).unwrap(); - - // create another validator with 3x the stake - let (other_stash, _) = make_validator(USER_SEED + 1, 300).unwrap(); - - // update the stash account's value/weight - ::Currency::make_free_balance_be(&stash, ::Currency::free_balance(&other_stash)); - Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); + // We want to have two validators: one, `stash`, is the one we will rebag. + // The other, `other_stash`, exists only so that the destination bag is not empty. + let (stash, controller) = make_validator(0, 100).unwrap(); + let (other_stash, _) = make_validator(1, 300).unwrap(); + + // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. + let new_balance = ::Currency::free_balance(&other_stash); + ::Currency::make_free_balance_be(&stash, new_balance); + Staking::bond_extra( + RawOrigin::Signed(stash.clone()).into(), + new_balance, + ).unwrap(); // verify preconditions let weight_of = Staking::weight_of_fn(); @@ -3925,17 +3925,19 @@ fn test_rebag() { 0, "destination bag should not be empty", ); - drop(node); - // caller will call rebag - let caller = whitelisted_caller(); + // Any unrelated person can call `rebag`, as long as they sign and pay the tx fee. + let caller = create_funded_user::("caller", 3, 100); // ensure it's distinct from the other accounts assert_ne!(caller, stash); + assert_ne!(caller, other_stash); assert_ne!(caller, controller); // call rebag Pallet::::rebag(RawOrigin::Signed(caller).into(), stash.clone()).unwrap(); + // node should no longer be misplaced + // note that we have to refresh the node let node = Node::::from_id(&stash).unwrap(); assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); }); From 4092fa8d50741c4195c0039af30b5bd302e9d5b3 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 14:16:58 +0200 Subject: [PATCH 041/241] ensure genesis accounts are placed into the correct nodes/bags --- frame/staking/src/tests.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index a41886d8b37c8..fa31e84ae47d3 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -147,6 +147,17 @@ fn basic_setup_works() { // New era is not being forced assert_eq!(Staking::force_era(), Forcing::NotForcing); + + // genesis accounts must exist in the proper bags + let weight_of = Staking::weight_of_fn(); + // for these stash ids, see + // https://github.com/paritytech/substrate/ + // blob/631d4cdbcad438248c2597213918d8207d85bf6e/frame/staking/src/mock.rs#L435-L441 + for genesis_stash_account_id in [11, 21, 31, 101] { + let node = crate::voter_bags::Node::::from_id(&genesis_stash_account_id) + .expect(&format!("node was created for account {}", genesis_stash_account_id)); + assert!(!node.is_misplaced(&weight_of)); + } }); } From 2182abe649de3f209682501f9719e9f1b70e720a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 15:03:20 +0200 Subject: [PATCH 042/241] bond_extra implicitly rebags --- frame/staking/src/benchmarking.rs | 11 +++++++- frame/staking/src/lib.rs | 34 ++++++++++++++---------- frame/staking/src/tests.rs | 43 ++++++++++++------------------- 3 files changed, 46 insertions(+), 42 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 39338fb518e4d..208769ce1ccac 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -678,9 +678,18 @@ benchmarks! { let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; // update the stash account's value/weight + // + // note that we have to manually update the ledger; if we were to just call + // `Staking::::bond_extra`, then it would implicitly rebag. We want to separate that step + // so we can measure it in isolation. let other_free_balance = T::Currency::free_balance(&other_stash); T::Currency::make_free_balance_be(&stash, other_free_balance); - Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), other_free_balance)?; + let controller = Staking::::bonded(&stash).ok_or("stash had no controller")?; + let mut ledger = Staking::::ledger(&controller).ok_or("controller had no ledger")?; + let extra = other_free_balance.checked_sub(&ledger.total).ok_or("balance did not increase")?; + ledger.total += extra; + ledger.active += extra; + Staking::::update_ledger(&controller, &ledger); // verify preconditions let weight_of = Staking::::weight_of_fn(); diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index c2c8dbbb36489..0abd609db6dfd 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1402,9 +1402,7 @@ pub mod pallet { Kicked(T::AccountId, T::AccountId), /// The election failed. No new era is planned. StakingElectionFailed, - /// Attempted to rebag an account. - /// - /// Contents: `who, from, to`. + /// Moved an account from one bag to another. \[who, from, to\]. Rebagged(T::AccountId, BagIdx, BagIdx), } @@ -1623,8 +1621,9 @@ pub mod pallet { // Last check: the new active amount of ledger must be more than ED. ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); - Self::deposit_event(Event::::Bonded(stash, extra)); + Self::deposit_event(Event::::Bonded(stash.clone(), extra)); Self::update_ledger(&controller, &ledger); + Self::do_rebag(&stash); } Ok(()) } @@ -2454,16 +2453,7 @@ pub mod pallet { stash: AccountIdOf, ) -> DispatchResult { ensure_signed(origin)?; - - // if no voter at that node, don't do anything. - // the caller just wasted the fee to call this. - if let Some((from, to)) = voter_bags::Node::::from_id(&stash).and_then(|node| { - let weight_of = Self::weight_of_fn(); - VoterList::update_position_for(node, weight_of) - }) { - Self::deposit_event(Event::::Rebagged(stash, from, to)); - }; - + Pallet::::do_rebag(&stash); Ok(()) } } @@ -3147,6 +3137,22 @@ impl Pallet { debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } } + + /// Move a stash account from one bag to another, depositing an event on success. + /// + /// If the stash changed bags, returns `Some((from, to))`. + pub fn do_rebag(stash: &T::AccountId) -> Option<(BagIdx, BagIdx)> { + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { + let weight_of = Self::weight_of_fn(); + VoterList::update_position_for(node, weight_of) + }); + if let Some((from, to)) = maybe_movement { + Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); + }; + maybe_movement + } } impl diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index fa31e84ae47d3..e0e6b83d4b742 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3874,12 +3874,13 @@ fn on_finalize_weight_is_nonzero() { #[test] fn test_rebag() { use crate::{ - testing_utils::{create_funded_user, create_stash_controller}, + testing_utils::create_stash_controller, voter_bags::{Bag, Node}, }; use frame_system::RawOrigin; - fn make_validator(n: u32, balance_factor: u32) -> Result<(AccountIdOf, AccountIdOf), &'static str> { + /// Make a validator and return its stash + fn make_validator(n: u32, balance_factor: u32) -> Result, &'static str> { let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); // Bond the full value of the stash @@ -3894,30 +3895,18 @@ fn test_rebag() { ValidatorPrefs::default(), ).unwrap(); - Ok((stash, controller)) + Ok(stash) } ExtBuilder::default().build_and_execute(|| { // We want to have two validators: one, `stash`, is the one we will rebag. // The other, `other_stash`, exists only so that the destination bag is not empty. - let (stash, controller) = make_validator(0, 100).unwrap(); - let (other_stash, _) = make_validator(1, 300).unwrap(); - - // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. - let new_balance = ::Currency::free_balance(&other_stash); - ::Currency::make_free_balance_be(&stash, new_balance); - Staking::bond_extra( - RawOrigin::Signed(stash.clone()).into(), - new_balance, - ).unwrap(); + let stash = make_validator(0, 100).unwrap(); + let other_stash = make_validator(1, 300).unwrap(); // verify preconditions let weight_of = Staking::weight_of_fn(); let node = Node::::from_id(&stash).unwrap(); - assert!( - node.is_misplaced(&weight_of), - "rebagging only makes sense when a node is misplaced", - ); assert_eq!( { let origin_bag = Bag::::get(node.bag_idx).unwrap(); @@ -3937,18 +3926,18 @@ fn test_rebag() { "destination bag should not be empty", ); - // Any unrelated person can call `rebag`, as long as they sign and pay the tx fee. - let caller = create_funded_user::("caller", 3, 100); - // ensure it's distinct from the other accounts - assert_ne!(caller, stash); - assert_ne!(caller, other_stash); - assert_ne!(caller, controller); - - // call rebag - Pallet::::rebag(RawOrigin::Signed(caller).into(), stash.clone()).unwrap(); + // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. + // + // This implicitly calls rebag, so the user stays in the best bag they qualify for. + let new_balance = ::Currency::free_balance(&other_stash); + ::Currency::make_free_balance_be(&stash, new_balance); + Staking::bond_extra( + RawOrigin::Signed(stash.clone()).into(), + new_balance, + ).unwrap(); // node should no longer be misplaced - // note that we have to refresh the node + // note that we refresh the node, in case the storage value has changed let node = Node::::from_id(&stash).unwrap(); assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); }); From 472baa97099a5fa8a33cef11b838a616342df889 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 15:23:06 +0200 Subject: [PATCH 043/241] types at top; doc public type --- frame/staking/src/voter_bags/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 626ff010573f3..f768f056115b2 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -33,6 +33,9 @@ use frame_support::DefaultNoBound; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; +/// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. +pub type VoterOf = Voter>; + /// Index type for a bag. pub type BagIdx = u8; @@ -500,8 +503,6 @@ impl Voter { } } -pub type VoterOf = Voter>; - /// Type of voter. /// /// Similar to [`crate::StakerStatus`], but somewhat more limited. From f1e7fe7add9b8cc27007975c9e1efe3893cec545 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 15:01:40 +0200 Subject: [PATCH 044/241] start sketching out adjustable thresholds --- frame/staking/src/lib.rs | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 0abd609db6dfd..34cdc0189d5d8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -960,6 +960,59 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; + + /// The list of thresholds separating the various voter bags. + /// + /// Voters are separated into unsorted bags according to their vote weight. This specifies + /// the thresholds separating the bags. A voter's bag is the largest bag for which the + /// voter's weight is less than or equal to its upper threshold. + /// + /// When voters are iterated, higher bags are iterated completely before lower bags. This + /// that iteration is _semi-sorted_: voters of higher weight tend to come before voters of + /// lower weight, but peer voters within a particular bag are sorted in insertion order. + /// + /// # Expressing the constant + /// + /// This constant must be sorted in strictly increasing order. Duplicate items are not + /// permitted. + /// + /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// specified within the bag. For any two threshold lists, if one ends with + /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists + /// will behave identically. + /// + /// # Calculation + /// + /// It is recommended to generate the set of thresholds in a geometric series, such that + /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * + /// constant_ratio).max(threshold[k] + 1)` for all `k`. + /// + /// Given the desire to compute `N` bags, the constant ratio can be computed with + /// `exp(ln(VoterBags::MAX) / N)`. + /// + /// # Examples + /// + /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, + /// and iteration is strictly in insertion order. + /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is equal to 2. + /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is approximately equal + /// to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will + /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. + /// + /// # Migration + /// + /// In the event that this list ever changes, the `VoterList` data structure will need to be + /// regenerated. + #[pallet::constant] + type VoterBagThresholds: Get<&'static [VoteWeight]>; + + /// A type sufficient to distinguish between all voter bags. + /// + /// For 256 bags or fewer, `u8` suffices. + type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen; } #[pallet::extra_constants] From 16e4774d05ca90ce843ab65d2b0004b06692de55 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 16:06:36 +0200 Subject: [PATCH 045/241] add integrity test for voter bag threshold requirements --- frame/staking/src/lib.rs | 24 +++++++++++++++++------- frame/staking/src/mock.rs | 6 ++++++ frame/staking/src/voter_bags/mod.rs | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 34cdc0189d5d8..b4d48fa398820 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -308,11 +308,11 @@ use frame_support::{ }; use pallet_session::historical; use sp_runtime::{ - Percent, Perbill, RuntimeDebug, DispatchError, + DispatchError, Perbill, Percent, RuntimeDebug, curve::PiecewiseLinear, traits::{ - Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, - AtLeast32BitUnsigned, Bounded, + AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, + StaticLookup, UniqueSaturatedInto, Zero, }, }; use sp_staking::{ @@ -1012,7 +1012,7 @@ pub mod pallet { /// A type sufficient to distinguish between all voter bags. /// /// For 256 bags or fewer, `u8` suffices. - type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen; + type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen + Bounded + UniqueSaturatedInto; } #[pallet::extra_constants] @@ -1550,14 +1550,24 @@ pub mod pallet { fn integrity_test() { sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| + sp_io::TestExternalities::new_empty().execute_with(|| { assert!( T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", T::SlashDeferDuration::get(), T::BondingDuration::get(), - ) - ); + ); + + assert!( + T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "Voter bag thresholds must strictly increase", + ); + + assert!( + T::BagIdx::max_value().saturated_into() >= T::VoterBagThresholds::get().len(), + "BagIdx must be sufficient to uniquely identify every bag", + ); + }); } } } diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index e0079cc3f375a..d9cc77653e986 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -243,6 +243,10 @@ impl onchain::Config for Test { type DataProvider = Staking; } +parameter_types! { + pub const VoterBagThresholds: &'static [VoteWeight] = &crate::voter_bags::thresholds::THRESHOLDS; +} + impl Config for Test { const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; @@ -263,6 +267,8 @@ impl Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = VoterBagThresholds; + type BagIdx = u8; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index f768f056115b2..29a42d35eb14a 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -21,7 +21,7 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -mod thresholds; +pub mod thresholds; use thresholds::THRESHOLDS; use crate::{ From 2a4ebf7f9fef531df96028716237e16ea6f248cd Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 16:47:23 +0200 Subject: [PATCH 046/241] get rid of BagIdx This reorganizes bag storage such that bags are always referred to by their upper threshold. This in turn means that adding and removing bags is cheaper; you only need to migrate certain voters, not all of them. --- frame/staking/src/lib.rs | 36 +++---- frame/staking/src/voter_bags/mod.rs | 109 +++++++++++---------- frame/staking/src/voter_bags/thresholds.rs | 4 +- 3 files changed, 73 insertions(+), 76 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index b4d48fa398820..7f6b5402cff45 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -312,7 +312,7 @@ use sp_runtime::{ curve::PiecewiseLinear, traits::{ AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, - StaticLookup, UniqueSaturatedInto, Zero, + StaticLookup, Zero, }, }; use sp_staking::{ @@ -321,7 +321,7 @@ use sp_staking::{ }; use frame_system::{ensure_signed, ensure_root, pallet_prelude::*, offchain::SendTransactionTypes}; use frame_election_provider_support::{ElectionProvider, VoteWeight, Supports, data_provider}; -use voter_bags::{BagIdx, VoterList, VoterType}; +use voter_bags::{VoterList, VoterType}; pub use weights::WeightInfo; pub use pallet::*; @@ -1008,11 +1008,6 @@ pub mod pallet { /// regenerated. #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; - - /// A type sufficient to distinguish between all voter bags. - /// - /// For 256 bags or fewer, `u8` suffices. - type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen + Bounded + UniqueSaturatedInto; } #[pallet::extra_constants] @@ -1311,6 +1306,8 @@ pub mod pallet { // The next storage items collectively comprise the voter bags: a composite data structure // designed to allow efficient iteration of the top N voters by stake, mostly. See // `mod voter_bags` for details. + // + // In each of these items, voter bags are indexed by their upper weight threshold. /// How many voters are registered. #[pallet::storage] @@ -1321,21 +1318,25 @@ pub mod pallet { /// This may not be the appropriate bag for the voter's weight if they have been rewarded or /// slashed. #[pallet::storage] - pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::BagIdx>; + pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; - /// The head and tail of each bag of voters. + /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which + /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] pub(crate) type VoterBags = StorageMap< _, - Twox64Concat, voter_bags::BagIdx, + Twox64Concat, VoteWeight, voter_bags::Bag, >; - /// The nodes comprising each bag. + /// Voter nodes store links forward and back within their respective bags, the stash id, and + /// whether the voter is a validator or nominator. + /// + /// There is nothing in this map directly identifying to which bag a particular node belongs. + /// However, the `Node` data structure has helpers which can provide that information. #[pallet::storage] - pub(crate) type VoterNodes = StorageDoubleMap< + pub(crate) type VoterNodes = StorageMap< _, - Twox64Concat, voter_bags::BagIdx, Twox64Concat, AccountIdOf, voter_bags::Node, >; @@ -1456,7 +1457,7 @@ pub mod pallet { /// The election failed. No new era is planned. StakingElectionFailed, /// Moved an account from one bag to another. \[who, from, to\]. - Rebagged(T::AccountId, BagIdx, BagIdx), + Rebagged(T::AccountId, VoteWeight, VoteWeight), } #[pallet::error] @@ -1562,11 +1563,6 @@ pub mod pallet { T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), "Voter bag thresholds must strictly increase", ); - - assert!( - T::BagIdx::max_value().saturated_into() >= T::VoterBagThresholds::get().len(), - "BagIdx must be sufficient to uniquely identify every bag", - ); }); } } @@ -3204,7 +3200,7 @@ impl Pallet { /// Move a stash account from one bag to another, depositing an event on success. /// /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag(stash: &T::AccountId) -> Option<(BagIdx, BagIdx)> { + pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 29a42d35eb14a..11a54a6fe10a9 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -23,41 +23,37 @@ pub mod thresholds; -use thresholds::THRESHOLDS; use crate::{ AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; -use frame_support::DefaultNoBound; +use frame_support::{DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; -/// Index type for a bag. -pub type BagIdx = u8; - -/// How many bags there are -pub const N_BAGS: BagIdx = 200; - -/// Given a certain vote weight, which bag should this voter contain? +/// Given a certain vote weight, which bag should contain this voter? +/// +/// Bags are identified by their upper threshold; the value returned by this function is guaranteed +/// to be a member of `T::VoterBagThresholds`. /// -/// Bags are separated by fixed thresholds. To the extent possible, each threshold is a constant -/// small multiple of the one before it. That ratio is [`thresholds::CONSTANT_RATIO`]. The exception -/// are the smallest bags, which are each at least 1 greater than the previous, and the largest bag, -/// which is defined as `u64::MAX`. +/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, +/// because in the event that bags are inserted or deleted, the number of affected voters which need +/// to be migrated is smaller. /// -/// Bags are arranged such that `bags[0]` is the largest bag, and `bags[N_BAGS-1]` is the smallest. -fn notional_bag_for(weight: VoteWeight) -> BagIdx { - let raw_bag = - THRESHOLDS.partition_point(|&threshold| weight > threshold) as BagIdx; - N_BAGS - 1 - raw_bag +/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this +/// function behaves as if it does. +fn notional_bag_for(weight: VoteWeight) -> VoteWeight { + let thresholds = T::VoterBagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| weight > threshold); + thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } -/// Find the actual bag containing the current voter. -fn current_bag_for(id: &AccountIdOf) -> Option { +/// Find the upper threshold of the actual bag containing the current voter. +fn current_bag_for(id: &AccountIdOf) -> Option { VoterBagFor::::try_get(id).ok() } @@ -110,10 +106,13 @@ impl VoterList { /// Iterate over all nodes in all bags in the voter list. /// - /// Note that this exhaustively attempts to try all possible bag indices. Full iteration can be - /// expensive; it's recommended to limit the number of items with `.take(n)`. + /// Full iteration can be expensive; it's recommended to limit the number of items with `.take(n)`. pub fn iter() -> impl Iterator> { - (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) + T::VoterBagThresholds::get() + .iter() + .copied() + .filter_map(Bag::get) + .flat_map(|bag| bag.iter()) } /// Insert a new voter into the appropriate bag in the voter list. @@ -147,7 +146,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); - let bag = notional_bag_for(weight); + let bag = notional_bag_for::(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -180,11 +179,11 @@ impl VoterList { count += 1; // clear the bag head/tail pointers as necessary - let bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); + let bag = bags.entry(node.bag_upper).or_insert_with(|| Bag::::get_or_make(node.bag_upper)); bag.remove_node(&node); // now get rid of the node itself - crate::VoterNodes::::remove(node.bag_idx, voter_id); + crate::VoterNodes::::remove(voter_id); crate::VoterBagFor::::remove(voter_id); } @@ -208,12 +207,12 @@ impl VoterList { pub fn update_position_for( mut node: Node, weight_of: impl Fn(&AccountIdOf) -> VoteWeight, - ) -> Option<(BagIdx, BagIdx)> { + ) -> Option<(VoteWeight, VoteWeight)> { node.is_misplaced(&weight_of).then(move || { - let old_idx = node.bag_idx; + let old_idx = node.bag_upper; // clear the old bag head/tail pointers as necessary - if let Some(mut bag) = Bag::::get(node.bag_idx) { + if let Some(mut bag) = Bag::::get(node.bag_upper) { bag.remove_node(&node); bag.put(); } else { @@ -226,9 +225,9 @@ impl VoterList { } // put the voter into the appropriate new bag - let new_idx = notional_bag_for(weight_of(&node.voter.id)); - node.bag_idx = new_idx; - let mut bag = Bag::::get_or_make(node.bag_idx); + let new_idx = notional_bag_for::(weight_of(&node.voter.id)); + node.bag_upper = new_idx; + let mut bag = Bag::::get_or_make(node.bag_upper); bag.insert_node(node); bag.put(); @@ -250,36 +249,40 @@ pub struct Bag { tail: Option>, #[codec(skip)] - bag_idx: BagIdx, + bag_upper: VoteWeight, } impl Bag { - /// Get a bag by idx. - pub fn get(bag_idx: BagIdx) -> Option> { - crate::VoterBags::::try_get(bag_idx).ok().map(|mut bag| { - bag.bag_idx = bag_idx; + /// Get a bag by its upper vote weight. + pub fn get(bag_upper: VoteWeight) -> Option> { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; bag }) } - /// Get a bag by idx or make it, appropriately initialized. - pub fn get_or_make(bag_idx: BagIdx) -> Bag { - Self::get(bag_idx).unwrap_or(Bag { bag_idx, ..Default::default() }) + /// Get a bag by its upper vote weight or make it, appropriately initialized. + pub fn get_or_make(bag_upper: VoteWeight) -> Bag { + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } /// Put the bag back into storage. pub fn put(self) { - crate::VoterBags::::insert(self.bag_idx, self); + crate::VoterBags::::insert(self.bag_upper, self); } /// Get the head node in this bag. pub fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) } /// Get the tail node in this bag. pub fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) } /// Iterate over the nodes in this bag. @@ -299,7 +302,7 @@ impl Bag { voter, prev: None, next: None, - bag_idx: self.bag_idx, + bag_upper: self.bag_upper, }); } @@ -329,7 +332,7 @@ impl Bag { } self.tail = Some(id.clone()); - crate::VoterBagFor::::insert(id, self.bag_idx); + crate::VoterBagFor::::insert(id, self.bag_upper); } /// Remove a voter node from this bag. @@ -362,14 +365,14 @@ pub struct Node { /// The bag index is not stored in storage, but injected during all fetch operations. #[codec(skip)] - pub(crate) bag_idx: BagIdx, + pub(crate) bag_upper: VoteWeight, } impl Node { /// Get a node by bag idx and account id. - pub fn get(bag_idx: BagIdx, account_id: &AccountIdOf) -> Option> { - crate::VoterNodes::::try_get(&bag_idx, account_id).ok().map(|mut node| { - node.bag_idx = bag_idx; + pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { + crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { + node.bag_upper = bag_upper; node }) } @@ -385,12 +388,12 @@ impl Node { /// Get a node by account id, assuming it's in the same bag as this node. pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { - Self::get(self.bag_idx, account_id) + Self::get(self.bag_upper, account_id) } /// Put the node back into storage. pub fn put(self) { - crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); + crate::VoterNodes::::insert(self.voter.id.clone(), self); } /// Get the previous node in the bag. @@ -448,7 +451,7 @@ impl Node { /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx + notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper } /// Update the voter type associated with a particular node by id. @@ -470,7 +473,7 @@ impl Node { /// /// This is a helper intended only for benchmarking and should not be used in production. #[cfg(any(test, feature = "runtime-benchmarks"))] - pub fn proper_bag_for(&self) -> BagIdx { + pub fn proper_bag_for(&self) -> VoteWeight { let weight_of = crate::Pallet::::weight_of_fn(); let current_weight = weight_of(&self.voter.id); notional_bag_for(current_weight) diff --git a/frame/staking/src/voter_bags/thresholds.rs b/frame/staking/src/voter_bags/thresholds.rs index 710403d75e64f..37b889405ac19 100644 --- a/frame/staking/src/voter_bags/thresholds.rs +++ b/frame/staking/src/voter_bags/thresholds.rs @@ -17,15 +17,13 @@ //! Generated voter bag thresholds. -use super::N_BAGS; - /// Ratio between adjacent bags; #[cfg(any(test, feature = "std"))] #[allow(unused)] pub const CONSTANT_RATIO: f64 = 1.2483305489016119; /// Upper thresholds for each bag. -pub const THRESHOLDS: [u64; N_BAGS as usize] = [ +pub const THRESHOLDS: [u64; 200] = [ 1, 2, 3, From cc6d0dff5a78b9eb5af7c74ca2c974fde5fd1020 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 1 Jul 2021 11:57:33 +0200 Subject: [PATCH 047/241] implement migration logic for when the threshold list changes --- frame/staking/src/voter_bags/mod.rs | 94 ++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 11a54a6fe10a9..a3283bbdb0262 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -30,7 +30,7 @@ use crate::{ use codec::{Encode, Decode}; use frame_support::{DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; -use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; +use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, marker::PhantomData}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; @@ -234,6 +234,94 @@ impl VoterList { (old_idx, new_idx) }) } + + /// Migrate the voter list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::VoterBagThresholds` has already been updated. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - No voter is changed unless required to by the difference between the old threshold list + /// and the new. + /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// new threshold set. + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + // we can't check all preconditions, but we can check one + debug_assert!( + crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); + + let mut affected_accounts = BTreeSet::new(); + let mut affected_old_bags = BTreeSet::new(); + + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_set.difference(&old_set).copied() { + let affected_bag = notional_bag_for::(inserted_bag); + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // a removed bag means that all members of that bag must be rebagged + for removed_bag in old_set.difference(&new_set).copied() { + if !affected_old_bags.insert(removed_bag) { + continue + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // migrate the + let weight_of = Pallet::::weight_of_fn(); + Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); + let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in old_set.difference(&new_set).copied() { + debug_assert!( + !VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + "no voter should be present in a removed bag", + ); + crate::VoterBags::::remove(removed_bag); + } + + debug_assert!( + { + let thresholds = T::VoterBagThresholds::get(); + crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + }, + "all `bag_upper` in storage must be members of the new thresholds", + ); + + num_affected + } } /// A Bag is a doubly-linked list of voters. @@ -481,7 +569,7 @@ impl Node { } /// Fundamental information about a voter. -#[derive(Clone, Encode, Decode)] +#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(feature = "std", derive(Debug))] pub struct Voter { /// Account Id of this voter @@ -509,7 +597,7 @@ impl Voter { /// Type of voter. /// /// Similar to [`crate::StakerStatus`], but somewhat more limited. -#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(feature = "std", derive(Debug))] pub enum VoterType { Validator, From 88195fe0f885fcf9b58d3f85c649d4e72ab456f2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 1 Jul 2021 14:27:25 +0200 Subject: [PATCH 048/241] start sketching out threshold proc macros --- Cargo.lock | 10 +++ Cargo.toml | 1 + frame/staking/bag-thresholds/Cargo.toml | 18 +++++ frame/staking/bag-thresholds/src/lib.rs | 92 +++++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 frame/staking/bag-thresholds/Cargo.toml create mode 100644 frame/staking/bag-thresholds/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index cb45b18013992..6323931a96459 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5530,6 +5530,16 @@ dependencies = [ "sp-arithmetic", ] +[[package]] +name = "pallet-staking-voter-bag-thresholds" +version = "3.0.0" +dependencies = [ + "proc-macro-crate 1.0.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pallet-sudo" version = "3.0.0" diff --git a/Cargo.toml b/Cargo.toml index f7552f0bbbc48..bec00c77bcbe5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ members = [ "frame/session/benchmarking", "frame/society", "frame/staking", + "frame/staking/bag-thresholds", "frame/staking/reward-curve", "frame/staking/reward-fn", "frame/sudo", diff --git a/frame/staking/bag-thresholds/Cargo.toml b/frame/staking/bag-thresholds/Cargo.toml new file mode 100644 index 0000000000000..44454eb4e28e6 --- /dev/null +++ b/frame/staking/bag-thresholds/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pallet-staking-voter-bag-thresholds" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Voter Bag Thresholds for FRAME Staking Pallet" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0.6" +proc-macro-crate = "1.0.0" +quote = "1.0.3" +syn = { version = "1.0.58", features = ["full", "visit"] } diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs new file mode 100644 index 0000000000000..5d87658e875c8 --- /dev/null +++ b/frame/staking/bag-thresholds/src/lib.rs @@ -0,0 +1,92 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Proc macros to generate voter bag thresholds and associated items. + +/// Calculate an appropriate constant ratio between thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn make_ratio(n: usize, bounds: impl std::ops::RangeBounds) -> f64; +/// ``` +/// +/// # Example: +/// +/// ``` +/// # use pallet_staking_voter_bag_thresholds::make_ratio; +/// /// Constant ratio between bag items. Approx. `1.248`. +/// const CONSTANT_RATIO: f64 = make_ratio!(200, ..); +/// ``` +#[proc_macro] +pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + todo!() +} + +/// Make a constant array of threshold values suitable for use as voter bag thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn make_thresholds(n: usize, bounds: impl std::ops::RangeBounds) -> [VoteWeight; n]; +/// ``` +/// +/// The output has these properties: +/// +/// - Its length is `n`. +/// - Its first item respects `bounds.start_bound()`. +/// - Its last item respects `bounds.end_bound()`. +/// - There exists a constant ratio (see [`make_ratio`]) called _ratio_. +/// +/// For all _k_, `output[k + 1] == (output[k] * ratio).round().min(output[k] + 1)`. +/// +/// # Example: +/// +/// ``` +/// # use pallet_staking_voter_bag_thresholds::make_thresholds; +/// const THRESHOLDS: &[u64] = &make_thresholds!(200, ..); +/// ``` +#[proc_macro] +pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + todo!() +} + +/// Make a constant array of threshold values suitable for use as voter bag thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn edit_thresholds( +/// thresholds: [VoteWeight; Old], +/// inserting: impl IntoIterator, +/// removing: impl IntoIterator, +/// ) -> [VoteWeight; New]; +/// ``` +/// +/// It is intended to be used to simply edit a thresholds list, inserting some items and removing +/// others, without needing to express the entire list. +/// +/// # Example: +/// +/// ``` +/// # use pallet_staking_voter_bag_thresholds::{edit_thresholds, make_thresholds}; +/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, ..), [12345, 54321], []); +/// ``` +#[proc_macro] +pub fn edit_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + todo!() +} From 70848eacdae04dd9413eb829814b1c136a092089 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 1 Jul 2021 15:48:55 +0200 Subject: [PATCH 049/241] further refine macro signatures --- frame/staking/bag-thresholds/src/lib.rs | 52 +++++++++++++++++++------ 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs index 5d87658e875c8..aec0ce4284a05 100644 --- a/frame/staking/bag-thresholds/src/lib.rs +++ b/frame/staking/bag-thresholds/src/lib.rs @@ -22,16 +22,27 @@ /// This macro can be thought of as a function with signature /// /// ```ignore -/// pub const fn make_ratio(n: usize, bounds: impl std::ops::RangeBounds) -> f64; +/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; /// ``` /// +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a +/// `::MAX` attribute available. +/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, +/// this will be the result of some calculation involving the existential deposit for a chain's +/// balance type. +/// /// # Example: /// /// ``` /// # use pallet_staking_voter_bag_thresholds::make_ratio; /// /// Constant ratio between bag items. Approx. `1.248`. -/// const CONSTANT_RATIO: f64 = make_ratio!(200, ..); +/// const CONSTANT_RATIO: f64 = make_ratio!(200, u64, 0); /// ``` +/// +/// # Calculation +/// +/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). #[proc_macro] pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { todo!() @@ -42,23 +53,37 @@ pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// This macro can be thought of as a function with signature /// /// ```ignore -/// pub const fn make_thresholds(n: usize, bounds: impl std::ops::RangeBounds) -> [VoteWeight; n]; +/// pub const fn make_thresholds(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> [VoteWeight; n]; /// ``` /// +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a +/// `::MAX` attribute available. +/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, +/// this will be the result of some calculation involving the existential deposit for a chain's +/// balance type. +/// /// The output has these properties: /// /// - Its length is `n`. -/// - Its first item respects `bounds.start_bound()`. -/// - Its last item respects `bounds.end_bound()`. +/// - Its first item is greater than or equal to `existential_weight`. +/// - Its last item is equal to `VoteWeight::MAX`. /// - There exists a constant ratio (see [`make_ratio`]) called _ratio_. /// -/// For all _k_, `output[k + 1] == (output[k] * ratio).round().min(output[k] + 1)`. +/// For all _k_ in `0..(n-1)`, `output[k + 1] == (output[k] * ratio).round()`. +/// +/// However, there are two exceptions to the ratio rule: +/// +/// - As thresholds may not duplicate, if `(output[k] * ratio).round() == output[k]`, then `output[k +/// + 1] == output[k] + 1`. +/// - Due to the previous exception in combination with the requirement that the final item is equal +/// to `VoteWeight::MAX`, the ratio of the final item may diverge from the common ratio. /// /// # Example: /// /// ``` /// # use pallet_staking_voter_bag_thresholds::make_thresholds; -/// const THRESHOLDS: &[u64] = &make_thresholds!(200, ..); +/// const THRESHOLDS: [u64; 200] = &make_thresholds!(200, u64, 0); /// ``` #[proc_macro] pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -70,10 +95,15 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// This macro can be thought of as a function with signature /// /// ```ignore -/// pub const fn edit_thresholds( +/// pub const fn edit_thresholds< +/// const Old: usize, +/// const Inserting: usize, +/// const Removing: usize, +/// const New: usize, +/// >( /// thresholds: [VoteWeight; Old], -/// inserting: impl IntoIterator, -/// removing: impl IntoIterator, +/// inserting: [VoteWeight; Inserting], +/// removing: [VoteWeight; Removing], /// ) -> [VoteWeight; New]; /// ``` /// @@ -84,7 +114,7 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// /// ``` /// # use pallet_staking_voter_bag_thresholds::{edit_thresholds, make_thresholds}; -/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, ..), [12345, 54321], []); +/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, u64, 0), [12345, 54321], []); /// ``` #[proc_macro] pub fn edit_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { From b6303213abaa160f2cce8cbd23ad0beed114ecc2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 11:36:29 +0200 Subject: [PATCH 050/241] WIP: implement make_ratio macro --- frame/staking/bag-thresholds/src/common.rs | 55 +++++++++++++++++++ frame/staking/bag-thresholds/src/lib.rs | 28 +++++++--- .../bag-thresholds/src/make_ratio_impl.rs | 51 +++++++++++++++++ frame/staking/bag-thresholds/tests/tests.rs | 30 ++++++++++ 4 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 frame/staking/bag-thresholds/src/common.rs create mode 100644 frame/staking/bag-thresholds/src/make_ratio_impl.rs create mode 100644 frame/staking/bag-thresholds/tests/tests.rs diff --git a/frame/staking/bag-thresholds/src/common.rs b/frame/staking/bag-thresholds/src/common.rs new file mode 100644 index 0000000000000..4aa5fc4892066 --- /dev/null +++ b/frame/staking/bag-thresholds/src/common.rs @@ -0,0 +1,55 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Features common between the other modules of this crate. + +use syn::{Expr, Result, Token, Type, parse::{Parse, ParseStream}}; + +/// Parse function-like input parameters +/// +/// ```ignore +/// fn my_fn(n: usize, VoteWeight: Type, existential_weight: VoteWeight); +/// ``` +/// +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any +/// expression which is evaluable in a const context. +/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a +/// `::MAX` attribute available. +/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, +/// this will be the result of some calculation involving the existential deposit for a chain's +/// balance type. It may be any expression which is evaluable in a const context. +pub struct ThresholdParams { + pub n: Expr, + pub comma1: Token![,], + pub vote_weight: Type, + pub comma2: Token![,], + pub existential_weight: Expr, + pub comma3: Option, +} + +impl Parse for ThresholdParams { + fn parse(input: ParseStream) -> Result { + Ok(Self { + n: input.parse()?, + comma1: input.parse()?, + vote_weight: input.parse()?, + comma2: input.parse()?, + existential_weight: input.parse()?, + comma3: input.parse()?, + }) + } +} diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs index aec0ce4284a05..6ae7c68e42516 100644 --- a/frame/staking/bag-thresholds/src/lib.rs +++ b/frame/staking/bag-thresholds/src/lib.rs @@ -17,6 +17,11 @@ //! Proc macros to generate voter bag thresholds and associated items. +use proc_macro::TokenStream; + +mod common; +mod make_ratio_impl; + /// Calculate an appropriate constant ratio between thresholds. /// /// This macro can be thought of as a function with signature @@ -25,12 +30,13 @@ /// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; /// ``` /// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any +/// expression which is evaluable in a const context. /// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a /// `::MAX` attribute available. /// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, /// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. +/// balance type. It may be any expression which is evaluable in a const context. /// /// # Example: /// @@ -44,8 +50,8 @@ /// /// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). #[proc_macro] -pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { - todo!() +pub fn make_ratio(input: TokenStream) -> TokenStream { + make_ratio_impl::make_ratio(input) } /// Make a constant array of threshold values suitable for use as voter bag thresholds. @@ -56,12 +62,13 @@ pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// pub const fn make_thresholds(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> [VoteWeight; n]; /// ``` /// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any +/// expression which is evaluable in a const context. /// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a /// `::MAX` attribute available. /// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, /// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. +/// balance type. It may be any expression which is evaluable in a const context. /// /// The output has these properties: /// @@ -86,7 +93,7 @@ pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// const THRESHOLDS: [u64; 200] = &make_thresholds!(200, u64, 0); /// ``` #[proc_macro] -pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +pub fn make_thresholds(input: TokenStream) -> TokenStream { todo!() } @@ -110,6 +117,11 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// It is intended to be used to simply edit a thresholds list, inserting some items and removing /// others, without needing to express the entire list. /// +/// Note that due to macro limitations, `inserting` and `removing` must be array literals. +/// +/// Note that `removing` overrides `inserting`: if a value appears in both lists, it does not appear +/// in the output. +/// /// # Example: /// /// ``` @@ -117,6 +129,6 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, u64, 0), [12345, 54321], []); /// ``` #[proc_macro] -pub fn edit_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +pub fn edit_thresholds(input: TokenStream) -> TokenStream { todo!() } diff --git a/frame/staking/bag-thresholds/src/make_ratio_impl.rs b/frame/staking/bag-thresholds/src/make_ratio_impl.rs new file mode 100644 index 0000000000000..29812e7efec63 --- /dev/null +++ b/frame/staking/bag-thresholds/src/make_ratio_impl.rs @@ -0,0 +1,51 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation for the `make_ratio` proc macro. + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, parse_quote}; + +use crate::common::ThresholdParams; + +/// Calculate an appropriate constant ratio between thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; +/// ``` +/// +/// # Calculation +/// +/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). +pub fn make_ratio(input: TokenStream) -> TokenStream { + let ThresholdParams{ + n, + vote_weight, + existential_weight, + .. + } = parse_macro_input!(input as ThresholdParams); + + let n = parse_quote!(#n) as f64; + let diff_weight = parse_quote!(#vote_weight::MAX - #existential_weight) as f64; + + let ratio = (diff_weight.ln() / n).exp(); + + quote!(#ratio).into() +} diff --git a/frame/staking/bag-thresholds/tests/tests.rs b/frame/staking/bag-thresholds/tests/tests.rs new file mode 100644 index 0000000000000..7618457476719 --- /dev/null +++ b/frame/staking/bag-thresholds/tests/tests.rs @@ -0,0 +1,30 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod make_ratio { + use pallet_staking_voter_bag_thresholds::make_ratio; + + #[test] + fn u64_200_0() { + assert_eq!(make_ratio!(200, u64, 0), 1.2483305489016119); + } + + #[test] + fn u64_64_0() { + assert_eq!(make_ratio!(64, u64, 0), 2.0); + } +} From 8ce3bc0e78b5290e8758eac0d8717976ecd2dca9 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 13:47:23 +0200 Subject: [PATCH 051/241] start rethinking the process of producing threshold lists The macro approach seems to be a non-starter; that only really works if we're throwing around numeric literals everywhere, and that's just not nice in this case. Instead, let's write helper functions and make it really easy to generate the tables in separate, permanent files, which humans can then edit. --- Cargo.lock | 37 +++-- Cargo.toml | 1 - frame/staking/Cargo.toml | 7 +- frame/staking/bag-thresholds/Cargo.toml | 18 --- frame/staking/bag-thresholds/src/common.rs | 55 ------- frame/staking/bag-thresholds/src/lib.rs | 134 ------------------ .../bag-thresholds/src/make_ratio_impl.rs | 51 ------- frame/staking/bag-thresholds/tests/tests.rs | 30 ---- frame/staking/src/bin/make_bags.rs | 4 +- frame/staking/src/voter_bags/mod.rs | 36 +++++ 10 files changed, 70 insertions(+), 303 deletions(-) delete mode 100644 frame/staking/bag-thresholds/Cargo.toml delete mode 100644 frame/staking/bag-thresholds/src/common.rs delete mode 100644 frame/staking/bag-thresholds/src/lib.rs delete mode 100644 frame/staking/bag-thresholds/src/make_ratio_impl.rs delete mode 100644 frame/staking/bag-thresholds/tests/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 6323931a96459..c391dee935205 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2261,6 +2261,19 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "git2" +version = "0.13.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url 2.2.1", +] + [[package]] name = "glob" version = "0.3.0" @@ -3140,6 +3153,18 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" +[[package]] +name = "libgit2-sys" +version = "0.12.21+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.5.2" @@ -3635,6 +3660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] @@ -5486,6 +5512,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", + "git2", "hex", "log", "pallet-authorship", @@ -5530,16 +5557,6 @@ dependencies = [ "sp-arithmetic", ] -[[package]] -name = "pallet-staking-voter-bag-thresholds" -version = "3.0.0" -dependencies = [ - "proc-macro-crate 1.0.0", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pallet-sudo" version = "3.0.0" diff --git a/Cargo.toml b/Cargo.toml index bec00c77bcbe5..f7552f0bbbc48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,7 +108,6 @@ members = [ "frame/session/benchmarking", "frame/society", "frame/staking", - "frame/staking/bag-thresholds", "frame/staking/reward-curve", "frame/staking/reward-fn", "frame/sudo", diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 3e080f2537014..1988e804c54e0 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -33,6 +33,9 @@ paste = "1.0" frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true } rand_chacha = { version = "0.2", default-features = false, optional = true } +# Optional imports for making voter bags lists +git2 = { version = "0.13.20", default-features = false, optional = true } + [dev-dependencies] sp-storage = { version = "3.0.0", path = "../../primitives/storage" } sp-tracing = { version = "3.0.0", path = "../../primitives/tracing" } @@ -71,8 +74,8 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] -make-bags = [] +make-bags = ["std", "git2"] [[bin]] name = "make_bags" -required-features = ["make-bags", "std"] +required-features = ["make-bags"] diff --git a/frame/staking/bag-thresholds/Cargo.toml b/frame/staking/bag-thresholds/Cargo.toml deleted file mode 100644 index 44454eb4e28e6..0000000000000 --- a/frame/staking/bag-thresholds/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "pallet-staking-voter-bag-thresholds" -version = "3.0.0" -authors = ["Parity Technologies "] -edition = "2018" -license = "Apache-2.0" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "Voter Bag Thresholds for FRAME Staking Pallet" - -[lib] -proc-macro = true - -[dependencies] -proc-macro2 = "1.0.6" -proc-macro-crate = "1.0.0" -quote = "1.0.3" -syn = { version = "1.0.58", features = ["full", "visit"] } diff --git a/frame/staking/bag-thresholds/src/common.rs b/frame/staking/bag-thresholds/src/common.rs deleted file mode 100644 index 4aa5fc4892066..0000000000000 --- a/frame/staking/bag-thresholds/src/common.rs +++ /dev/null @@ -1,55 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Features common between the other modules of this crate. - -use syn::{Expr, Result, Token, Type, parse::{Parse, ParseStream}}; - -/// Parse function-like input parameters -/// -/// ```ignore -/// fn my_fn(n: usize, VoteWeight: Type, existential_weight: VoteWeight); -/// ``` -/// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any -/// expression which is evaluable in a const context. -/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a -/// `::MAX` attribute available. -/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, -/// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. It may be any expression which is evaluable in a const context. -pub struct ThresholdParams { - pub n: Expr, - pub comma1: Token![,], - pub vote_weight: Type, - pub comma2: Token![,], - pub existential_weight: Expr, - pub comma3: Option, -} - -impl Parse for ThresholdParams { - fn parse(input: ParseStream) -> Result { - Ok(Self { - n: input.parse()?, - comma1: input.parse()?, - vote_weight: input.parse()?, - comma2: input.parse()?, - existential_weight: input.parse()?, - comma3: input.parse()?, - }) - } -} diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs deleted file mode 100644 index 6ae7c68e42516..0000000000000 --- a/frame/staking/bag-thresholds/src/lib.rs +++ /dev/null @@ -1,134 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Proc macros to generate voter bag thresholds and associated items. - -use proc_macro::TokenStream; - -mod common; -mod make_ratio_impl; - -/// Calculate an appropriate constant ratio between thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; -/// ``` -/// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any -/// expression which is evaluable in a const context. -/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a -/// `::MAX` attribute available. -/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, -/// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. It may be any expression which is evaluable in a const context. -/// -/// # Example: -/// -/// ``` -/// # use pallet_staking_voter_bag_thresholds::make_ratio; -/// /// Constant ratio between bag items. Approx. `1.248`. -/// const CONSTANT_RATIO: f64 = make_ratio!(200, u64, 0); -/// ``` -/// -/// # Calculation -/// -/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). -#[proc_macro] -pub fn make_ratio(input: TokenStream) -> TokenStream { - make_ratio_impl::make_ratio(input) -} - -/// Make a constant array of threshold values suitable for use as voter bag thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn make_thresholds(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> [VoteWeight; n]; -/// ``` -/// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any -/// expression which is evaluable in a const context. -/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a -/// `::MAX` attribute available. -/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, -/// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. It may be any expression which is evaluable in a const context. -/// -/// The output has these properties: -/// -/// - Its length is `n`. -/// - Its first item is greater than or equal to `existential_weight`. -/// - Its last item is equal to `VoteWeight::MAX`. -/// - There exists a constant ratio (see [`make_ratio`]) called _ratio_. -/// -/// For all _k_ in `0..(n-1)`, `output[k + 1] == (output[k] * ratio).round()`. -/// -/// However, there are two exceptions to the ratio rule: -/// -/// - As thresholds may not duplicate, if `(output[k] * ratio).round() == output[k]`, then `output[k -/// + 1] == output[k] + 1`. -/// - Due to the previous exception in combination with the requirement that the final item is equal -/// to `VoteWeight::MAX`, the ratio of the final item may diverge from the common ratio. -/// -/// # Example: -/// -/// ``` -/// # use pallet_staking_voter_bag_thresholds::make_thresholds; -/// const THRESHOLDS: [u64; 200] = &make_thresholds!(200, u64, 0); -/// ``` -#[proc_macro] -pub fn make_thresholds(input: TokenStream) -> TokenStream { - todo!() -} - -/// Make a constant array of threshold values suitable for use as voter bag thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn edit_thresholds< -/// const Old: usize, -/// const Inserting: usize, -/// const Removing: usize, -/// const New: usize, -/// >( -/// thresholds: [VoteWeight; Old], -/// inserting: [VoteWeight; Inserting], -/// removing: [VoteWeight; Removing], -/// ) -> [VoteWeight; New]; -/// ``` -/// -/// It is intended to be used to simply edit a thresholds list, inserting some items and removing -/// others, without needing to express the entire list. -/// -/// Note that due to macro limitations, `inserting` and `removing` must be array literals. -/// -/// Note that `removing` overrides `inserting`: if a value appears in both lists, it does not appear -/// in the output. -/// -/// # Example: -/// -/// ``` -/// # use pallet_staking_voter_bag_thresholds::{edit_thresholds, make_thresholds}; -/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, u64, 0), [12345, 54321], []); -/// ``` -#[proc_macro] -pub fn edit_thresholds(input: TokenStream) -> TokenStream { - todo!() -} diff --git a/frame/staking/bag-thresholds/src/make_ratio_impl.rs b/frame/staking/bag-thresholds/src/make_ratio_impl.rs deleted file mode 100644 index 29812e7efec63..0000000000000 --- a/frame/staking/bag-thresholds/src/make_ratio_impl.rs +++ /dev/null @@ -1,51 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Implementation for the `make_ratio` proc macro. - -use proc_macro::TokenStream; -use quote::quote; -use syn::{parse_macro_input, parse_quote}; - -use crate::common::ThresholdParams; - -/// Calculate an appropriate constant ratio between thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; -/// ``` -/// -/// # Calculation -/// -/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). -pub fn make_ratio(input: TokenStream) -> TokenStream { - let ThresholdParams{ - n, - vote_weight, - existential_weight, - .. - } = parse_macro_input!(input as ThresholdParams); - - let n = parse_quote!(#n) as f64; - let diff_weight = parse_quote!(#vote_weight::MAX - #existential_weight) as f64; - - let ratio = (diff_weight.ln() / n).exp(); - - quote!(#ratio).into() -} diff --git a/frame/staking/bag-thresholds/tests/tests.rs b/frame/staking/bag-thresholds/tests/tests.rs deleted file mode 100644 index 7618457476719..0000000000000 --- a/frame/staking/bag-thresholds/tests/tests.rs +++ /dev/null @@ -1,30 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod make_ratio { - use pallet_staking_voter_bag_thresholds::make_ratio; - - #[test] - fn u64_200_0() { - assert_eq!(make_ratio!(200, u64, 0), 1.2483305489016119); - } - - #[test] - fn u64_64_0() { - assert_eq!(make_ratio!(64, u64, 0), 2.0); - } -} diff --git a/frame/staking/src/bin/make_bags.rs b/frame/staking/src/bin/make_bags.rs index 664c7b5470ba0..a5773c59522c3 100644 --- a/frame/staking/src/bin/make_bags.rs +++ b/frame/staking/src/bin/make_bags.rs @@ -17,7 +17,7 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::voter_bags::N_BAGS; +const N_BAGS: usize = 200; fn main() { let ratio = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); @@ -27,7 +27,7 @@ fn main() { while thresholds.len() < N_BAGS as usize { let prev_item: u64 = thresholds.last().copied().unwrap_or_default(); - let item = (prev_item as f64 * ratio).max(prev_item as f64 + 1.0); + let item = (prev_item as f64 * ratio).round().max(prev_item as f64 + 1.0); thresholds.push(item as u64); } diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index a3283bbdb0262..dd5869afe007a 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -603,3 +603,39 @@ pub enum VoterType { Validator, Nominator, } + +/// Support code to ease the process of generating voter bags. +#[cfg(feature = "make-bags")] +pub mod make_bags { + use crate::{AccountIdOf, Config}; + use frame_election_provider_support::VoteWeight; + use frame_support::traits::{Currency, CurrencyToVote}; + use std::path::{Path, PathBuf}; + + /// Return the path to a header file used in this repository if is exists. + /// + /// Just searches the git working directory root for files matching certain patterns; it's + /// pretty naive. + fn path_to_header_file() -> Option { + let repo = git2::Repository::open_from_env().ok()?; + let workdir = repo.workdir()?; + for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER"] { + let path = workdir.join(file_name); + if path.exists() { + return Some(path); + } + } + None + } + + /// Compute the existential weight for the specified configuration. + /// + /// Note that this value depends on the current issuance, a quantity known to change over time. + /// This makes the project of computing a static value suitable for inclusion in a static, + /// generated file _excitingly unstable_. + pub fn existential_weight() -> VoteWeight { + let existential_deposit = >>::minimum_balance(); + let issuance = >>::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) + } +} From 202def0ba988b599bca37b1f2f4207cf56f147c6 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 15:35:53 +0200 Subject: [PATCH 052/241] write helper functions to emit voter bags module --- Cargo.lock | 20 ++++- frame/staking/Cargo.toml | 9 +- frame/staking/src/voter_bags/mod.rs | 126 +++++++++++++++++++++++++++- 3 files changed, 148 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c391dee935205..0576c4bfa07b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4621,6 +4621,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-format" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" +dependencies = [ + "arrayvec 0.4.12", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -5508,6 +5518,7 @@ dependencies = [ name = "pallet-staking" version = "3.0.0" dependencies = [ + "chrono", "frame-benchmarking", "frame-election-provider-support", "frame-support", @@ -5515,6 +5526,7 @@ dependencies = [ "git2", "hex", "log", + "num-format", "pallet-authorship", "pallet-balances", "pallet-session", @@ -10057,18 +10069,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ "proc-macro2", "quote", diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 1988e804c54e0..cd53cc5262afc 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -34,7 +34,9 @@ frame-benchmarking = { version = "3.1.0", default-features = false, path = "../b rand_chacha = { version = "0.2", default-features = false, optional = true } # Optional imports for making voter bags lists +chrono = { version = "0.4.19", optional = true } git2 = { version = "0.13.20", default-features = false, optional = true } +num-format = { version = "0.4.0", optional = true } [dev-dependencies] sp-storage = { version = "3.0.0", path = "../../primitives/storage" } @@ -74,7 +76,12 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] -make-bags = ["std", "git2"] +make-bags = [ + "chrono", + "git2", + "num-format", + "std", +] [[bin]] name = "make_bags" diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index dd5869afe007a..c439f369744f3 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -609,8 +609,8 @@ pub enum VoterType { pub mod make_bags { use crate::{AccountIdOf, Config}; use frame_election_provider_support::VoteWeight; - use frame_support::traits::{Currency, CurrencyToVote}; - use std::path::{Path, PathBuf}; + use frame_support::traits::{Currency, CurrencyToVote, Get}; + use std::{io::Write, path::{Path, PathBuf}}; /// Return the path to a header file used in this repository if is exists. /// @@ -638,4 +638,126 @@ pub mod make_bags { let issuance = >>::total_issuance(); T::CurrencyToVote::to_vote(existential_deposit, issuance) } + + /// Compute the constant ratio for the thresholds. + /// + /// This ratio ensures that each bag, with the possible exceptions of certain small ones and the + /// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` + /// space. + pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { + (((VoteWeight::MAX - existential_weight) as f64).ln() / (n_bags as f64)).exp() + } + + /// Compute the list of bag thresholds. + /// + /// Returns a list of exactly `n_bags` elements, except in the case of overflow. + /// The first element is always `existential_weight`. + /// The last element is always `VoteWeight::MAX`. + /// + /// All other elements are computed from the previous according to the formula + /// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1); + pub fn thresholds( + existential_weight: VoteWeight, + constant_ratio: f64, + n_bags: usize, + ) -> Vec { + const WEIGHT_LIMIT: f64 = VoteWeight::MAX as f64; + + let mut thresholds = Vec::with_capacity(n_bags); + + if n_bags > 1 { + thresholds.push(existential_weight); + } + + while n_bags > 0 && thresholds.len() < n_bags - 1 { + let last = thresholds.last().copied().unwrap_or(existential_weight); + let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); + if successor < WEIGHT_LIMIT { + thresholds.push(successor as VoteWeight); + } + } + + thresholds.push(VoteWeight::MAX); + + debug_assert_eq!(thresholds.len(), n_bags); + debug_assert!(n_bags == 0 || thresholds[0] == existential_weight); + debug_assert!(n_bags == 0 || thresholds[thresholds.len() - 1] == VoteWeight::MAX); + + thresholds + } + + /// Write a thresholds module to the path specified. + /// + /// The `output` path should terminate with a Rust module name, i.e. `foo/bar/thresholds.rs`. + /// + /// This generated module contains, in order: + /// + /// - The contents of the header file in this repository's root, if found. + /// - Module documentation noting that this is autogenerated and when. + /// - Some associated constants. + /// - The constant array of thresholds. + pub fn generate_thresholds_module(n_bags: usize, output: &Path) -> Result<(), std::io::Error> { + // ensure the file is accessable + if let Some(parent) = output.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)?; + } + } + + // copy the header file + if let Some(header_path) = path_to_header_file() { + std::fs::copy(header_path, output)?; + } + + // open an append buffer + let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; + let mut buf = std::io::BufWriter::new(file); + + // module docs + let now = chrono::Utc::now(); + writeln!(buf)?; + writeln!(buf, "//! Autogenerated voter bag thresholds.")?; + writeln!(buf, "//!")?; + writeln!(buf, "//! Generated on {}", now.to_rfc3339())?; + writeln!( + buf, + "//! for the {} runtime.", + ::Version::get().spec_name, + )?; + + // existential weight + let existential_weight = existential_weight::(); + writeln!(buf)?; + writeln!(buf, "/// Existential weight for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", existential_weight)?; + + // constant ratio + let constant_ratio = constant_ratio(existential_weight, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Constant ratio between bags for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const CONSTANT_RATIO: f64 = {};", constant_ratio)?; + + // thresholds + let thresholds = thresholds(existential_weight, constant_ratio, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; + writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; + let mut num_buf = num_format::Buffer::new(); + let format = num_format::CustomFormat::builder() + .grouping(num_format::Grouping::Standard) + .separator("_") + .build() + .expect("format described here meets all constraints"); + for threshold in thresholds { + num_buf.write_formatted(&threshold, &format); + writeln!(buf, " {},", num_buf.as_str())?; + } + writeln!(buf, "];")?; + + Ok(()) + } } From 4da82040493f670464ef768f177036de585c517b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 17:27:28 +0200 Subject: [PATCH 053/241] WIP: demo generating voter bags for a realistic runtime This isn't yet done, becuase it seems to take a Very Long Time to run, and it really shouldn't. Need to look into that. Still, it's a lot closer than it was this morning. --- Cargo.lock | 19 +- Cargo.toml | 1 + bin/node/runtime/src/lib.rs | 2 + bin/node/runtime/voter-bags/Cargo.toml | 16 ++ .../node/runtime/voter-bags/src/main.rs | 42 ++-- frame/staking/Cargo.toml | 4 - .../src/{voter_bags/mod.rs => voter_bags.rs} | 35 ++- frame/staking/src/voter_bags/thresholds.rs | 227 ------------------ 8 files changed, 91 insertions(+), 255 deletions(-) create mode 100644 bin/node/runtime/voter-bags/Cargo.toml rename frame/staking/src/bin/make_bags.rs => bin/node/runtime/voter-bags/src/main.rs (53%) rename frame/staking/src/{voter_bags/mod.rs => voter_bags.rs} (96%) delete mode 100644 frame/staking/src/voter_bags/thresholds.rs diff --git a/Cargo.lock b/Cargo.lock index 0576c4bfa07b8..7fc5552cdbe35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4461,6 +4461,16 @@ dependencies = [ "substrate-wasm-builder", ] +[[package]] +name = "node-runtime-voter-bags" +version = "3.0.0" +dependencies = [ + "node-runtime", + "pallet-staking", + "sp-io", + "structopt", +] + [[package]] name = "node-template" version = "3.0.0" @@ -6764,14 +6774,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -6786,9 +6795,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "region" diff --git a/Cargo.toml b/Cargo.toml index f7552f0bbbc48..fb25a9f120784 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "bin/node/rpc", "bin/node/rpc-client", "bin/node/runtime", + "bin/node/runtime/voter-bags", "bin/node/testing", "bin/utils/chain-spec-builder", "bin/utils/subkey", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index fd7fd4213366f..8dcf3fa210258 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -481,6 +481,7 @@ parameter_types! { pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; pub OffchainRepeat: BlockNumber = 5; + pub const VoterBagThresholds: &'static [u64] = &[]; } use frame_election_provider_support::onchain; @@ -510,6 +511,7 @@ impl pallet_staking::Config for Runtime { type GenesisElectionProvider = onchain::OnChainSequentialPhragmen>; type WeightInfo = pallet_staking::weights::SubstrateWeight; + type VoterBagThresholds = VoterBagThresholds; } parameter_types! { diff --git a/bin/node/runtime/voter-bags/Cargo.toml b/bin/node/runtime/voter-bags/Cargo.toml new file mode 100644 index 0000000000000..38f58ba9d4bd0 --- /dev/null +++ b/bin/node/runtime/voter-bags/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "node-runtime-voter-bags" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Voter Bag generation script for pallet-staking and node-runtime." +readme = "README.md" + +[dependencies] +node-runtime = { version = "2.0.0", path = ".." } +pallet-staking = { version = "3.0.0", path = "../../../../frame/staking", features = ["make-bags"] } +sp-io = { version = "3.0.0", path = "../../../../primitives/io" } +structopt = "0.3.21" diff --git a/frame/staking/src/bin/make_bags.rs b/bin/node/runtime/voter-bags/src/main.rs similarity index 53% rename from frame/staking/src/bin/make_bags.rs rename to bin/node/runtime/voter-bags/src/main.rs index a5773c59522c3..56e5f47cb1b00 100644 --- a/frame/staking/src/bin/make_bags.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,24 +17,32 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -const N_BAGS: usize = 200; - -fn main() { - let ratio = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); - println!("pub const CONSTANT_RATIO: f64 = {};", ratio); - - let mut thresholds = Vec::with_capacity(N_BAGS as usize); - - while thresholds.len() < N_BAGS as usize { - let prev_item: u64 = thresholds.last().copied().unwrap_or_default(); - let item = (prev_item as f64 * ratio).round().max(prev_item as f64 + 1.0); - thresholds.push(item as u64); +use pallet_staking::voter_bags::make_bags::generate_thresholds_module; +use std::path::PathBuf; +use structopt::{clap::arg_enum, StructOpt}; + +arg_enum!{ + #[derive(Debug)] + enum Runtime { + Node, } +} - *thresholds.last_mut().unwrap() = u64::MAX; - - println!("pub const THRESHOLDS: [u64; {}] = {:#?};", N_BAGS, thresholds); +#[derive(Debug, StructOpt)] +struct Opt { + /// How many bags to generate. + #[structopt( + long, + default_value = "200", + )] + n_bags: usize, + + /// Where to write the output. + output: PathBuf, +} - debug_assert_eq!(thresholds.len(), N_BAGS as usize); - debug_assert_eq!(*thresholds.last().unwrap(), u64::MAX); +fn main() -> Result<(), std::io::Error> { + let Opt {n_bags, output} = Opt::from_args(); + let mut ext = sp_io::TestExternalities::new_empty(); + ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index cd53cc5262afc..d69d1d4da82c3 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -82,7 +82,3 @@ make-bags = [ "num-format", "std", ] - -[[bin]] -name = "make_bags" -required-features = ["make-bags"] diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags.rs similarity index 96% rename from frame/staking/src/voter_bags/mod.rs rename to frame/staking/src/voter_bags.rs index c439f369744f3..b0d6ed9519475 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags.rs @@ -21,8 +21,6 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -pub mod thresholds; - use crate::{ AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, VotingDataOf, slashing::SlashingSpans, @@ -605,6 +603,39 @@ pub enum VoterType { } /// Support code to ease the process of generating voter bags. +/// +/// The process of adding voter bags to a runtime requires only four steps. +/// +/// 1. Update the runtime definition. +/// +/// ```ignore +/// parameter_types!{ +/// pub const VoterBagThresholds: &'static [u64] = &[]; +/// } +/// +/// impl pallet_staking::Config for Runtime { +/// // +/// type VoterBagThresholds = VoterBagThresholds; +/// } +/// ``` +/// +/// 2. Write a little program to generate the definitions. This can be a near-identical copy of +/// `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime +/// definitions with the +/// +/// 3. Run that program: +/// +/// ```sh,notrust +/// $ cargo run -p node-runtime-voter-bags -- bin/node/runtime/src/voter_bags.rs +/// ``` +/// +/// 4. Update the runtime definition. +/// +/// ```diff,notrust +/// + mod voter_bags; +/// - pub const VoterBagThresholds: &'static [u64] = &[]; +/// + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; +/// ``` #[cfg(feature = "make-bags")] pub mod make_bags { use crate::{AccountIdOf, Config}; diff --git a/frame/staking/src/voter_bags/thresholds.rs b/frame/staking/src/voter_bags/thresholds.rs deleted file mode 100644 index 37b889405ac19..0000000000000 --- a/frame/staking/src/voter_bags/thresholds.rs +++ /dev/null @@ -1,227 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Generated voter bag thresholds. - -/// Ratio between adjacent bags; -#[cfg(any(test, feature = "std"))] -#[allow(unused)] -pub const CONSTANT_RATIO: f64 = 1.2483305489016119; - -/// Upper thresholds for each bag. -pub const THRESHOLDS: [u64; 200] = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 11, - 13, - 16, - 19, - 23, - 28, - 34, - 42, - 52, - 64, - 79, - 98, - 122, - 152, - 189, - 235, - 293, - 365, - 455, - 567, - 707, - 882, - 1101, - 1374, - 1715, - 2140, - 2671, - 3334, - 4161, - 5194, - 6483, - 8092, - 10101, - 12609, - 15740, - 19648, - 24527, - 30617, - 38220, - 47711, - 59559, - 74349, - 92812, - 115860, - 144631, - 180547, - 225382, - 281351, - 351219, - 438437, - 547314, - 683228, - 852894, - 1064693, - 1329088, - 1659141, - 2071156, - 2585487, - 3227542, - 4029039, - 5029572, - 6278568, - 7837728, - 9784075, - 12213759, - 15246808, - 19033056, - 23759545, - 29659765, - 37025190, - 46219675, - 57697432, - 72025466, - 89911589, - 112239383, - 140111850, - 174905902, - 218340380, - 272560966, - 340246180, - 424739700, - 530215542, - 661884258, - 826250339, - 1031433539, - 1287569995, - 1607312958, - 2006457867, - 2504722650, - 3126721800, - 3903182340, - 4872461752, - 6082442853, - 7592899225, - 9478448057, - 11832236265, - 14770541991, - 18438518791, - 23017366283, - 28733281486, - 35868633049, - 44775910382, - 55895136784, - 69775606782, - 87103021514, - 108733362657, - 135735178289, - 169442369618, - 211520086272, - 264046985399, - 329617918218, - 411472116776, - 513653213392, - 641208997818, - 800440780206, - 999214678517, - 1247350208103, - 1557105369953, - 1943782201171, - 2426482702132, - 3029052483452, - 3781258749319, - 4720260810076, - 5892445768000, - 7355720059940, - 9182370059991, - 11462633057206, - 14309155016159, - 17862555335640, - 22298373506924, - 27835740839511, - 34748205641269, - 43377246621511, - 54149142084871, - 67596028261358, - 84382187063069, - 105336861893959, - 131495222627659, - 164149503440725, - 204912839732087, - 255798957699744, - 319321653273781, - 398618974707429, - 497608243499122, - 621179571745225, - 775437435763184, - 968002239825113, - 1208386767378873, - 1508466116607513, - 1883064335344139, - 2350686735357198, - 2934434062644189, - 3663143684136207, - 4572814165923224, - 5708383617772005, - 7125949654914296, - 8895540644164415, - 11104575135106362, - 13862180373726516, - 17304583234907174, - 21601839888145304, - 26966236644853160, - 33662776992680304, - 42022272880825152, - 52457686971413784, - 65484533171133904, - 81746343238087392, - 102046457525101200, - 127387710335774608, - 159021970366777056, - 198511983555374656, - 247808573395228608, - 309347012448991104, - 386167325851522816, - 482064469848099072, - 601775804251442048, - 751215120036911616, - 937764783138868096, - 1170640426476344320, - 1461346206149632000, - 1824243111658058240, - 2277258404906088192, - 2842771234587226112, - 3548718175673985024, - 4429973308136232448, - 5530071011365192704, - 6903356581082402816, - 8617670910126150656, - 10757701857491230720, - 13429167864681918464, - 18446744073709551615, -]; From 498ab6528cbacadaef1f7b40f7a8a571b422b26f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 17:29:40 +0200 Subject: [PATCH 054/241] rm unnecessary arg_enum --- bin/node/runtime/voter-bags/src/main.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 56e5f47cb1b00..2d14539cf21fb 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -19,14 +19,7 @@ use pallet_staking::voter_bags::make_bags::generate_thresholds_module; use std::path::PathBuf; -use structopt::{clap::arg_enum, StructOpt}; - -arg_enum!{ - #[derive(Debug)] - enum Runtime { - Node, - } -} +use structopt::StructOpt; #[derive(Debug, StructOpt)] struct Opt { From 734ea86d569ca648afa65f46ad6bae95dfdcf5d0 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:19:37 +0200 Subject: [PATCH 055/241] fix voter bags math Turns out that when you're working in exponential space, you need to divide, not subtract, in order to keep the math working properly. Also neaten up the output a little bit to make it easier to read. --- frame/staking/src/voter_bags.rs | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index b0d6ed9519475..88973b62c114d 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -621,7 +621,7 @@ pub enum VoterType { /// /// 2. Write a little program to generate the definitions. This can be a near-identical copy of /// `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime -/// definitions with the +/// definitions with the various calculations here. /// /// 3. Run that program: /// @@ -659,6 +659,15 @@ pub mod make_bags { None } + /// Create an underscore formatter: a formatter which inserts `_` every 3 digits of a number. + fn underscore_formatter() -> num_format::CustomFormat { + num_format::CustomFormat::builder() + .grouping(num_format::Grouping::Standard) + .separator("_") + .build() + .expect("format described here meets all constraints") + } + /// Compute the existential weight for the specified configuration. /// /// Note that this value depends on the current issuance, a quantity known to change over time. @@ -676,7 +685,7 @@ pub mod make_bags { /// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` /// space. pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { - (((VoteWeight::MAX - existential_weight) as f64).ln() / (n_bags as f64)).exp() + ((VoteWeight::MAX as f64 / existential_weight as f64).ln() / ((n_bags - 1) as f64)).exp() } /// Compute the list of bag thresholds. @@ -705,6 +714,9 @@ pub mod make_bags { let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); if successor < WEIGHT_LIMIT { thresholds.push(successor as VoteWeight); + } else { + eprintln!("unexpectedly exceeded weight limit; breaking threshold generation loop"); + break } } @@ -744,6 +756,10 @@ pub mod make_bags { let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; let mut buf = std::io::BufWriter::new(file); + // create underscore formatter and format buffer + let mut num_buf = num_format::Buffer::new(); + let format = underscore_formatter(); + // module docs let now = chrono::Utc::now(); writeln!(buf)?; @@ -758,11 +774,12 @@ pub mod make_bags { // existential weight let existential_weight = existential_weight::(); + num_buf.write_formatted(&existential_weight, &format); writeln!(buf)?; writeln!(buf, "/// Existential weight for this runtime.")?; writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; writeln!(buf, "#[allow(unused)]")?; - writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", existential_weight)?; + writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", num_buf.as_str())?; // constant ratio let constant_ratio = constant_ratio(existential_weight, n_bags); @@ -777,15 +794,10 @@ pub mod make_bags { writeln!(buf)?; writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; - let mut num_buf = num_format::Buffer::new(); - let format = num_format::CustomFormat::builder() - .grouping(num_format::Grouping::Standard) - .separator("_") - .build() - .expect("format described here meets all constraints"); for threshold in thresholds { num_buf.write_formatted(&threshold, &format); - writeln!(buf, " {},", num_buf.as_str())?; + // u64::MAX, with spacers every 3 digits, is 26 characters wide + writeln!(buf, " {:>26},", num_buf.as_str())?; } writeln!(buf, "];")?; From 3cfa8cf8f7004f747e6f863b6dc5612bf99cb95c Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:31:21 +0200 Subject: [PATCH 056/241] add computed voter bags thresholds to node --- bin/node/runtime/src/lib.rs | 5 +- bin/node/runtime/src/voter_bags.rs | 235 +++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 bin/node/runtime/src/voter_bags.rs diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 8dcf3fa210258..c4c394175d446 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -93,6 +93,9 @@ pub mod constants; use constants::{time::*, currency::*}; use sp_runtime::generic::Era; +/// Generated voter bag information. +mod voter_bags; + // Make the WASM binary available. #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); @@ -481,7 +484,7 @@ parameter_types! { pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; pub OffchainRepeat: BlockNumber = 5; - pub const VoterBagThresholds: &'static [u64] = &[]; + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; } use frame_election_provider_support::onchain; diff --git a/bin/node/runtime/src/voter_bags.rs b/bin/node/runtime/src/voter_bags.rs new file mode 100644 index 0000000000000..f8f6201e94dc3 --- /dev/null +++ b/bin/node/runtime/src/voter_bags.rs @@ -0,0 +1,235 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated voter bag thresholds. +//! +//! Generated on 2021-07-05T09:17:40.469754927+00:00 +//! for the node runtime. + +/// Existential weight for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const EXISTENTIAL_WEIGHT: u64 = 100_000_000_000_000; + +/// Constant ratio between bags for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 1.0628253590743408; + +/// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 200] = [ + 100_000_000_000_000, + 106_282_535_907_434, + 112_959_774_389_150, + 120_056_512_776_105, + 127_599_106_300_477, + 135_615_565_971_369, + 144_135_662_599_590, + 153_191_037_357_827, + 162_815_319_286_803, + 173_044_250_183_800, + 183_915_817_337_347, + 195_470_394_601_017, + 207_750_892_330_229, + 220_802_916_738_890, + 234_674_939_267_673, + 249_418_476_592_914, + 265_088_281_944_639, + 281_742_548_444_211, + 299_443_125_216_738, + 318_255_747_080_822, + 338_250_278_668_647, + 359_500_973_883_001, + 382_086_751_654_776, + 406_091_489_025_036, + 431_604_332_640_068, + 458_720_029_816_222, + 487_539_280_404_019, + 518_169_110_758_247, + 550_723_271_202_866, + 585_322_658_466_782, + 622_095_764_659_305, + 661_179_154_452_653, + 702_717_972_243_610, + 746_866_481_177_808, + 793_788_636_038_393, + 843_658_692_126_636, + 896_661_852_395_681, + 952_994_955_240_703, + 1_012_867_205_499_736, + 1_076_500_951_379_881, + 1_144_132_510_194_192, + 1_216_013_045_975_769, + 1_292_409_502_228_280, + 1_373_605_593_276_862, + 1_459_902_857_901_004, + 1_551_621_779_162_291, + 1_649_102_974_585_730, + 1_752_708_461_114_642, + 1_862_822_999_536_805, + 1_979_855_523_374_646, + 2_104_240_657_545_975, + 2_236_440_332_435_128, + 2_376_945_499_368_703, + 2_526_277_953_866_680, + 2_684_992_273_439_945, + 2_853_677_877_130_641, + 3_032_961_214_443_876, + 3_223_508_091_799_862, + 3_426_026_145_146_232, + 3_641_267_467_913_124, + 3_870_031_404_070_482, + 4_113_167_516_660_186, + 4_371_578_742_827_277, + 4_646_224_747_067_156, + 4_938_125_485_141_739, + 5_248_364_991_899_922, + 5_578_095_407_069_235, + 5_928_541_253_969_291, + 6_301_003_987_036_955, + 6_696_866_825_051_405, + 7_117_599_888_008_300, + 7_564_765_656_719_910, + 8_040_024_775_416_580, + 8_545_142_218_898_723, + 9_081_993_847_142_344, + 9_652_573_371_700_016, + 10_258_999_759_768_490, + 10_903_525_103_419_522, + 11_588_542_983_217_942, + 12_316_597_357_287_042, + 13_090_392_008_832_678, + 13_912_800_587_211_472, + 14_786_877_279_832_732, + 15_715_868_154_526_436, + 16_703_223_214_499_558, + 17_752_609_210_649_358, + 18_867_923_258_814_856, + 20_053_307_312_537_008, + 21_313_163_545_075_252, + 22_652_170_697_804_756, + 24_075_301_455_707_600, + 25_587_840_914_485_432, + 27_195_406_207_875_088, + 28_903_967_368_057_400, + 30_719_869_496_628_636, + 32_649_856_328_471_220, + 34_701_095_276_033_064, + 36_881_204_047_022_752, + 39_198_278_934_370_992, + 41_660_924_883_519_016, + 44_278_287_448_695_240, + 47_060_086_756_856_400, + 50_016_653_605_425_536, + 53_158_967_827_883_320, + 56_498_699_069_691_424, + 60_048_250_125_977_912, + 63_820_803_001_928_304, + 67_830_367_866_937_216, + 72_091_835_084_322_176, + 76_621_030_509_822_880, + 81_434_774_264_248_528, + 86_550_943_198_537_824, + 91_988_537_283_208_848, + 97_767_750_168_749_840, + 103_910_044_178_992_000, + 110_438_230_015_967_792, + 117_376_551_472_255_616, + 124_750_775_465_407_920, + 132_588_287_728_824_640, + 140_918_194_514_440_064, + 149_771_430_684_917_568, + 159_180_874_596_775_264, + 169_181_470_201_085_280, + 179_810_356_815_193_344, + 191_107_007_047_393_216, + 203_113_373_386_768_288, + 215_874_044_002_592_672, + 229_436_408_331_885_600, + 243_850_833_070_063_392, + 259_170_849_218_267_264, + 275_453_350_882_006_752, + 292_758_806_559_399_232, + 311_151_483_703_668_992, + 330_699_687_393_865_920, + 351_476_014_000_157_824, + 373_557_620_785_735_808, + 397_026_512_446_556_096, + 421_969_845_653_044_224, + 448_480_252_724_740_928, + 476_656_185_639_923_904, + 506_602_281_657_757_760, + 538_429_751_910_786_752, + 572_256_794_410_890_176, + 608_209_033_002_485_632, + 646_419_983_893_124_352, + 687_031_551_494_039_552, + 730_194_555_412_054_016, + 776_069_290_549_944_960, + 824_826_122_395_314_176, + 876_646_119_708_695_936, + 931_721_726_960_522_368, + 990_257_479_014_182_144, + 1_052_470_760_709_299_712, + 1_118_592_614_166_106_112, + 1_188_868_596_808_997_376, + 1_263_559_693_295_730_432, + 1_342_943_284_738_898_688, + 1_427_314_178_819_094_784, + 1_516_985_704_615_302_400, + 1_612_290_876_218_400_768, + 1_713_583_629_449_105_408, + 1_821_240_136_273_157_632, + 1_935_660_201_795_120_128, + 2_057_268_749_018_809_600, + 2_186_517_396_888_336_384, + 2_323_886_137_470_138_880, + 2_469_885_118_504_583_168, + 2_625_056_537_947_004_416, + 2_789_976_657_533_970_944, + 2_965_257_942_852_572_160, + 3_151_551_337_860_326_400, + 3_349_548_682_302_620_672, + 3_559_985_281_005_267_968, + 3_783_642_634_583_792_128, + 4_021_351_341_710_503_936, + 4_273_994_183_717_548_544, + 4_542_509_402_991_247_872, + 4_827_894_187_332_742_144, + 5_131_208_373_224_844_288, + 5_453_578_381_757_959_168, + 5_796_201_401_831_965_696, + 6_160_349_836_169_256_960, + 6_547_376_026_650_146_816, + 6_958_717_276_519_173_120, + 7_395_901_188_113_309_696, + 7_860_551_335_934_872_576, + 8_354_393_296_137_270_272, + 8_879_261_054_815_360_000, + 9_437_103_818_898_946_048, + 10_029_993_254_943_105_024, + 10_660_131_182_698_121_216, + 11_329_857_752_030_707_712, + 12_041_660_133_563_240_448, + 12_798_181_755_305_525_248, + 13_602_232_119_581_272_064, + 14_456_797_236_706_498_560, + 15_365_050_714_167_523_328, + 16_330_365_542_480_556_032, + 17_356_326_621_502_140_416, + 18_446_744_073_709_551_615, +]; From a05df6996cdbc8f072ba7690a268775c03f0f3de Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:47:28 +0200 Subject: [PATCH 057/241] fixup some docs --- frame/staking/src/lib.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 7f6b5402cff45..e3fc886c0c93f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -968,8 +968,9 @@ pub mod pallet { /// voter's weight is less than or equal to its upper threshold. /// /// When voters are iterated, higher bags are iterated completely before lower bags. This - /// that iteration is _semi-sorted_: voters of higher weight tend to come before voters of - /// lower weight, but peer voters within a particular bag are sorted in insertion order. + /// means that iteration is _semi-sorted_: voters of higher weight tend to come before + /// voters of lower weight, but peer voters within a particular bag are sorted in insertion + /// order. /// /// # Expressing the constant /// @@ -987,8 +988,8 @@ pub mod pallet { /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * /// constant_ratio).max(threshold[k] + 1)` for all `k`. /// - /// Given the desire to compute `N` bags, the constant ratio can be computed with - /// `exp(ln(VoterBags::MAX) / N)`. + /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use + /// them, the `make-bags` feature must be enabled. /// /// # Examples /// @@ -1004,8 +1005,9 @@ pub mod pallet { /// /// # Migration /// - /// In the event that this list ever changes, the `VoterList` data structure will need to be - /// regenerated. + /// In the event that this list ever changes, a copy of the old bags list must be retained. + /// With that `VoterList::migrate` can be called, which will perform the appropriate + /// migration. #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; } From 2838b83ae63d94d57f2a20b6b54d00ba62589a13 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:48:06 +0200 Subject: [PATCH 058/241] iter from large bags to small, fulfuilling the contract --- frame/staking/src/voter_bags.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 88973b62c114d..559b84eb7adb4 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -108,6 +108,7 @@ impl VoterList { pub fn iter() -> impl Iterator> { T::VoterBagThresholds::get() .iter() + .rev() .copied() .filter_map(Bag::get) .flat_map(|bag| bag.iter()) From 3f177243bf35818586958cb74b26efda3177974d Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 14:14:59 +0200 Subject: [PATCH 059/241] make tests compile --- bin/node/runtime/voter-bags/src/main.rs | 35 ++++++++- frame/staking/Cargo.toml | 14 +++- frame/staking/src/benchmarking.rs | 2 +- frame/staking/src/lib.rs | 4 +- frame/staking/src/mock.rs | 9 ++- frame/staking/src/mock/voter_bags.rs | 100 ++++++++++++++++++++++++ frame/staking/src/tests.rs | 4 +- frame/staking/src/voter_bags.rs | 6 +- 8 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 frame/staking/src/mock/voter_bags.rs diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 2d14539cf21fb..fbd86adb185e0 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -18,8 +18,26 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. use pallet_staking::voter_bags::make_bags::generate_thresholds_module; -use std::path::PathBuf; -use structopt::StructOpt; +use pallet_staking::mock::Test; +use std::path::{Path, PathBuf}; +use structopt::{StructOpt, clap::arg_enum}; + +arg_enum!{ + #[derive(Debug)] + enum Runtime { + Node, + StakingMock, + } +} + +impl Runtime { + fn generate_thresholds(&self) -> Box Result<(), std::io::Error>> { + match self { + Runtime::Node => Box::new(generate_thresholds_module::), + Runtime::StakingMock => Box::new(generate_thresholds_module::), + } + } +} #[derive(Debug, StructOpt)] struct Opt { @@ -30,12 +48,21 @@ struct Opt { )] n_bags: usize, + /// Which runtime to generate. + #[structopt( + long, + case_insensitive = true, + default_value = "Node", + possible_values = &Runtime::variants(), + )] + runtime: Runtime, + /// Where to write the output. output: PathBuf, } fn main() -> Result<(), std::io::Error> { - let Opt {n_bags, output} = Opt::from_args(); + let Opt {n_bags, output, runtime } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) + ext.execute_with(|| runtime.generate_thresholds()(n_bags, &output)) } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index d69d1d4da82c3..d4d25c86c3940 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -16,14 +16,19 @@ targets = ["x86_64-unknown-linux-gnu"] static_assertions = "1.1.0" serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +sp-core = { version = "3.0.0", path = "../../primitives/core", optional = true } sp-std = { version = "3.0.0", default-features = false, path = "../../primitives/std" } sp-io ={ version = "3.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "3.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "3.0.0", default-features = false, path = "../../primitives/staking" } +sp-tracing = { version = "3.0.0", path = "../../primitives/tracing", optional = true } frame-support = { version = "3.0.0", default-features = false, path = "../support" } frame-system = { version = "3.0.0", default-features = false, path = "../system" } -pallet-session = { version = "3.0.0", default-features = false, features = ["historical"], path = "../session" } pallet-authorship = { version = "3.0.0", default-features = false, path = "../authorship" } +pallet-balances = { version = "3.0.0", path = "../balances", optional = true } +pallet-session = { version = "3.0.0", default-features = false, features = ["historical"], path = "../session" } +pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve", optional = true } +pallet-timestamp = { version = "3.0.0", path = "../timestamp", optional = true } sp-application-crypto = { version = "3.0.0", default-features = false, path = "../../primitives/application-crypto" } frame-election-provider-support = { version = "3.0.0", default-features = false, path = "../election-provider-support" } log = { version = "0.4.14", default-features = false } @@ -45,7 +50,7 @@ sp-core = { version = "3.0.0", path = "../../primitives/core" } sp-npos-elections = { version = "3.0.0", path = "../../primitives/npos-elections", features = ["mocks"] } pallet-balances = { version = "3.0.0", path = "../balances" } pallet-timestamp = { version = "3.0.0", path = "../timestamp" } -pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } +pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } substrate-test-utils = { version = "3.0.0", path = "../../test-utils" } frame-benchmarking = { version = "3.1.0", path = "../benchmarking" } frame-election-provider-support = { version = "3.0.0", features = ["runtime-benchmarks"], path = "../election-provider-support" } @@ -80,5 +85,10 @@ make-bags = [ "chrono", "git2", "num-format", + "pallet-staking-reward-curve", + "pallet-balances", + "pallet-timestamp", + "sp-core", + "sp-tracing", "std", ] diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 208769ce1ccac..fbae528a95058 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -700,7 +700,7 @@ benchmarks! { ); ensure!( { - let origin_bag = Bag::::get(node.bag_idx).ok_or("origin bag not found")?; + let origin_bag = Bag::::get(node.bag_upper).ok_or("origin bag not found")?; origin_bag.iter().count() == 1 }, "stash should be the only node in origin bag", diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index e3fc886c0c93f..0c761e2528bd6 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -274,8 +274,8 @@ #![recursion_limit = "128"] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(test)] -mod mock; +#[cfg(any(test, feature = "make-bags"))] +pub mod mock; #[cfg(test)] mod tests; #[cfg(any(feature = "runtime-benchmarks", test))] diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index d9cc77653e986..a584b83c539c8 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,6 +17,12 @@ //! Test utilities +// This module needs to exist when the `make-bags` feature is enabled so that we can generate the +// appropriate thresholds, but we don't care if it's mostly unused in that case. +#![cfg_attr(feature = "make-bags", allow(unused))] + +mod voter_bags; + use crate::*; use crate as staking; use frame_support::{ @@ -244,7 +250,7 @@ impl onchain::Config for Test { } parameter_types! { - pub const VoterBagThresholds: &'static [VoteWeight] = &crate::voter_bags::thresholds::THRESHOLDS; + pub const VoterBagThresholds: &'static [VoteWeight] = &voter_bags::THRESHOLDS; } impl Config for Test { @@ -268,7 +274,6 @@ impl Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); type VoterBagThresholds = VoterBagThresholds; - type BagIdx = u8; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/mock/voter_bags.rs b/frame/staking/src/mock/voter_bags.rs new file mode 100644 index 0000000000000..453b03e36ab22 --- /dev/null +++ b/frame/staking/src/mock/voter_bags.rs @@ -0,0 +1,100 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated voter bag thresholds. +//! +//! Generated on 2021-07-05T12:08:52.871368217+00:00 +//! for the test runtime. + +/// Existential weight for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const EXISTENTIAL_WEIGHT: u64 = 1; + +/// Constant ratio between bags for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 2.0000000000000000; + +/// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 65] = [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1_024, + 2_048, + 4_096, + 8_192, + 16_384, + 32_768, + 65_536, + 131_072, + 262_144, + 524_288, + 1_048_576, + 2_097_152, + 4_194_304, + 8_388_608, + 16_777_216, + 33_554_432, + 67_108_864, + 134_217_728, + 268_435_456, + 536_870_912, + 1_073_741_824, + 2_147_483_648, + 4_294_967_296, + 8_589_934_592, + 17_179_869_184, + 34_359_738_368, + 68_719_476_736, + 137_438_953_472, + 274_877_906_944, + 549_755_813_888, + 1_099_511_627_776, + 2_199_023_255_552, + 4_398_046_511_104, + 8_796_093_022_208, + 17_592_186_044_416, + 35_184_372_088_832, + 70_368_744_177_664, + 140_737_488_355_328, + 281_474_976_710_656, + 562_949_953_421_312, + 1_125_899_906_842_624, + 2_251_799_813_685_248, + 4_503_599_627_370_496, + 9_007_199_254_740_992, + 18_014_398_509_481_984, + 36_028_797_018_963_968, + 72_057_594_037_927_936, + 144_115_188_075_855_872, + 288_230_376_151_711_744, + 576_460_752_303_423_488, + 1_152_921_504_606_846_976, + 2_305_843_009_213_693_952, + 4_611_686_018_427_387_904, + 9_223_372_036_854_775_808, + 18_446_744_073_709_551_615, +]; diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index e0e6b83d4b742..864fcb0c35683 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3909,7 +3909,7 @@ fn test_rebag() { let node = Node::::from_id(&stash).unwrap(); assert_eq!( { - let origin_bag = Bag::::get(node.bag_idx).unwrap(); + let origin_bag = Bag::::get(node.bag_upper).unwrap(); origin_bag.iter().count() }, 1, @@ -3919,7 +3919,7 @@ fn test_rebag() { assert!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); assert_ne!( { - let destination_bag = Bag::::get(other_node.bag_idx); + let destination_bag = Bag::::get(other_node.bag_upper); destination_bag.iter().count() }, 0, diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 559b84eb7adb4..1d748142f6ce7 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -556,14 +556,14 @@ impl Node { existed } - /// Get the index of the bag that this node _should_ be in, given its vote weight. + /// Get the upper threshold of the bag that this node _should_ be in, given its vote weight. /// /// This is a helper intended only for benchmarking and should not be used in production. #[cfg(any(test, feature = "runtime-benchmarks"))] pub fn proper_bag_for(&self) -> VoteWeight { let weight_of = crate::Pallet::::weight_of_fn(); let current_weight = weight_of(&self.voter.id); - notional_bag_for(current_weight) + notional_bag_for::(current_weight) } } @@ -788,7 +788,7 @@ pub mod make_bags { writeln!(buf, "/// Constant ratio between bags for this runtime.")?; writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; writeln!(buf, "#[allow(unused)]")?; - writeln!(buf, "pub const CONSTANT_RATIO: f64 = {};", constant_ratio)?; + writeln!(buf, "pub const CONSTANT_RATIO: f64 = {:.16};", constant_ratio)?; // thresholds let thresholds = thresholds(existential_weight, constant_ratio, n_bags); From b73fd7f2c46d4699739467b8c7b5d843e50b3f6e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 14:56:12 +0200 Subject: [PATCH 060/241] add VoterBagThresholds to some configs --- frame/babe/src/mock.rs | 1 + frame/grandpa/src/mock.rs | 1 + frame/offences/benchmarking/src/mock.rs | 1 + frame/session/benchmarking/src/mock.rs | 1 + 4 files changed, 4 insertions(+) diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 6c1cc89cf1ed0..3a18c11616c05 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -215,6 +215,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_offences::Config for Test { diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index ebe5996c9dab5..27f7b3d488437 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -221,6 +221,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_offences::Config for Test { diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index cd72780ec5ad2..84527ea4ab033 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -180,6 +180,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_im_online::Config for Test { diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 591e54f067bb5..6415dd8a9fede 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -185,6 +185,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl crate::Config for Test {} From 4861789b0bc0ca027879beb75b558ec0d8f6d203 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 6 Jul 2021 14:08:11 +0200 Subject: [PATCH 061/241] ensure that iteration covers all voters even with implied final bag --- frame/staking/src/voter_bags.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 1d748142f6ce7..b10fdac4eb374 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -26,9 +26,9 @@ use crate::{ VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; -use frame_support::{DefaultNoBound, traits::Get}; +use frame_support::{DebugNoBound, DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; -use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, marker::PhantomData}; +use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, iter, marker::PhantomData}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; @@ -106,10 +106,22 @@ impl VoterList { /// /// Full iteration can be expensive; it's recommended to limit the number of items with `.take(n)`. pub fn iter() -> impl Iterator> { - T::VoterBagThresholds::get() - .iter() - .rev() - .copied() + // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to + // omit the final bound, we need to ensure that we explicitly include that threshold in the + // list. + // + // It's important to retain the ability to omit the final bound because it makes tests much + // easier; they can just configure `type VoterBagThresholds = ()`. + let thresholds = T::VoterBagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter.rev()) + } else { + // otherwise, insert it here. + Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) + }; + iter .filter_map(Bag::get) .flat_map(|bag| bag.iter()) } @@ -331,6 +343,7 @@ impl VoterList { /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. #[derive(DefaultNoBound, Encode, Decode)] +#[cfg_attr(feature = "std", derive(DebugNoBound))] pub struct Bag { head: Option>, tail: Option>, @@ -444,7 +457,7 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] +#[cfg_attr(feature = "std", derive(DebugNoBound))] pub struct Node { voter: Voter>, prev: Option>, From 59041b7250a4ebd95fdf76458d84de87b5b9511f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 6 Jul 2021 14:30:22 +0200 Subject: [PATCH 062/241] use sp_std::boxed::Box; --- frame/staking/src/voter_bags.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index b10fdac4eb374..cc7e124fc7cbb 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -28,7 +28,12 @@ use crate::{ use codec::{Encode, Decode}; use frame_support::{DebugNoBound, DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; -use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, iter, marker::PhantomData}; +use sp_std::{ + boxed::Box, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + iter, + marker::PhantomData, +}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; From f5c8a4d7e5a7c05e38823d22750d1afa1f0db681 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 6 Jul 2021 15:29:02 +0200 Subject: [PATCH 063/241] fix unused import --- frame/staking/src/voter_bags.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index cc7e124fc7cbb..6e65120e7ba4e 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -26,7 +26,7 @@ use crate::{ VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; -use frame_support::{DebugNoBound, DefaultNoBound, traits::Get}; +use frame_support::{DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; use sp_std::{ boxed::Box, @@ -348,7 +348,7 @@ impl VoterList { /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. #[derive(DefaultNoBound, Encode, Decode)] -#[cfg_attr(feature = "std", derive(DebugNoBound))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] pub struct Bag { head: Option>, tail: Option>, @@ -462,7 +462,7 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(DebugNoBound))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] pub struct Node { voter: Voter>, prev: Option>, From 5e05e3fd87ffa80d36a60cf07fc11a6e04a817fe Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 7 Jul 2021 10:47:08 +0200 Subject: [PATCH 064/241] add some more voter bags tests --- Cargo.lock | 35 +++++++++---------- frame/staking/Cargo.toml | 1 + frame/staking/src/voter_bags.rs | 60 ++++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec41ab10c5148..21ed44dce2826 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1701,7 +1701,7 @@ dependencies = [ "num-traits", "parity-scale-codec", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -1711,7 +1711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" dependencies = [ "byteorder", - "rand 0.8.3", + "rand 0.8.4", "rustc-hex", "static_assertions", ] @@ -3382,7 +3382,7 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log", - "rand 0.8.3", + "rand 0.8.4", "smallvec 1.6.1", "socket2 0.4.0", "void", @@ -4055,7 +4055,7 @@ dependencies = [ "num-complex", "num-rational 0.4.0", "num-traits", - "rand 0.8.3", + "rand 0.8.4", "rand_distr", "simba", "typenum", @@ -4919,7 +4919,7 @@ dependencies = [ "paste 1.0.4", "pretty_assertions 0.7.2", "pwasm-utils", - "rand 0.8.3", + "rand 0.8.4", "rand_pcg 0.3.0", "serde", "smallvec 1.6.1", @@ -5532,6 +5532,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.11.1", "paste 1.0.4", + "rand 0.8.4", "rand_chacha 0.2.2", "serde", "sp-application-crypto", @@ -5773,7 +5774,7 @@ dependencies = [ "log", "memmap2", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -6476,7 +6477,7 @@ checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ "env_logger 0.8.3", "log", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -6544,9 +6545,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha 0.3.0", @@ -6614,7 +6615,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051b398806e42b9cd04ad9ec8f81e355d0a382c543ac6672c62f5a5b452ef142" dependencies = [ "num-traits", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -7685,7 +7686,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.11.1", "prost", - "rand 0.8.3", + "rand 0.8.4", "sc-block-builder", "sc-client-api", "sc-finality-grandpa", @@ -8704,7 +8705,7 @@ dependencies = [ "futures 0.3.15", "httparse", "log", - "rand 0.8.3", + "rand 0.8.4", "sha-1 0.9.4", ] @@ -9552,7 +9553,7 @@ dependencies = [ "lazy_static", "nalgebra", "num-traits", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -9956,7 +9957,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.3", + "rand 0.8.4", "redox_syscall 0.2.5", "remove_dir_all", "winapi 0.3.9", @@ -10626,7 +10627,7 @@ dependencies = [ "ipnet", "lazy_static", "log", - "rand 0.8.3", + "rand 0.8.4", "smallvec 1.6.1", "thiserror", "tinyvec", @@ -11292,7 +11293,7 @@ dependencies = [ "mach", "memoffset 0.6.1", "more-asserts", - "rand 0.8.3", + "rand 0.8.4", "region", "thiserror", "wasmtime-environ", @@ -11461,7 +11462,7 @@ dependencies = [ "log", "nohash-hasher", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", "static_assertions", ] diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index d4d25c86c3940..8897262431606 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -54,6 +54,7 @@ pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-cur substrate-test-utils = { version = "3.0.0", path = "../../test-utils" } frame-benchmarking = { version = "3.1.0", path = "../benchmarking" } frame-election-provider-support = { version = "3.0.0", features = ["runtime-benchmarks"], path = "../election-provider-support" } +rand = "0.8.4" rand_chacha = { version = "0.2" } parking_lot = "0.11.1" hex = "0.4" diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 6e65120e7ba4e..1772414621361 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -758,7 +758,10 @@ pub mod make_bags { /// - Module documentation noting that this is autogenerated and when. /// - Some associated constants. /// - The constant array of thresholds. - pub fn generate_thresholds_module(n_bags: usize, output: &Path) -> Result<(), std::io::Error> { + pub fn generate_thresholds_module( + n_bags: usize, + output: &Path, + ) -> Result<(), std::io::Error> { // ensure the file is accessable if let Some(parent) = output.parent() { if !parent.exists() { @@ -823,3 +826,58 @@ pub mod make_bags { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::mock::{ExtBuilder, Staking, Test}; + use frame_support::traits::Currency; + use substrate_test_utils::assert_eq_uvec; + use super::*; + + const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; + + #[test] + fn voter_list_includes_genesis_accounts() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); + assert_eq_uvec!(GENESIS_VOTER_IDS, have_voters); + }); + } + + /// This tests the property that when iterating through the `VoterList`, we iterate from higher + /// bags to lower. + #[test] + fn iteration_is_semi_sorted() { + use rand::seq::SliceRandom; + let mut rng = rand::thread_rng(); + + // Randomly sort the list of voters. Later we'll give each of these a stake such that it + // fits into a different bag. + let voters = { + let mut v = vec![0; GENESIS_VOTER_IDS.len()]; + v.copy_from_slice(&GENESIS_VOTER_IDS); + v.shuffle(&mut rng); + v + }; + + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // initialize the voters' deposits + let existential_deposit = ::Currency::minimum_balance(); + let mut balance = existential_deposit + 1; + for voter_id in voters.iter().rev() { + ::Currency::make_free_balance_be(voter_id, balance); + let controller = Staking::bonded(voter_id).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = balance; + ledger.active = balance; + Staking::update_ledger(&controller, &ledger); + Staking::do_rebag(voter_id); + + balance *= 2; + } + + let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); + assert_eq!(voters, have_voters); + }); + } +} From dbf3a84c84da3ad8ca64e68aea720e9733252882 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 7 Jul 2021 13:44:15 +0200 Subject: [PATCH 065/241] file_header.txt --- frame/staking/src/voter_bags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 1772414621361..dcdba815cdaa6 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -669,7 +669,7 @@ pub mod make_bags { fn path_to_header_file() -> Option { let repo = git2::Repository::open_from_env().ok()?; let workdir = repo.workdir()?; - for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER"] { + for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { let path = workdir.join(file_name); if path.exists() { return Some(path); From d5e0c8dedebcea88b9dc792ea3075f8d0ae2ffb1 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 9 Jul 2021 10:08:04 +0200 Subject: [PATCH 066/241] integrity test to ensure min bag exceeds existential weight --- frame/staking/src/lib.rs | 11 +++++++++++ frame/staking/src/voter_bags.rs | 27 +++++++++++++++------------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index c87c0b53556ce..5b834283acc2e 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1568,6 +1568,17 @@ pub mod pallet { T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), "Voter bag thresholds must strictly increase", ); + + assert!( + { + let existential_weight = voter_bags::existential_weight::(); + T::VoterBagThresholds::get() + .first() + .map(|&lowest_threshold| lowest_threshold >= existential_weight) + .unwrap_or(true) + }, + "Smallest bag should not be smaller than existential weight", + ); }); } } diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index dcdba815cdaa6..ff6113a18c41a 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -621,6 +621,20 @@ pub enum VoterType { Nominator, } +/// Compute the existential weight for the specified configuration. +/// +/// Note that this value depends on the current issuance, a quantity known to change over time. +/// This makes the project of computing a static value suitable for inclusion in a static, +/// generated file _excitingly unstable_. +#[cfg(any(feature = "std", feature = "make-bags"))] +pub fn existential_weight() -> VoteWeight { + use frame_support::traits::{Currency, CurrencyToVote}; + + let existential_deposit = >>::minimum_balance(); + let issuance = >>::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) +} + /// Support code to ease the process of generating voter bags. /// /// The process of adding voter bags to a runtime requires only four steps. @@ -657,7 +671,7 @@ pub enum VoterType { /// ``` #[cfg(feature = "make-bags")] pub mod make_bags { - use crate::{AccountIdOf, Config}; + use crate::{AccountIdOf, Config, voter_bags::existential_weight}; use frame_election_provider_support::VoteWeight; use frame_support::traits::{Currency, CurrencyToVote, Get}; use std::{io::Write, path::{Path, PathBuf}}; @@ -687,17 +701,6 @@ pub mod make_bags { .expect("format described here meets all constraints") } - /// Compute the existential weight for the specified configuration. - /// - /// Note that this value depends on the current issuance, a quantity known to change over time. - /// This makes the project of computing a static value suitable for inclusion in a static, - /// generated file _excitingly unstable_. - pub fn existential_weight() -> VoteWeight { - let existential_deposit = >>::minimum_balance(); - let issuance = >>::total_issuance(); - T::CurrencyToVote::to_vote(existential_deposit, issuance) - } - /// Compute the constant ratio for the thresholds. /// /// This ratio ensures that each bag, with the possible exceptions of certain small ones and the From dbfb9512120a4cec144264729649464281d52429 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 9 Jul 2021 10:22:28 +0200 Subject: [PATCH 067/241] add more debug assertions about node list length --- frame/staking/src/voter_bags.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index ff6113a18c41a..04362be720d97 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -104,7 +104,18 @@ impl VoterList { /// Decode the length of the voter list. pub fn decode_len() -> Option { - crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) + let maybe_len = crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()); + debug_assert_eq!( + maybe_len.unwrap_or_default(), + crate::VoterNodes::::iter().count(), + "stored length must match count of nodes", + ); + debug_assert_eq!( + maybe_len.unwrap_or_default() as u32, + crate::CounterForNominators::::get() + crate::CounterForValidators::::get(), + "voter count must be sum of validator and nominator count", + ); + maybe_len } /// Iterate over all nodes in all bags in the voter list. From 06e69a8e1bbd8ca2cc9a82885a0bd811a0b6ee80 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 9 Jul 2021 10:43:55 +0200 Subject: [PATCH 068/241] rm unused imports --- frame/staking/src/voter_bags.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 04362be720d97..6457770e632ea 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -682,9 +682,9 @@ pub fn existential_weight() -> VoteWeight { /// ``` #[cfg(feature = "make-bags")] pub mod make_bags { - use crate::{AccountIdOf, Config, voter_bags::existential_weight}; + use crate::{Config, voter_bags::existential_weight}; use frame_election_provider_support::VoteWeight; - use frame_support::traits::{Currency, CurrencyToVote, Get}; + use frame_support::traits::Get; use std::{io::Write, path::{Path, PathBuf}}; /// Return the path to a header file used in this repository if is exists. From 6f15cc3c0c11f67268b2dcfd38f487ff569cd102 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Sat, 10 Jul 2021 21:48:53 +0200 Subject: [PATCH 069/241] Kian enters --- frame/staking/src/lib.rs | 23 ++++++++++++++++++----- frame/staking/src/mock/voter_bags.rs | 1 + frame/staking/src/voter_bags.rs | 20 ++++++++++++++++---- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 5b834283acc2e..00458ea9f49b0 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1907,7 +1907,6 @@ pub mod pallet { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; Self::chill_stash(&ledger.stash); - VoterList::::remove(&ledger.stash); Ok(()) } @@ -2964,8 +2963,6 @@ impl Pallet { Self::do_remove_validator(stash); Self::do_remove_nominator(stash); - VoterList::::remove(stash); - frame_system::Pallet::::dec_consumers(stash); Ok(()) @@ -3089,6 +3086,10 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a nominator to the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { CounterForNominators::::mutate(|x| x.saturating_inc()) @@ -3102,6 +3103,10 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// Returns true if `who` was removed from `Nominators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_remove_nominator(who: &T::AccountId) -> bool { if Nominators::::contains_key(who) { Nominators::::remove(who); @@ -3114,10 +3119,14 @@ impl Pallet { } } - /// This function will add a validator to the `Validators` storage map, - /// and keep track of the `CounterForValidators`. + /// This function will add a validator to the `Validators` storage map, and keep track of the + /// `CounterForValidators`. /// /// If the validator already exists, their preferences will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { if !Validators::::contains_key(who) { CounterForValidators::::mutate(|x| x.saturating_inc()) @@ -3131,6 +3140,10 @@ impl Pallet { /// and keep track of the `CounterForValidators`. /// /// Returns true if `who` was removed from `Validators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_remove_validator(who: &T::AccountId) -> bool { if Validators::::contains_key(who) { Validators::::remove(who); diff --git a/frame/staking/src/mock/voter_bags.rs b/frame/staking/src/mock/voter_bags.rs index 453b03e36ab22..6177c615bc59a 100644 --- a/frame/staking/src/mock/voter_bags.rs +++ b/frame/staking/src/mock/voter_bags.rs @@ -31,6 +31,7 @@ pub const EXISTENTIAL_WEIGHT: u64 = 1; pub const CONSTANT_RATIO: f64 = 2.0000000000000000; /// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 65] = [5, 15, 25, 50, 100, 1000, 2000, 3000] pub const THRESHOLDS: [u64; 65] = [ 1, 2, diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 6457770e632ea..34efce5c59f02 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -120,7 +120,8 @@ impl VoterList { /// Iterate over all nodes in all bags in the voter list. /// - /// Full iteration can be expensive; it's recommended to limit the number of items with `.take(n)`. + /// Full iteration can be expensive; it's recommended to limit the number of items with + /// `.take(n)`. pub fn iter() -> impl Iterator> { // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to // omit the final bound, we need to ensure that we explicitly include that threshold in the @@ -157,14 +158,14 @@ impl VoterList { } /// Insert a new voter into the appropriate bag in the voter list. - pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { + fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { Self::insert_many(sp_std::iter::once(voter), weight_of); } /// Insert several voters into the appropriate bags in the voter list. /// /// This is more efficient than repeated calls to `Self::insert`. - pub fn insert_many( + fn insert_many( voters: impl IntoIterator>, weight_of: impl Fn(&T::AccountId) -> VoteWeight, ) -> u32 { @@ -459,7 +460,17 @@ impl Bag { /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()` and `node.put()` after use. fn remove_node(&mut self, node: &Node) { - node.excise(); + // TODO: we could merge this function here. + // node.excise(); + if let Some(mut prev) = node.prev() { + prev.next = self.next.clone(); + prev.put(); + } + if let Some(mut next) = node.next() { + next.prev = self.prev.clone(); + next.put(); + } + // IDEA: debug_assert! prev.next.prev == self // clear the bag head/tail pointers as necessary if self.head.as_ref() == Some(&node.voter.id) { @@ -487,6 +498,7 @@ pub struct Node { impl Node { /// Get a node by bag idx and account id. pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { + // debug_assert!(bag_upper is in Threshold) crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { node.bag_upper = bag_upper; node From 1b545e482e1fd3912cf09a7ff4fc753003736eee Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Mon, 12 Jul 2021 10:02:22 +0200 Subject: [PATCH 070/241] Update frame/election-provider-support/src/onchain.rs Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> --- frame/election-provider-support/src/onchain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index 89358fce12af8..47add4e70461c 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -78,7 +78,7 @@ impl ElectionProvider for OnChainSequen Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; let stake_map: BTreeMap = - voters.iter().map(|(v, s, _)| (v.clone(), *s)).collect(); +voters.iter().map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)).collect(); let stake_of = |w: &T::AccountId| -> VoteWeight { stake_map.get(w).cloned().unwrap_or_default() From e0a00fd187343fd2c28d2060268b82dd1f9b4cce Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 21 Jul 2021 12:23:06 -0700 Subject: [PATCH 071/241] Suggestions for #9081 (Store voters in unsorted bags) (#9328) * Add some debug asserts to node::get and remove_node * Improve the debug asserts in remove_node * improve debug asserts * Space * Remove bad assertions * Tests: WIP take_works * Take test * Doc comment * Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> * Test storage is cleaned up; * formatting * Switch to simpler thresholds * Update the storage cleanup test * Remove hardcoded values from benchmark to make it more robust * Fix tests to acces bags properly * Sanity check WIP; tests failing * Update sanity checks to be more correct * Improve storage cleanup tests * WIP remote_ext_tests * Some notes on next steps * Remove some stuff that was for remote-ext tests * Some more cleanup to reduce diff * More :clean: * Mo cleanin * small fix * A lot of changes from kian Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: kianenigma --- bin/node/runtime/voter-bags/src/main.rs | 2 +- frame/staking/src/benchmarking.rs | 10 +- frame/staking/src/lib.rs | 57 ++- frame/staking/src/mock.rs | 40 +- frame/staking/src/mock/voter_bags.rs | 101 ----- frame/staking/src/tests.rs | 34 +- frame/staking/src/voter_bags.rs | 471 +++++++++++++++++++++--- 7 files changed, 519 insertions(+), 196 deletions(-) delete mode 100644 frame/staking/src/mock/voter_bags.rs diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index fbd86adb185e0..9ae50914d6927 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -62,7 +62,7 @@ struct Opt { } fn main() -> Result<(), std::io::Error> { - let Opt {n_bags, output, runtime } = Opt::from_args(); + let Opt { n_bags, output, runtime } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); ext.execute_with(|| runtime.generate_thresholds()(n_bags, &output)) } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index fbae528a95058..3eeb739371517 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -671,11 +671,15 @@ benchmarks! { // Clean up any existing state. clear_validators_and_nominators::(); + let thresholds = T::VoterBagThresholds::get(); + // stash controls the node account - let (stash, controller) = make_validator(USER_SEED, 100)?; + let bag0_thresh = thresholds[0]; + let (stash, controller) = make_validator(USER_SEED, bag0_thresh as u32)?; - // create another validator with 3x the stake - let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; + // create another validator with more stake + let bag2_thresh = thresholds[2]; + let (other_stash, _) = make_validator(USER_SEED + 1, bag2_thresh as u32)?; // update the stash account's value/weight // diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 00458ea9f49b0..f3eb9d68968f2 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -780,6 +780,7 @@ pub mod migrations { log!(info, "Migrating staking to Releases::V8_0_0"); let migrated = VoterList::::regenerate(); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); StorageVersion::::put(Releases::V8_0_0); log!( @@ -1313,7 +1314,7 @@ pub mod pallet { /// How many voters are registered. #[pallet::storage] - pub(crate) type VoterCount = StorageValue<_, u32, ValueQuery>; + pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; /// Which bag currently contains a particular voter. /// @@ -1397,32 +1398,62 @@ pub mod pallet { MinNominatorBond::::put(self.min_nominator_bond); MinValidatorBond::::put(self.min_validator_bond); + let mut num_voters: u32 = 0; for &(ref stash, ref controller, balance, ref status) in &self.stakers { + log!( + trace, + "inserting genesis staker: {:?} => {:?} => {:?}", + stash, + balance, + status + ); assert!( T::Currency::free_balance(&stash) >= balance, "Stash does not have enough balance to bond." ); - let _ = >::bond( + + if let Err(why) = >::bond( T::Origin::from(Some(stash.clone()).into()), T::Lookup::unlookup(controller.clone()), balance, RewardDestination::Staked, - ); - let _ = match status { + ) { + // TODO: later on, fix all the tests that trigger these warnings, and + // make these assertions. Genesis stakers should all be correct! + log!(warn, "failed to bond staker at genesis: {:?}.", why); + continue; + } + match status { StakerStatus::Validator => { - >::validate( + if let Err(why) = >::validate( T::Origin::from(Some(controller.clone()).into()), Default::default(), - ) + ) { + log!(warn, "failed to validate staker at genesis: {:?}.", why); + } else { + num_voters +=1 ; + } }, StakerStatus::Nominator(votes) => { - >::nominate( + if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), - ) - }, _ => Ok(()) + ) { + log!(warn, "failed to nominate staker at genesis: {:?}.", why); + } else { + num_voters += 1; + } + } + _ => () }; } + + // all voters are inserted sanely. + assert_eq!( + CounterForVoters::::get(), + num_voters, + "not all genesis stakers were inserted into bags, something is wrong." + ); } } @@ -3096,7 +3127,7 @@ impl Pallet { } Nominators::::insert(who, nominations); VoterList::::insert_as(who, VoterType::Nominator); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); } /// This function will remove a nominator from the `Nominators` storage map, @@ -3112,7 +3143,7 @@ impl Pallet { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); true } else { false @@ -3133,7 +3164,7 @@ impl Pallet { } Validators::::insert(who, prefs); VoterList::::insert_as(who, VoterType::Validator); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); } /// This function will remove a validator from the `Validators` storage map, @@ -3149,7 +3180,7 @@ impl Pallet { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); true } else { false diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index a584b83c539c8..7c745ac5361ec 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,14 +17,9 @@ //! Test utilities -// This module needs to exist when the `make-bags` feature is enabled so that we can generate the -// appropriate thresholds, but we don't care if it's mostly unused in that case. -#![cfg_attr(feature = "make-bags", allow(unused))] - -mod voter_bags; - -use crate::*; use crate as staking; +use crate::*; +use frame_election_provider_support::onchain; use frame_support::{ assert_ok, parameter_types, traits::{Currency, FindAuthor, Get, OnInitialize, OneSessionHandler}, @@ -39,7 +34,6 @@ use sp_runtime::{ }; use sp_staking::offence::{OffenceDetails, OnOffenceHandler}; use std::{cell::RefCell, collections::HashSet}; -use frame_election_provider_support::onchain; pub const INIT_TIMESTAMP: u64 = 30_000; pub const BLOCK_TIME: u64 = 1000; @@ -249,8 +243,11 @@ impl onchain::Config for Test { type DataProvider = Staking; } +/// Thresholds used for bags. +const THRESHOLDS: [VoteWeight; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; + parameter_types! { - pub const VoterBagThresholds: &'static [VoteWeight] = &voter_bags::THRESHOLDS; + pub const VoterBagThresholds: &'static [VoteWeight] = &THRESHOLDS; } impl Config for Test { @@ -387,14 +384,9 @@ impl ExtBuilder { } fn build(self) -> sp_io::TestExternalities { sp_tracing::try_init_simple(); - let mut storage = frame_system::GenesisConfig::default() - .build_storage::() - .unwrap(); - let balance_factor = if ExistentialDeposit::get() > 1 { - 256 - } else { - 1 - }; + + let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let balance_factor = if ExistentialDeposit::get() > 1 { 256 } else { 1 }; let num_validators = self.num_validators.unwrap_or(self.validator_count); // Check that the number of validators is sensible. @@ -511,6 +503,9 @@ fn check_count() { let validator_count = Validators::::iter().count() as u32; assert_eq!(nominator_count, CounterForNominators::::get()); assert_eq!(validator_count, CounterForValidators::::get()); + + let voters_count = CounterForVoters::::get(); + assert_eq!(voters_count, nominator_count + validator_count); } fn check_ledgers() { @@ -839,3 +834,14 @@ pub(crate) fn staking_events() -> Vec> { pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { (Balances::free_balance(who), Balances::reserved_balance(who)) } + +use crate::voter_bags::Bag; +/// Returns the nodes of all non-empty bags. +pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { + VoterBagThresholds::get().into_iter().filter_map(|t| { + Bag::::get(*t).map(|bag| ( + *t, + bag.iter().map(|n| n.voter().id).collect::>() + )) + }).collect::>() +} diff --git a/frame/staking/src/mock/voter_bags.rs b/frame/staking/src/mock/voter_bags.rs deleted file mode 100644 index 6177c615bc59a..0000000000000 --- a/frame/staking/src/mock/voter_bags.rs +++ /dev/null @@ -1,101 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Autogenerated voter bag thresholds. -//! -//! Generated on 2021-07-05T12:08:52.871368217+00:00 -//! for the test runtime. - -/// Existential weight for this runtime. -#[cfg(any(test, feature = "std"))] -#[allow(unused)] -pub const EXISTENTIAL_WEIGHT: u64 = 1; - -/// Constant ratio between bags for this runtime. -#[cfg(any(test, feature = "std"))] -#[allow(unused)] -pub const CONSTANT_RATIO: f64 = 2.0000000000000000; - -/// Upper thresholds delimiting the bag list. -pub const THRESHOLDS: [u64; 65] = [5, 15, 25, 50, 100, 1000, 2000, 3000] -pub const THRESHOLDS: [u64; 65] = [ - 1, - 2, - 4, - 8, - 16, - 32, - 64, - 128, - 256, - 512, - 1_024, - 2_048, - 4_096, - 8_192, - 16_384, - 32_768, - 65_536, - 131_072, - 262_144, - 524_288, - 1_048_576, - 2_097_152, - 4_194_304, - 8_388_608, - 16_777_216, - 33_554_432, - 67_108_864, - 134_217_728, - 268_435_456, - 536_870_912, - 1_073_741_824, - 2_147_483_648, - 4_294_967_296, - 8_589_934_592, - 17_179_869_184, - 34_359_738_368, - 68_719_476_736, - 137_438_953_472, - 274_877_906_944, - 549_755_813_888, - 1_099_511_627_776, - 2_199_023_255_552, - 4_398_046_511_104, - 8_796_093_022_208, - 17_592_186_044_416, - 35_184_372_088_832, - 70_368_744_177_664, - 140_737_488_355_328, - 281_474_976_710_656, - 562_949_953_421_312, - 1_125_899_906_842_624, - 2_251_799_813_685_248, - 4_503_599_627_370_496, - 9_007_199_254_740_992, - 18_014_398_509_481_984, - 36_028_797_018_963_968, - 72_057_594_037_927_936, - 144_115_188_075_855_872, - 288_230_376_151_711_744, - 576_460_752_303_423_488, - 1_152_921_504_606_846_976, - 2_305_843_009_213_693_952, - 4_611_686_018_427_387_904, - 9_223_372_036_854_775_808, - 18_446_744_073_709_551_615, -]; diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 864fcb0c35683..15a82e4858b30 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -148,16 +148,13 @@ fn basic_setup_works() { // New era is not being forced assert_eq!(Staking::force_era(), Forcing::NotForcing); - // genesis accounts must exist in the proper bags - let weight_of = Staking::weight_of_fn(); - // for these stash ids, see - // https://github.com/paritytech/substrate/ - // blob/631d4cdbcad438248c2597213918d8207d85bf6e/frame/staking/src/mock.rs#L435-L441 - for genesis_stash_account_id in [11, 21, 31, 101] { - let node = crate::voter_bags::Node::::from_id(&genesis_stash_account_id) - .expect(&format!("node was created for account {}", genesis_stash_account_id)); - assert!(!node.is_misplaced(&weight_of)); - } + // check the bags + assert_eq!(CounterForVoters::::get(), 4); + + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101])], + ); }); } @@ -3871,6 +3868,18 @@ fn on_finalize_weight_is_nonzero() { }) } +// end-to-end nodes of the voter bags operation. +mod voter_bags { + + #[test] + fn rebag_works() { + todo!() + } +} +/* +// TODO: this needs some love, retire it in favour of the one above. Use the mock data, don't make +// it complicated with data setup, use the simplest data possible, instead check multiple +// edge-cases. #[test] fn test_rebag() { use crate::{ @@ -3901,8 +3910,8 @@ fn test_rebag() { ExtBuilder::default().build_and_execute(|| { // We want to have two validators: one, `stash`, is the one we will rebag. // The other, `other_stash`, exists only so that the destination bag is not empty. - let stash = make_validator(0, 100).unwrap(); - let other_stash = make_validator(1, 300).unwrap(); + let stash = make_validator(0, 2000).unwrap(); + let other_stash = make_validator(1, 9000).unwrap(); // verify preconditions let weight_of = Staking::weight_of_fn(); @@ -3942,6 +3951,7 @@ fn test_rebag() { assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); }); } +*/ mod election_data_provider { use super::*; diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 34efce5c59f02..0141f71ef156c 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -21,12 +21,8 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -use crate::{ - AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, - VotingDataOf, slashing::SlashingSpans, -}; -use codec::{Encode, Decode}; -use frame_support::{DefaultNoBound, traits::Get}; +use codec::{Decode, Encode}; +use frame_support::{ensure, traits::Get, DefaultNoBound}; use sp_runtime::SaturatedConversion; use sp_std::{ boxed::Box, @@ -35,6 +31,11 @@ use sp_std::{ marker::PhantomData, }; +use crate::{ + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, + VoteWeight, VoterBagFor, VotingDataOf, +}; + /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; @@ -77,7 +78,7 @@ pub struct VoterList(PhantomData); impl VoterList { /// Remove all data associated with the voter list from storage. pub fn clear() { - crate::VoterCount::::kill(); + crate::CounterForVoters::::kill(); crate::VoterBagFor::::remove_all(None); crate::VoterBags::::remove_all(None); crate::VoterNodes::::remove_all(None); @@ -104,13 +105,14 @@ impl VoterList { /// Decode the length of the voter list. pub fn decode_len() -> Option { - let maybe_len = crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()); + let maybe_len = crate::CounterForVoters::::try_get().ok().map(|n| n.saturated_into()); debug_assert_eq!( maybe_len.unwrap_or_default(), crate::VoterNodes::::iter().count(), "stored length must match count of nodes", ); debug_assert_eq!( + // TODO: this case will fail in migration pre check maybe_len.unwrap_or_default() as u32, crate::CounterForNominators::::get() + crate::CounterForValidators::::get(), "voter count must be sum of validator and nominator count", @@ -175,6 +177,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); let bag = notional_bag_for::(weight); + crate::log!(debug, "inserting {:?} into bag {:?}", voter, bag); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -183,13 +186,15 @@ impl VoterList { bag.put(); } - crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_add(count) + }); count } /// Remove a voter (by id) from the voter list. pub fn remove(voter: &AccountIdOf) { - Self::remove_many(sp_std::iter::once(voter)) + Self::remove_many(sp_std::iter::once(voter)); } /// Remove many voters (by id) from the voter list. @@ -219,7 +224,9 @@ impl VoterList { bag.put(); } - crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_sub(count)); + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_sub(count) + }); } /// Update a voter's position in the voter list. @@ -227,7 +234,7 @@ impl VoterList { /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they /// are moved into the correct bag. /// - /// Returns `true` if the voter moved. + /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. /// /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient @@ -350,6 +357,50 @@ impl VoterList { num_affected } + + /// Sanity check the voter list. + /// + /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) + /// is being used, after all other staking data (such as counter) has been updated. It checks + /// that: + /// + /// * Iterate all voters in list and make sure there are no duplicates. + /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. + /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. + /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are + /// checked per *any* update to `VoterList`. + pub(super) fn sanity_check() -> Result<(), String> { + let mut seen_in_list = BTreeSet::new(); + ensure!( + Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), + String::from("duplicate identified") + ); + + let iter_count = Self::iter().collect::>().len() as u32; + let stored_count = crate::CounterForVoters::::get(); + ensure!( + iter_count == stored_count, + format!("iter_count ({}) != voter_count ({})", iter_count, stored_count), + ); + + let validators = crate::CounterForValidators::::get(); + let nominators = crate::CounterForNominators::::get(); + ensure!( + validators + nominators == stored_count, + format!( + "validators {} + nominators {} != voters {}", + validators, nominators, stored_count + ), + ); + + let _ = T::VoterBagThresholds::get() + .into_iter() + .map(|t| Bag::::get(*t).unwrap_or_default()) + .map(|b| b.sanity_check()) + .collect::>()?; + + Ok(()) + } } /// A Bag is a doubly-linked list of voters. @@ -384,6 +435,10 @@ impl Bag { /// Get a bag by its upper vote weight or make it, appropriately initialized. pub fn get_or_make(bag_upper: VoteWeight) -> Bag { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } @@ -438,9 +493,9 @@ impl Bag { node.put(); // update the previous tail - if let Some(mut tail) = self.tail() { - tail.next = Some(id.clone()); - tail.put(); + if let Some(mut old_tail) = self.tail() { + old_tail.next = Some(id.clone()); + old_tail.put(); } // update the internal bag links @@ -458,19 +513,19 @@ impl Bag { /// the first place. Generally, use [`VoterList::remove`] instead. /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()` and `node.put()` after use. + /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` + /// to update storage for the bag and `node`. fn remove_node(&mut self, node: &Node) { - // TODO: we could merge this function here. - // node.excise(); + // Update previous node. if let Some(mut prev) = node.prev() { - prev.next = self.next.clone(); + prev.next = node.next.clone(); prev.put(); } + // Update next node. if let Some(mut next) = node.next() { - next.prev = self.prev.clone(); + next.prev = node.prev.clone(); next.put(); } - // IDEA: debug_assert! prev.next.prev == self // clear the bag head/tail pointers as necessary if self.head.as_ref() == Some(&node.voter.id) { @@ -480,6 +535,45 @@ impl Bag { self.tail = node.prev.clone(); } } + + /// Sanity check this bag. + /// + /// Should be called by the call-site, after each mutating operation on a bag. The call site of + /// this struct is always `VoterList`. + /// + /// * Ensures head has no prev. + /// * Ensures tail has no next. + /// * Ensures there are no loops, traversal from head to tail is correct. + fn sanity_check(&self) -> Result<(), String> { + ensure!( + self.head() + .map(|head| head.prev().is_none()) + // if there is no head, then there must not be a tail, meaning that the bag is + // empty. + .unwrap_or_else(|| self.tail.is_none()), + String::from("head has a prev") + ); + + ensure!( + self.tail() + .map(|tail| tail.next().is_none()) + // if there is no tail, then there must not be a head, meaning that the bag is + // empty. + .unwrap_or_else(|| self.head.is_none()), + String::from("tail has a next") + ); + + let mut seen_in_bag = BTreeSet::new(); + ensure!( + self.iter() + .map(|node| node.voter.id) + // each voter is only seen once, thus there is no cycle within a bag + .all(|voter| seen_in_bag.insert(voter)), + String::from("Duplicate found in bag.") + ); + + Ok(()) + } } /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. @@ -498,7 +592,10 @@ pub struct Node { impl Node { /// Get a node by bag idx and account id. pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { - // debug_assert!(bag_upper is in Threshold) + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { node.bag_upper = bag_upper; node @@ -563,20 +660,6 @@ impl Node { } } - /// Remove this node from the linked list. - /// - /// Modifies storage, but only modifies the adjacent nodes. Does not modify `self` or any bag. - fn excise(&self) { - if let Some(mut prev) = self.prev() { - prev.next = self.next.clone(); - prev.put(); - } - if let Some(mut next) = self.next() { - next.prev = self.prev.clone(); - next.put(); - } - } - /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper @@ -606,6 +689,12 @@ impl Node { let current_weight = weight_of(&self.voter.id); notional_bag_for::(current_weight) } + + #[cfg(any(test, feature = "runtime-benchmarks"))] + /// Get the underlying voter. + pub fn voter(&self) -> &Voter { + &self.voter + } } /// Fundamental information about a voter. @@ -706,7 +795,7 @@ pub mod make_bags { fn path_to_header_file() -> Option { let repo = git2::Repository::open_from_env().ok()?; let workdir = repo.workdir()?; - for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { + for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { let path = workdir.join(file_name); if path.exists() { return Some(path); @@ -853,23 +942,181 @@ pub mod make_bags { } } +// This is the highest level of abstraction provided by this module. More generic tests are here, +// among those related to `VoterList` struct. #[cfg(test)] -mod tests { - use crate::mock::{ExtBuilder, Staking, Test}; +mod voter_list { use frame_support::traits::Currency; - use substrate_test_utils::assert_eq_uvec; use super::*; + use crate::mock::*; - const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; + #[test] + fn basic_setup_works() { + // make sure ALL relevant data structures are setup correctly. + // TODO: we are not checking all of them yet. + ExtBuilder::default().build_and_execute(|| { + assert_eq!(crate::CounterForVoters::::get(), 4); + + let weight_of = Staking::weight_of_fn(); + assert_eq!(weight_of(&11), 1000); + assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); + + assert_eq!(weight_of(&21), 1000); + assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); + + assert_eq!(weight_of(&31), 1); + assert_eq!(VoterBagFor::::get(31).unwrap(), 10); + + assert_eq!(VoterBagFor::::get(41), None); // this staker is chilled! + + assert_eq!(weight_of(&101), 500); + assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); + + // iteration of the bags would yield: + assert_eq!( + VoterList::::iter().map(|n| n.voter().id).collect::>(), + vec![11, 21, 101, 31], + // ^^ note the order of insertion in genesis! + ); + + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101])], + ); + }) + } #[test] - fn voter_list_includes_genesis_accounts() { - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); - assert_eq_uvec!(GENESIS_VOTER_IDS, have_voters); - }); + fn notional_bag_for_works() { + todo!(); + } + + #[test] + fn iteration_is_semi_sorted() { + ExtBuilder::default().build_and_execute(|| { + // add some new validators to the genesis state. + bond_validator(51, 50, 2000); + bond_validator(61, 60, 2000); + + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // when + let iteration = VoterList::::iter().map(|node| node.voter.id).collect::>(); + + // then + assert_eq!(iteration, vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, // last bag. + ]); + }) + } + + /// This tests that we can `take` x voters, even if that quantity ends midway through a list. + #[test] + fn take_works() { + ExtBuilder::default().build_and_execute(|| { + // add some new validators to the genesis state. + bond_validator(51, 50, 2000); + bond_validator(61, 60, 2000); + + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // when + let iteration = VoterList::::iter() + .map(|node| node.voter.id) + .take(4) + .collect::>(); + + // then + assert_eq!(iteration, vec![ + 51, 61, // best bag, fully iterated + 11, 21, // middle bag, partially iterated + ]); + }) + } + + #[test] + fn storage_is_cleaned_up_as_voters_are_removed() {} + + #[test] + fn insert_works() { + todo!() + } + + #[test] + fn insert_as_works() { + // insert a new one with role + // update the status of already existing one. + todo!() } + #[test] + fn remove_works() { + todo!() + } + + #[test] + fn update_position_for_works() { + // alter the genesis state to require a re-bag, then ensure this fixes it. Might be similar + // `rebag_works()` + todo!(); + } +} + +#[cfg(test)] +mod bags { + #[test] + fn get_works() { + todo!() + } + + #[test] + fn insert_works() { + todo!() + } + + #[test] + fn remove_works() { + todo!() + } +} + +#[cfg(test)] +mod voter_node { + #[test] + fn get_voting_data_works() { + todo!() + } + + #[test] + fn is_misplaced_works() { + todo!() + } +} + +// TODO: I've created simpler versions of these tests above. We can probably remove the ones below +// now. Peter was likely not very familiar with the staking mock and he came up with these rather +// complicated test setups. Please see my versions above, we can test the same properties, easily, +// without the need to alter the stakers so much. +/* +#[cfg(test)] +mod tests { + use frame_support::traits::Currency; + + use super::*; + use crate::mock::*; + + const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; + /// This tests the property that when iterating through the `VoterList`, we iterate from higher /// bags to lower. #[test] @@ -888,8 +1135,7 @@ mod tests { ExtBuilder::default().validator_pool(true).build_and_execute(|| { // initialize the voters' deposits - let existential_deposit = ::Currency::minimum_balance(); - let mut balance = existential_deposit + 1; + let mut balance = 10; for voter_id in voters.iter().rev() { ::Currency::make_free_balance_be(voter_id, balance); let controller = Staking::bonded(voter_id).unwrap(); @@ -899,11 +1145,138 @@ mod tests { Staking::update_ledger(&controller, &ledger); Staking::do_rebag(voter_id); - balance *= 2; + // Increase balance to the next threshold. + balance += 10; } let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); assert_eq!(voters, have_voters); }); } + + /// This tests that we can `take` x voters, even if that quantity ends midway through a list. + #[test] + fn take_works() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // initialize the voters' deposits + let mut balance = 0; // This will be 10 on the first loop iteration because 0 % 3 == 0 + for (idx, voter_id) in GENESIS_VOTER_IDS.iter().enumerate() { + if idx % 3 == 0 { + // This increases the balance by 10, which is the amount each threshold + // increases by. Thus this will increase the balance by 1 bag. + // + // This will create 2 bags, the lower threshold bag having + // 3 voters with balance 10, and the higher threshold bag having + // 2 voters with balance 20. + balance += 10; + } + + ::Currency::make_free_balance_be(voter_id, balance); + let controller = Staking::bonded(voter_id).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = balance; + ledger.active = balance; + Staking::update_ledger(&controller, &ledger); + Staking::do_rebag(voter_id); + } + + let bag_thresh10 = Bag::::get(10) + .unwrap() + .iter() + .map(|node| node.voter.id) + .collect::>(); + assert_eq!(bag_thresh10, vec![11, 21, 31]); + + let bag_thresh20 = Bag::::get(20) + .unwrap() + .iter() + .map(|node| node.voter.id) + .collect::>(); + assert_eq!(bag_thresh20, vec![41, 101]); + + let voters: Vec<_> = VoterList::::iter() + // take 4/5 from [41, 101],[11, 21, 31], demonstrating that we can do a + // take that stops mid bag. + .take(4) + .map(|node| node.voter.id) + .collect(); + + assert_eq!(voters, vec![41, 101, 11, 21]); + }); + } + + #[test] + fn storage_is_cleaned_up_as_voters_are_removed() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // Initialize voters deposits so there are 5 bags with one voter each. + let mut balance = 10; + for voter_id in GENESIS_VOTER_IDS.iter() { + ::Currency::make_free_balance_be(voter_id, balance); + let controller = Staking::bonded(voter_id).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = balance; + ledger.active = balance; + Staking::update_ledger(&controller, &ledger); + Staking::do_rebag(voter_id); + + // Increase balance to the next threshold. + balance += 10; + } + + let voter_list_storage_items_eq = |mut v: Vec| { + v.sort(); + let mut voters: Vec<_> = + VoterList::::iter().map(|node| node.voter.id).collect(); + voters.sort(); + assert_eq!(voters, v); + + let mut nodes: Vec<_> = + ::VoterNodes::iter_keys().collect(); + nodes.sort(); + assert_eq!(nodes, v); + + let mut flat_bags: Vec<_> = ::VoterBags::iter() + // We always get the bag with the Bag getter because the bag_upper + // is only initialized in the getter. + .flat_map(|(key, _bag)| Bag::::get(key).unwrap().iter()) + .map(|node| node.voter.id) + .collect(); + flat_bags.sort(); + assert_eq!(flat_bags, v); + + let mut bags_for: Vec<_> = + ::VoterBagFor::iter_keys().collect(); + bags_for.sort(); + assert_eq!(bags_for, v); + }; + + let genesis_voters = vec![101, 41, 31, 21, 11]; + voter_list_storage_items_eq(genesis_voters); + assert_eq!(::CounterForVoters::get(), 5); + + // Remove 1 voter, + VoterList::::remove(&101); + let remaining_voters = vec![41, 31, 21, 11]; + // and assert they have been cleaned up. + voter_list_storage_items_eq(remaining_voters.clone()); + assert_eq!(::CounterForVoters::get(), 4); + + // Now remove the remaining voters so we have 0 left, + remaining_voters.iter().for_each(|v| VoterList::::remove(v)); + // and assert all of them have been cleaned up. + voter_list_storage_items_eq(vec![]); + assert_eq!(::CounterForVoters::get(), 0); + + + // TODO bags do not get cleaned up from storages + // - is this ok? I assume its ok if this is not cleaned just because voters are removed + // but it should be cleaned up if we migrate thresholds + assert_eq!(::VoterBags::iter().collect::>().len(), 6); + // and the voter list has no one in it. + assert_eq!(VoterList::::iter().collect::>().len(), 0); + assert_eq!(::VoterBagFor::iter().collect::>().len(), 0); + assert_eq!(::VoterNodes::iter().collect::>().len(), 0); + }); + } } +*/ From 48ccfc9df1e95675f0cbc7719cf46533befe5c6d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 21 Jul 2021 17:21:43 -0700 Subject: [PATCH 072/241] merge fallout --- bin/node/runtime/voter-bags/Cargo.toml | 6 +++--- frame/staking/Cargo.toml | 11 ++++++----- frame/staking/src/voter_bags.rs | 9 ++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/node/runtime/voter-bags/Cargo.toml b/bin/node/runtime/voter-bags/Cargo.toml index 38f58ba9d4bd0..55b9dca859db8 100644 --- a/bin/node/runtime/voter-bags/Cargo.toml +++ b/bin/node/runtime/voter-bags/Cargo.toml @@ -10,7 +10,7 @@ description = "Voter Bag generation script for pallet-staking and node-runtime." readme = "README.md" [dependencies] -node-runtime = { version = "2.0.0", path = ".." } -pallet-staking = { version = "3.0.0", path = "../../../../frame/staking", features = ["make-bags"] } -sp-io = { version = "3.0.0", path = "../../../../primitives/io" } +node-runtime = { version = "3.0.0-dev", path = ".." } +pallet-staking = { version = "4.0.0-dev", path = "../../../../frame/staking", features = ["make-bags"] } +sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } structopt = "0.3.21" diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 22c5fa9ef6ecd..bfe2e04475452 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -26,11 +26,11 @@ pallet-session = { version = "4.0.0-dev", default-features = false, features = [ pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../authorship" } sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } -sp-core = { version = "3.0.0", path = "../../primitives/core", optional = true } -pallet-timestamp = { version = "3.0.0", path = "../timestamp", optional = true } -pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve", optional = true } -pallet-balances = { version = "3.0.0", path = "../balances", optional = true } -sp-tracing = { version = "3.0.0", path = "../../primitives/tracing", optional = true } +sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = true } +pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp", optional = true } +pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve", optional = true } +pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true } +sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true } log = { version = "0.4.14", default-features = false } paste = "1.0" @@ -47,6 +47,7 @@ num-format = { version = "0.4.0", optional = true } sp-storage = { version = "4.0.0-dev", path = "../../primitives/storage" } sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections", features = ["mocks"] } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 0141f71ef156c..edc72e7be6865 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -369,11 +369,11 @@ impl VoterList { /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `VoterList`. - pub(super) fn sanity_check() -> Result<(), String> { + pub(super) fn sanity_check() -> Result<(), std::string::String> { let mut seen_in_list = BTreeSet::new(); ensure!( Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), - String::from("duplicate identified") + String::from("duplicate identified"), ); let iter_count = Self::iter().collect::>().len() as u32; @@ -544,7 +544,7 @@ impl Bag { /// * Ensures head has no prev. /// * Ensures tail has no next. /// * Ensures there are no loops, traversal from head to tail is correct. - fn sanity_check(&self) -> Result<(), String> { + fn sanity_check(&self) -> Result<(), std::string::String> { ensure!( self.head() .map(|head| head.prev().is_none()) @@ -698,8 +698,7 @@ impl Node { } /// Fundamental information about a voter. -#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr(feature = "std", derive(Debug))] +#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord, sp_runtime::RuntimeDebug)] pub struct Voter { /// Account Id of this voter pub id: AccountId, From 1c56016e3e016d103c52d5abd2003e594cf665c0 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 21 Jul 2021 17:54:27 -0700 Subject: [PATCH 073/241] Run cargo +nightly fmt --- bin/node/runtime/src/voter_bags.rs | 378 +++++++++--------- bin/node/runtime/voter-bags/src/main.rs | 12 +- .../election-provider-support/src/onchain.rs | 6 +- frame/staking/src/lib.rs | 47 +-- frame/staking/src/mock.rs | 13 +- frame/staking/src/tests.rs | 7 +- frame/staking/src/voter_bags.rs | 78 ++-- primitives/npos-elections/src/lib.rs | 5 +- 8 files changed, 265 insertions(+), 281 deletions(-) diff --git a/bin/node/runtime/src/voter_bags.rs b/bin/node/runtime/src/voter_bags.rs index f8f6201e94dc3..c4c731a58badc 100644 --- a/bin/node/runtime/src/voter_bags.rs +++ b/bin/node/runtime/src/voter_bags.rs @@ -32,195 +32,195 @@ pub const CONSTANT_RATIO: f64 = 1.0628253590743408; /// Upper thresholds delimiting the bag list. pub const THRESHOLDS: [u64; 200] = [ - 100_000_000_000_000, - 106_282_535_907_434, - 112_959_774_389_150, - 120_056_512_776_105, - 127_599_106_300_477, - 135_615_565_971_369, - 144_135_662_599_590, - 153_191_037_357_827, - 162_815_319_286_803, - 173_044_250_183_800, - 183_915_817_337_347, - 195_470_394_601_017, - 207_750_892_330_229, - 220_802_916_738_890, - 234_674_939_267_673, - 249_418_476_592_914, - 265_088_281_944_639, - 281_742_548_444_211, - 299_443_125_216_738, - 318_255_747_080_822, - 338_250_278_668_647, - 359_500_973_883_001, - 382_086_751_654_776, - 406_091_489_025_036, - 431_604_332_640_068, - 458_720_029_816_222, - 487_539_280_404_019, - 518_169_110_758_247, - 550_723_271_202_866, - 585_322_658_466_782, - 622_095_764_659_305, - 661_179_154_452_653, - 702_717_972_243_610, - 746_866_481_177_808, - 793_788_636_038_393, - 843_658_692_126_636, - 896_661_852_395_681, - 952_994_955_240_703, - 1_012_867_205_499_736, - 1_076_500_951_379_881, - 1_144_132_510_194_192, - 1_216_013_045_975_769, - 1_292_409_502_228_280, - 1_373_605_593_276_862, - 1_459_902_857_901_004, - 1_551_621_779_162_291, - 1_649_102_974_585_730, - 1_752_708_461_114_642, - 1_862_822_999_536_805, - 1_979_855_523_374_646, - 2_104_240_657_545_975, - 2_236_440_332_435_128, - 2_376_945_499_368_703, - 2_526_277_953_866_680, - 2_684_992_273_439_945, - 2_853_677_877_130_641, - 3_032_961_214_443_876, - 3_223_508_091_799_862, - 3_426_026_145_146_232, - 3_641_267_467_913_124, - 3_870_031_404_070_482, - 4_113_167_516_660_186, - 4_371_578_742_827_277, - 4_646_224_747_067_156, - 4_938_125_485_141_739, - 5_248_364_991_899_922, - 5_578_095_407_069_235, - 5_928_541_253_969_291, - 6_301_003_987_036_955, - 6_696_866_825_051_405, - 7_117_599_888_008_300, - 7_564_765_656_719_910, - 8_040_024_775_416_580, - 8_545_142_218_898_723, - 9_081_993_847_142_344, - 9_652_573_371_700_016, - 10_258_999_759_768_490, - 10_903_525_103_419_522, - 11_588_542_983_217_942, - 12_316_597_357_287_042, - 13_090_392_008_832_678, - 13_912_800_587_211_472, - 14_786_877_279_832_732, - 15_715_868_154_526_436, - 16_703_223_214_499_558, - 17_752_609_210_649_358, - 18_867_923_258_814_856, - 20_053_307_312_537_008, - 21_313_163_545_075_252, - 22_652_170_697_804_756, - 24_075_301_455_707_600, - 25_587_840_914_485_432, - 27_195_406_207_875_088, - 28_903_967_368_057_400, - 30_719_869_496_628_636, - 32_649_856_328_471_220, - 34_701_095_276_033_064, - 36_881_204_047_022_752, - 39_198_278_934_370_992, - 41_660_924_883_519_016, - 44_278_287_448_695_240, - 47_060_086_756_856_400, - 50_016_653_605_425_536, - 53_158_967_827_883_320, - 56_498_699_069_691_424, - 60_048_250_125_977_912, - 63_820_803_001_928_304, - 67_830_367_866_937_216, - 72_091_835_084_322_176, - 76_621_030_509_822_880, - 81_434_774_264_248_528, - 86_550_943_198_537_824, - 91_988_537_283_208_848, - 97_767_750_168_749_840, - 103_910_044_178_992_000, - 110_438_230_015_967_792, - 117_376_551_472_255_616, - 124_750_775_465_407_920, - 132_588_287_728_824_640, - 140_918_194_514_440_064, - 149_771_430_684_917_568, - 159_180_874_596_775_264, - 169_181_470_201_085_280, - 179_810_356_815_193_344, - 191_107_007_047_393_216, - 203_113_373_386_768_288, - 215_874_044_002_592_672, - 229_436_408_331_885_600, - 243_850_833_070_063_392, - 259_170_849_218_267_264, - 275_453_350_882_006_752, - 292_758_806_559_399_232, - 311_151_483_703_668_992, - 330_699_687_393_865_920, - 351_476_014_000_157_824, - 373_557_620_785_735_808, - 397_026_512_446_556_096, - 421_969_845_653_044_224, - 448_480_252_724_740_928, - 476_656_185_639_923_904, - 506_602_281_657_757_760, - 538_429_751_910_786_752, - 572_256_794_410_890_176, - 608_209_033_002_485_632, - 646_419_983_893_124_352, - 687_031_551_494_039_552, - 730_194_555_412_054_016, - 776_069_290_549_944_960, - 824_826_122_395_314_176, - 876_646_119_708_695_936, - 931_721_726_960_522_368, - 990_257_479_014_182_144, - 1_052_470_760_709_299_712, - 1_118_592_614_166_106_112, - 1_188_868_596_808_997_376, - 1_263_559_693_295_730_432, - 1_342_943_284_738_898_688, - 1_427_314_178_819_094_784, - 1_516_985_704_615_302_400, - 1_612_290_876_218_400_768, - 1_713_583_629_449_105_408, - 1_821_240_136_273_157_632, - 1_935_660_201_795_120_128, - 2_057_268_749_018_809_600, - 2_186_517_396_888_336_384, - 2_323_886_137_470_138_880, - 2_469_885_118_504_583_168, - 2_625_056_537_947_004_416, - 2_789_976_657_533_970_944, - 2_965_257_942_852_572_160, - 3_151_551_337_860_326_400, - 3_349_548_682_302_620_672, - 3_559_985_281_005_267_968, - 3_783_642_634_583_792_128, - 4_021_351_341_710_503_936, - 4_273_994_183_717_548_544, - 4_542_509_402_991_247_872, - 4_827_894_187_332_742_144, - 5_131_208_373_224_844_288, - 5_453_578_381_757_959_168, - 5_796_201_401_831_965_696, - 6_160_349_836_169_256_960, - 6_547_376_026_650_146_816, - 6_958_717_276_519_173_120, - 7_395_901_188_113_309_696, - 7_860_551_335_934_872_576, - 8_354_393_296_137_270_272, - 8_879_261_054_815_360_000, - 9_437_103_818_898_946_048, + 100_000_000_000_000, + 106_282_535_907_434, + 112_959_774_389_150, + 120_056_512_776_105, + 127_599_106_300_477, + 135_615_565_971_369, + 144_135_662_599_590, + 153_191_037_357_827, + 162_815_319_286_803, + 173_044_250_183_800, + 183_915_817_337_347, + 195_470_394_601_017, + 207_750_892_330_229, + 220_802_916_738_890, + 234_674_939_267_673, + 249_418_476_592_914, + 265_088_281_944_639, + 281_742_548_444_211, + 299_443_125_216_738, + 318_255_747_080_822, + 338_250_278_668_647, + 359_500_973_883_001, + 382_086_751_654_776, + 406_091_489_025_036, + 431_604_332_640_068, + 458_720_029_816_222, + 487_539_280_404_019, + 518_169_110_758_247, + 550_723_271_202_866, + 585_322_658_466_782, + 622_095_764_659_305, + 661_179_154_452_653, + 702_717_972_243_610, + 746_866_481_177_808, + 793_788_636_038_393, + 843_658_692_126_636, + 896_661_852_395_681, + 952_994_955_240_703, + 1_012_867_205_499_736, + 1_076_500_951_379_881, + 1_144_132_510_194_192, + 1_216_013_045_975_769, + 1_292_409_502_228_280, + 1_373_605_593_276_862, + 1_459_902_857_901_004, + 1_551_621_779_162_291, + 1_649_102_974_585_730, + 1_752_708_461_114_642, + 1_862_822_999_536_805, + 1_979_855_523_374_646, + 2_104_240_657_545_975, + 2_236_440_332_435_128, + 2_376_945_499_368_703, + 2_526_277_953_866_680, + 2_684_992_273_439_945, + 2_853_677_877_130_641, + 3_032_961_214_443_876, + 3_223_508_091_799_862, + 3_426_026_145_146_232, + 3_641_267_467_913_124, + 3_870_031_404_070_482, + 4_113_167_516_660_186, + 4_371_578_742_827_277, + 4_646_224_747_067_156, + 4_938_125_485_141_739, + 5_248_364_991_899_922, + 5_578_095_407_069_235, + 5_928_541_253_969_291, + 6_301_003_987_036_955, + 6_696_866_825_051_405, + 7_117_599_888_008_300, + 7_564_765_656_719_910, + 8_040_024_775_416_580, + 8_545_142_218_898_723, + 9_081_993_847_142_344, + 9_652_573_371_700_016, + 10_258_999_759_768_490, + 10_903_525_103_419_522, + 11_588_542_983_217_942, + 12_316_597_357_287_042, + 13_090_392_008_832_678, + 13_912_800_587_211_472, + 14_786_877_279_832_732, + 15_715_868_154_526_436, + 16_703_223_214_499_558, + 17_752_609_210_649_358, + 18_867_923_258_814_856, + 20_053_307_312_537_008, + 21_313_163_545_075_252, + 22_652_170_697_804_756, + 24_075_301_455_707_600, + 25_587_840_914_485_432, + 27_195_406_207_875_088, + 28_903_967_368_057_400, + 30_719_869_496_628_636, + 32_649_856_328_471_220, + 34_701_095_276_033_064, + 36_881_204_047_022_752, + 39_198_278_934_370_992, + 41_660_924_883_519_016, + 44_278_287_448_695_240, + 47_060_086_756_856_400, + 50_016_653_605_425_536, + 53_158_967_827_883_320, + 56_498_699_069_691_424, + 60_048_250_125_977_912, + 63_820_803_001_928_304, + 67_830_367_866_937_216, + 72_091_835_084_322_176, + 76_621_030_509_822_880, + 81_434_774_264_248_528, + 86_550_943_198_537_824, + 91_988_537_283_208_848, + 97_767_750_168_749_840, + 103_910_044_178_992_000, + 110_438_230_015_967_792, + 117_376_551_472_255_616, + 124_750_775_465_407_920, + 132_588_287_728_824_640, + 140_918_194_514_440_064, + 149_771_430_684_917_568, + 159_180_874_596_775_264, + 169_181_470_201_085_280, + 179_810_356_815_193_344, + 191_107_007_047_393_216, + 203_113_373_386_768_288, + 215_874_044_002_592_672, + 229_436_408_331_885_600, + 243_850_833_070_063_392, + 259_170_849_218_267_264, + 275_453_350_882_006_752, + 292_758_806_559_399_232, + 311_151_483_703_668_992, + 330_699_687_393_865_920, + 351_476_014_000_157_824, + 373_557_620_785_735_808, + 397_026_512_446_556_096, + 421_969_845_653_044_224, + 448_480_252_724_740_928, + 476_656_185_639_923_904, + 506_602_281_657_757_760, + 538_429_751_910_786_752, + 572_256_794_410_890_176, + 608_209_033_002_485_632, + 646_419_983_893_124_352, + 687_031_551_494_039_552, + 730_194_555_412_054_016, + 776_069_290_549_944_960, + 824_826_122_395_314_176, + 876_646_119_708_695_936, + 931_721_726_960_522_368, + 990_257_479_014_182_144, + 1_052_470_760_709_299_712, + 1_118_592_614_166_106_112, + 1_188_868_596_808_997_376, + 1_263_559_693_295_730_432, + 1_342_943_284_738_898_688, + 1_427_314_178_819_094_784, + 1_516_985_704_615_302_400, + 1_612_290_876_218_400_768, + 1_713_583_629_449_105_408, + 1_821_240_136_273_157_632, + 1_935_660_201_795_120_128, + 2_057_268_749_018_809_600, + 2_186_517_396_888_336_384, + 2_323_886_137_470_138_880, + 2_469_885_118_504_583_168, + 2_625_056_537_947_004_416, + 2_789_976_657_533_970_944, + 2_965_257_942_852_572_160, + 3_151_551_337_860_326_400, + 3_349_548_682_302_620_672, + 3_559_985_281_005_267_968, + 3_783_642_634_583_792_128, + 4_021_351_341_710_503_936, + 4_273_994_183_717_548_544, + 4_542_509_402_991_247_872, + 4_827_894_187_332_742_144, + 5_131_208_373_224_844_288, + 5_453_578_381_757_959_168, + 5_796_201_401_831_965_696, + 6_160_349_836_169_256_960, + 6_547_376_026_650_146_816, + 6_958_717_276_519_173_120, + 7_395_901_188_113_309_696, + 7_860_551_335_934_872_576, + 8_354_393_296_137_270_272, + 8_879_261_054_815_360_000, + 9_437_103_818_898_946_048, 10_029_993_254_943_105_024, 10_660_131_182_698_121_216, 11_329_857_752_030_707_712, diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 9ae50914d6927..2a2be8297eb22 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,12 +17,11 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::voter_bags::make_bags::generate_thresholds_module; -use pallet_staking::mock::Test; +use pallet_staking::{mock::Test, voter_bags::make_bags::generate_thresholds_module}; use std::path::{Path, PathBuf}; -use structopt::{StructOpt, clap::arg_enum}; +use structopt::{clap::arg_enum, StructOpt}; -arg_enum!{ +arg_enum! { #[derive(Debug)] enum Runtime { Node, @@ -42,10 +41,7 @@ impl Runtime { #[derive(Debug, StructOpt)] struct Opt { /// How many bags to generate. - #[structopt( - long, - default_value = "200", - )] + #[structopt(long, default_value = "200")] n_bags: usize, /// Which runtime to generate. diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index c3ca7580f66e7..6709fe33d9b95 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -77,8 +77,10 @@ impl ElectionProvider for OnChainSequen let (desired_targets, _) = Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; - let stake_map: BTreeMap = -voters.iter().map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)).collect(); + let stake_map: BTreeMap = voters + .iter() + .map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)) + .collect(); let stake_of = |w: &T::AccountId| -> VoteWeight { stake_map.get(w).cloned().unwrap_or_default() }; diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index a617cabfe78cc..dbcf66292308f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -276,10 +276,10 @@ #[cfg(any(test, feature = "make-bags"))] pub mod mock; -#[cfg(test)] -mod tests; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod testing_utils; +#[cfg(test)] +mod tests; pub mod inflation; pub mod slashing; @@ -303,19 +303,19 @@ use frame_system::{ensure_root, ensure_signed, offchain::SendTransactionTypes, p pub use pallet::*; use pallet_session::historical; use sp_runtime::{ - DispatchError, Perbill, Percent, RuntimeDebug, curve::PiecewiseLinear, traits::{ AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, StaticLookup, Zero, }, + DispatchError, Perbill, Percent, RuntimeDebug, }; use sp_staking::{ offence::{Offence, OffenceDetails, OffenceError, OnOffenceHandler, ReportOffence}, SessionIndex, }; -use voter_bags::{VoterList, VoterType}; use sp_std::{collections::btree_map::BTreeMap, convert::From, prelude::*, result}; +use voter_bags::{VoterList, VoterType}; pub use weights::WeightInfo; const STAKING_ID: LockIdentifier = *b"staking "; @@ -763,7 +763,10 @@ pub mod migrations { pub fn pre_migrate() -> Result<(), &'static str> { ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); - ensure!(VoterList::::decode_len().unwrap_or_default() == 0, "voter list already exists"); + ensure!( + VoterList::::decode_len().unwrap_or_default() == 0, + "voter list already exists" + ); Ok(()) } @@ -783,7 +786,8 @@ pub mod migrations { T::WeightInfo::regenerate( CounterForValidators::::get(), CounterForNominators::::get(), - ).saturating_add(T::DbWeight::get().reads(2)) + ) + .saturating_add(T::DbWeight::get().reads(2)) } } @@ -1305,7 +1309,6 @@ pub mod pallet { #[pallet::storage] pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; - // The next storage items collectively comprise the voter bags: a composite data structure // designed to allow efficient iteration of the top N voters by stake, mostly. See // `mod voter_bags` for details. @@ -1321,16 +1324,14 @@ pub mod pallet { /// This may not be the appropriate bag for the voter's weight if they have been rewarded or /// slashed. #[pallet::storage] - pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; + pub(crate) type VoterBagFor = + StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] - pub(crate) type VoterBags = StorageMap< - _, - Twox64Concat, VoteWeight, - voter_bags::Bag, - >; + pub(crate) type VoterBags = + StorageMap<_, Twox64Concat, VoteWeight, voter_bags::Bag>; /// Voter nodes store links forward and back within their respective bags, the stash id, and /// whether the voter is a validator or nominator. @@ -1338,11 +1339,8 @@ pub mod pallet { /// There is nothing in this map directly identifying to which bag a particular node belongs. /// However, the `Node` data structure has helpers which can provide that information. #[pallet::storage] - pub(crate) type VoterNodes = StorageMap< - _, - Twox64Concat, AccountIdOf, - voter_bags::Node, - >; + pub(crate) type VoterNodes = + StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::Node>; // End of voter bags data. @@ -1421,7 +1419,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue; + continue } match status { StakerStatus::Validator => { @@ -1431,7 +1429,7 @@ pub mod pallet { ) { log!(warn, "failed to validate staker at genesis: {:?}.", why); } else { - num_voters +=1 ; + num_voters += 1; } }, StakerStatus::Nominator(votes) => { @@ -1443,8 +1441,8 @@ pub mod pallet { } else { num_voters += 1; } - } - _ => () + }, + _ => (), }; } @@ -2502,10 +2500,7 @@ pub mod pallet { /// /// Anyone can call this function about any stash. #[pallet::weight(T::WeightInfo::rebag())] - pub fn rebag( - origin: OriginFor, - stash: AccountIdOf, - ) -> DispatchResult { + pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { ensure_signed(origin)?; Pallet::::do_rebag(&stash); Ok(()) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 1f9a4c7a73832..57f87a036516a 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -825,10 +825,11 @@ pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { use crate::voter_bags::Bag; /// Returns the nodes of all non-empty bags. pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { - VoterBagThresholds::get().into_iter().filter_map(|t| { - Bag::::get(*t).map(|bag| ( - *t, - bag.iter().map(|n| n.voter().id).collect::>() - )) - }).collect::>() + VoterBagThresholds::get() + .into_iter() + .filter_map(|t| { + Bag::::get(*t) + .map(|bag| (*t, bag.iter().map(|n| n.voter().id).collect::>())) + }) + .collect::>() } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 780f39736621d..4724d2cd89919 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -25,6 +25,7 @@ use frame_support::{ weights::{extract_actual_weight, GetDispatchInfo}, }; use mock::*; +use pallet_balances::Error as BalancesError; use sp_npos_elections::supports_eq_unordered; use sp_runtime::{ assert_eq_error_rate, @@ -32,7 +33,6 @@ use sp_runtime::{ }; use sp_staking::offence::OffenceDetails; use substrate_test_utils::assert_eq_uvec; -use pallet_balances::Error as BalancesError; #[test] fn force_unstake_works() { @@ -174,10 +174,7 @@ fn basic_setup_works() { // check the bags assert_eq!(CounterForVoters::::get(), 4); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101])], - ); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); }); } diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index edc72e7be6865..4917e3446d884 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -97,10 +97,7 @@ impl VoterList { let validators_iter = Validators::::iter().map(|(id, _)| Voter::validator(id)); let weight_of = Pallet::::weight_of_fn(); - Self::insert_many( - nominators_iter.chain(validators_iter), - weight_of, - ) + Self::insert_many(nominators_iter.chain(validators_iter), weight_of) } /// Decode the length of the voter list. @@ -140,9 +137,7 @@ impl VoterList { // otherwise, insert it here. Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) }; - iter - .filter_map(Bag::get) - .flat_map(|bag| bag.iter()) + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } /// Insert a new voter into the appropriate bag in the voter list. @@ -212,7 +207,9 @@ impl VoterList { count += 1; // clear the bag head/tail pointers as necessary - let bag = bags.entry(node.bag_upper).or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); bag.remove_node(&node); // now get rid of the node itself @@ -470,12 +467,7 @@ impl Bag { /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. fn insert(&mut self, voter: VoterOf) { - self.insert_node(Node:: { - voter, - prev: None, - next: None, - bag_upper: self.bag_upper, - }); + self.insert_node(Node:: { voter, prev: None, next: None, bag_upper: self.bag_upper }); } /// Insert a voter node into this bag. @@ -656,7 +648,7 @@ impl Node { (!targets.is_empty()) .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) - } + }, } } @@ -708,17 +700,11 @@ pub struct Voter { impl Voter { pub fn nominator(id: AccountId) -> Self { - Self { - id, - voter_type: VoterType::Nominator, - } + Self { id, voter_type: VoterType::Nominator } } pub fn validator(id: AccountId) -> Self { - Self { - id, - voter_type: VoterType::Validator, - } + Self { id, voter_type: VoterType::Validator } } } @@ -782,10 +768,13 @@ pub fn existential_weight() -> VoteWeight { /// ``` #[cfg(feature = "make-bags")] pub mod make_bags { - use crate::{Config, voter_bags::existential_weight}; + use crate::{voter_bags::existential_weight, Config}; use frame_election_provider_support::VoteWeight; use frame_support::traits::Get; - use std::{io::Write, path::{Path, PathBuf}}; + use std::{ + io::Write, + path::{Path, PathBuf}, + }; /// Return the path to a header file used in this repository if is exists. /// @@ -797,7 +786,7 @@ pub mod make_bags { for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { let path = workdir.join(file_name); if path.exists() { - return Some(path); + return Some(path) } } None @@ -945,9 +934,9 @@ pub mod make_bags { // among those related to `VoterList` struct. #[cfg(test)] mod voter_list { - use frame_support::traits::Currency; use super::*; use crate::mock::*; + use frame_support::traits::Currency; #[test] fn basic_setup_works() { @@ -978,10 +967,7 @@ mod voter_list { // ^^ note the order of insertion in genesis! ); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101])], - ); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); }) } @@ -1007,11 +993,14 @@ mod voter_list { let iteration = VoterList::::iter().map(|node| node.voter.id).collect::>(); // then - assert_eq!(iteration, vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, // last bag. - ]); + assert_eq!( + iteration, + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, // last bag. + ] + ); }) } @@ -1030,16 +1019,17 @@ mod voter_list { ); // when - let iteration = VoterList::::iter() - .map(|node| node.voter.id) - .take(4) - .collect::>(); + let iteration = + VoterList::::iter().map(|node| node.voter.id).take(4).collect::>(); // then - assert_eq!(iteration, vec![ - 51, 61, // best bag, fully iterated - 11, 21, // middle bag, partially iterated - ]); + assert_eq!( + iteration, + vec![ + 51, 61, // best bag, fully iterated + 11, 21, // middle bag, partially iterated + ] + ); }) } diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index cdcb90d4df644..ad2b3229a0881 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -499,7 +499,10 @@ impl Support { pub type Supports = Vec<(A, Support)>; #[cfg(feature = "mocks")] -pub fn supports_eq_unordered(a: &Supports, b: &Supports) -> bool { +pub fn supports_eq_unordered( + a: &Supports, + b: &Supports, +) -> bool { let map: BTreeMap<_, _> = a.iter().cloned().collect(); b.iter().all(|(id, b_support)| { let a_support = match map.get(id) { From 1f02cb4dc1389a9ad6886faaf58b3b3597934e14 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 22 Jul 2021 11:29:20 +0200 Subject: [PATCH 074/241] Fix a bunch of stuff, remove not needed runtime arg of make-bags --- bin/node/runtime/voter-bags/src/main.rs | 38 +++++-------------------- frame/staking/src/lib.rs | 8 ++++-- frame/staking/src/voter_bags.rs | 26 ++++++----------- 3 files changed, 20 insertions(+), 52 deletions(-) diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 2a2be8297eb22..1340285c29a1a 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,26 +17,9 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::{mock::Test, voter_bags::make_bags::generate_thresholds_module}; -use std::path::{Path, PathBuf}; -use structopt::{clap::arg_enum, StructOpt}; - -arg_enum! { - #[derive(Debug)] - enum Runtime { - Node, - StakingMock, - } -} - -impl Runtime { - fn generate_thresholds(&self) -> Box Result<(), std::io::Error>> { - match self { - Runtime::Node => Box::new(generate_thresholds_module::), - Runtime::StakingMock => Box::new(generate_thresholds_module::), - } - } -} +use pallet_staking::{voter_bags::make_bags::generate_thresholds_module}; +use std::path::PathBuf; +use structopt::StructOpt; #[derive(Debug, StructOpt)] struct Opt { @@ -44,21 +27,14 @@ struct Opt { #[structopt(long, default_value = "200")] n_bags: usize, - /// Which runtime to generate. - #[structopt( - long, - case_insensitive = true, - default_value = "Node", - possible_values = &Runtime::variants(), - )] - runtime: Runtime, - /// Where to write the output. output: PathBuf, } fn main() -> Result<(), std::io::Error> { - let Opt { n_bags, output, runtime } = Opt::from_args(); + let Opt { n_bags, output } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| runtime.generate_thresholds()(n_bags, &output)) + ext.execute_with(|| { + generate_thresholds_module::(n_bags, &output) + }) } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index dbcf66292308f..1350582d99fd8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -271,13 +271,15 @@ //! - [Session](../pallet_session/index.html): Used to manage sessions. Also, a list of new //! validators is stored in the Session pallet's `Validators` at the end of each era. -#![recursion_limit = "128"] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(any(test, feature = "make-bags"))] -pub mod mock; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod testing_utils; +#[cfg(any(feature = "runtime-benchmarks", test))] +pub mod benchmarking; + +#[cfg(test)] +pub(crate) mod mock; #[cfg(test)] mod tests; diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 4917e3446d884..687f24f539178 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -366,29 +366,20 @@ impl VoterList { /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `VoterList`. - pub(super) fn sanity_check() -> Result<(), std::string::String> { + pub(super) fn sanity_check() -> Result<(), &'static str> { let mut seen_in_list = BTreeSet::new(); ensure!( Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), - String::from("duplicate identified"), + "duplicate identified", ); let iter_count = Self::iter().collect::>().len() as u32; let stored_count = crate::CounterForVoters::::get(); - ensure!( - iter_count == stored_count, - format!("iter_count ({}) != voter_count ({})", iter_count, stored_count), - ); + ensure!(iter_count == stored_count, "iter_count != voter_count"); let validators = crate::CounterForValidators::::get(); let nominators = crate::CounterForNominators::::get(); - ensure!( - validators + nominators == stored_count, - format!( - "validators {} + nominators {} != voters {}", - validators, nominators, stored_count - ), - ); + ensure!(validators + nominators == stored_count, "validators + nominators != voters"); let _ = T::VoterBagThresholds::get() .into_iter() @@ -536,14 +527,14 @@ impl Bag { /// * Ensures head has no prev. /// * Ensures tail has no next. /// * Ensures there are no loops, traversal from head to tail is correct. - fn sanity_check(&self) -> Result<(), std::string::String> { + fn sanity_check(&self) -> Result<(), &'static str> { ensure!( self.head() .map(|head| head.prev().is_none()) // if there is no head, then there must not be a tail, meaning that the bag is // empty. .unwrap_or_else(|| self.tail.is_none()), - String::from("head has a prev") + "head has a prev" ); ensure!( @@ -552,7 +543,7 @@ impl Bag { // if there is no tail, then there must not be a head, meaning that the bag is // empty. .unwrap_or_else(|| self.head.is_none()), - String::from("tail has a next") + "tail has a next" ); let mut seen_in_bag = BTreeSet::new(); @@ -561,7 +552,7 @@ impl Bag { .map(|node| node.voter.id) // each voter is only seen once, thus there is no cycle within a bag .all(|voter| seen_in_bag.insert(voter)), - String::from("Duplicate found in bag.") + "Duplicate found in bag" ); Ok(()) @@ -936,7 +927,6 @@ pub mod make_bags { mod voter_list { use super::*; use crate::mock::*; - use frame_support::traits::Currency; #[test] fn basic_setup_works() { From 2f83f164f907a5479845edd7bf1ed168475a4a74 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 22 Jul 2021 12:23:17 +0200 Subject: [PATCH 075/241] add logs --- frame/staking/src/voter_bags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 687f24f539178..d5bae70b29f46 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -172,7 +172,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); let bag = notional_bag_for::(weight); - crate::log!(debug, "inserting {:?} into bag {:?}", voter, bag); + crate::log!(debug, "inserting {:?} with weight {} into bag {:?}", voter, weight, bag); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } From 0df47b0bfa8a1e05a45ffcb1613e0bf0c1ca839a Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Fri, 23 Jul 2021 02:09:47 +0200 Subject: [PATCH 076/241] Glue the new staking bags to the election snapshot (#9415) * Glue the new staking bags to the election snapshot * add CheckedRem (#9412) * add CheckedRem * fix * Run fmt * Test comment Co-authored-by: Xiliang Chen Co-authored-by: emostov <32168567+emostov@users.noreply.github.com> --- bin/node/runtime/voter-bags/src/main.rs | 6 +- .../election-provider-multi-phase/src/lib.rs | 61 ++++++++++++++++++- .../election-provider-multi-phase/src/mock.rs | 8 ++- frame/staking/src/lib.rs | 18 +++--- frame/staking/src/tests.rs | 30 +++++++-- frame/staking/src/voter_bags.rs | 13 ++-- primitives/arithmetic/src/traits.rs | 6 +- primitives/npos-elections/src/lib.rs | 6 +- 8 files changed, 114 insertions(+), 34 deletions(-) diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 1340285c29a1a..a92af37fb5bf8 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,7 +17,7 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::{voter_bags::make_bags::generate_thresholds_module}; +use pallet_staking::voter_bags::make_bags::generate_thresholds_module; use std::path::PathBuf; use structopt::StructOpt; @@ -34,7 +34,5 @@ struct Opt { fn main() -> Result<(), std::io::Error> { let Opt { n_bags, output } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| { - generate_thresholds_module::(n_bags, &output) - }) + ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) } diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 905492d6ca04c..3841817d04f8b 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -632,6 +632,15 @@ pub mod pallet { #[pallet::constant] type SignedDepositWeight: Get>; + /// The number of snapshot voters to fetch per block. + /// + /// In the future, this value will make more sense with multi-block snapshot. + /// + /// Also, note the data type: If the voters are represented by a `u32` in `type + /// CompactSolution`, the same `u32` is used here to ensure bounds are respected. + #[pallet::constant] + type VoterSnapshotPerBlock: Get>; + /// Handler for the slashed deposits. type SlashHandler: OnUnbalanced>; @@ -1285,8 +1294,11 @@ impl Pallet { /// /// Returns `Ok(consumed_weight)` if operation is okay. pub fn create_snapshot() -> Result { + // we don't impose any limits on the targets for now, the assumption is that + // `T::DataProvider` will sensibly return small values to use. let target_limit = >::max_value().saturated_into::(); - let voter_limit = >::max_value().saturated_into::(); + // for now we have just a single block snapshot. + let voter_limit = T::VoterSnapshotPerBlock::get().saturated_into::(); let (targets, w1) = T::DataProvider::targets(Some(target_limit)).map_err(ElectionError::DataProvider)?; @@ -1970,8 +1982,8 @@ mod tests { }) } - #[test] - fn snapshot_creation_fails_if_too_big() { + fn snapshot_too_big_failure_onchain_fallback() { + // the `MockStaking` is designed such that if it has too many targets, it simply fails. ExtBuilder::default().build_and_execute(|| { Targets::set((0..(TargetIndex::max_value() as AccountId) + 1).collect::>()); @@ -1987,6 +1999,49 @@ mod tests { roll_to(29); let (supports, _) = MultiPhase::elect().unwrap(); assert!(supports.len() > 0); + }); + } + + #[test] + fn snapshot_too_big_failure_no_fallback() { + // and if the backup mode is nothing, we go into the emergency mode.. + ExtBuilder::default().fallback(FallbackStrategy::Nothing).build_and_execute(|| { + crate::mock::Targets::set( + (0..(TargetIndex::max_value() as AccountId) + 1).collect::>(), + ); + + // Signed phase failed to open. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + // Unsigned phase failed to open. + roll_to(25); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + roll_to(29); + let err = MultiPhase::elect().unwrap_err(); + assert_eq!(err, ElectionError::NoFallbackConfigured); + assert_eq!(MultiPhase::current_phase(), Phase::Emergency); + }); + } + + #[test] + fn snapshot_too_big_truncate() { + // but if there are too many voters, we simply truncate them. + ExtBuilder::default().build_and_execute(|| { + // we have 8 voters in total. + assert_eq!(crate::mock::Voters::get().len(), 8); + // but we want to take 4. + crate::mock::VoterSnapshotPerBlock::set(2); + + // Signed phase opens just fine. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Signed); + + assert_eq!( + MultiPhase::snapshot_metadata().unwrap(), + SolutionOrSnapshotSize { voters: 2, targets: 4 } + ); }) } diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index c5007733c1e33..8e6424844f1e4 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -270,6 +270,7 @@ parameter_types! { pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxLength: u32 = 256; pub static MockWeightInfo: bool = false; + pub static VoterSnapshotPerBlock: VoterIndex = u32::max_value(); pub static EpochLength: u64 = 30; } @@ -379,6 +380,7 @@ impl crate::Config for Runtime { type Fallback = Fallback; type ForceOrigin = frame_system::EnsureRoot; type CompactSolution = TestCompact; + type VoterSnapshotPerBlock = VoterSnapshotPerBlock; } impl frame_system::offchain::SendTransactionTypes for Runtime @@ -410,9 +412,9 @@ impl ElectionDataProvider for StakingMock { fn voters( maybe_max_len: Option, ) -> data_provider::Result<(Vec<(AccountId, VoteWeight, Vec)>, Weight)> { - let voters = Voters::get(); - if maybe_max_len.map_or(false, |max_len| voters.len() > max_len) { - return Err("Voters too big") + let mut voters = Voters::get(); + if let Some(max_len) = maybe_max_len { + voters.truncate(max_len) } Ok((voters, 0)) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 4ac2b7404d0f3..3574c34602bbb 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -273,10 +273,10 @@ #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(any(feature = "runtime-benchmarks", test))] -pub mod testing_utils; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod benchmarking; +#[cfg(any(feature = "runtime-benchmarks", test))] +pub mod testing_utils; #[cfg(test)] pub(crate) mod mock; @@ -3111,12 +3111,6 @@ impl Pallet { maybe_max_len: Option, voter_count: usize, ) -> Vec> { - debug_assert_eq!( - voter_count, - VoterList::::decode_len().unwrap_or_default(), - "voter_count must be accurate", - ); - let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); let weight_of = Self::weight_of_fn(); @@ -3240,8 +3234,15 @@ impl let nominator_count = CounterForNominators::::get(); let validator_count = CounterForValidators::::get(); let voter_count = nominator_count.saturating_add(validator_count) as usize; + + // check a few counters one last time... debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); + debug_assert_eq!( + voter_count, + VoterList::::decode_len().unwrap_or_default(), + "voter_count must be accurate", + ); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( @@ -3249,6 +3250,7 @@ impl validator_count, slashing_span_count as u32, ); + Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index cec490cdbfb67..4ae38611492e5 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -4006,9 +4006,21 @@ mod election_data_provider { } #[test] - fn respects_len_limits() { - ExtBuilder::default().build_and_execute(|| { + fn respects_snapshot_len_limits() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // sum of all validators and nominators who'd be voters. + assert_eq!(VoterList::::decode_len().unwrap(), 5); + + // if limits is less.. assert_eq!(Staking::voters(Some(1)).unwrap().0.len(), 1); + + // if limit is equal.. + assert_eq!(Staking::voters(Some(5)).unwrap().0.len(), 5); + + // if limit is more. + assert_eq!(Staking::voters(Some(55)).unwrap().0.len(), 5); + + // if target limit is less, then we return an error. assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); }); } @@ -4068,12 +4080,22 @@ mod election_data_provider { #[test] #[should_panic] - fn count_check_works() { + fn count_check_prevents_validator_insert() { ExtBuilder::default().build_and_execute(|| { // We should never insert into the validators or nominators map directly as this will // not keep track of the count. This test should panic as we verify the count is accurate - // after every test using the `post_checks` in `mock`. + // after every test using the `post_conditions` in `mock`. Validators::::insert(987654321, ValidatorPrefs::default()); + }) + } + + #[test] + #[should_panic] + fn count_check_prevents_nominator_insert() { + ExtBuilder::default().build_and_execute(|| { + // We should never insert into the validators or nominators map directly as this will + // not keep track of the count. This test should panic as we verify the count is accurate + // after every test using the `post_conditions` in `mock`. Nominators::::insert( 987654321, Nominations { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index d5bae70b29f46..1927c4c6c049c 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -620,15 +620,13 @@ impl Node { weight_of: impl Fn(&T::AccountId) -> VoteWeight, slashing_spans: &BTreeMap, SlashingSpans>, ) -> Option> { + let voter_weight = weight_of(&self.voter.id); match self.voter.voter_type { - VoterType::Validator => Some(( - self.voter.id.clone(), - weight_of(&self.voter.id), - sp_std::vec![self.voter.id.clone()], - )), + VoterType::Validator => + Some((self.voter.id.clone(), voter_weight, sp_std::vec![self.voter.id.clone()])), VoterType::Nominator => { let Nominations { submitted_in, mut targets, .. } = - Nominators::::get(self.voter.id.clone())?; + Nominators::::get(&self.voter.id)?; // Filter out nomination targets which were nominated before the most recent // slashing span. targets.retain(|stash| { @@ -637,8 +635,7 @@ impl Node { .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) }); - (!targets.is_empty()) - .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) + (!targets.is_empty()).then(move || (self.voter.id.clone(), voter_weight, targets)) }, } } diff --git a/primitives/arithmetic/src/traits.rs b/primitives/arithmetic/src/traits.rs index a441a0dcbc08d..53341117b1fee 100644 --- a/primitives/arithmetic/src/traits.rs +++ b/primitives/arithmetic/src/traits.rs @@ -20,8 +20,8 @@ use codec::HasCompact; pub use integer_sqrt::IntegerSquareRoot; pub use num_traits::{ - checked_pow, Bounded, CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedShl, CheckedShr, - CheckedSub, One, Signed, Unsigned, Zero, + checked_pow, Bounded, CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedRem, CheckedShl, + CheckedShr, CheckedSub, One, Signed, Unsigned, Zero, }; use sp_std::{ self, @@ -55,6 +55,7 @@ pub trait BaseArithmetic: + CheckedSub + CheckedMul + CheckedDiv + + CheckedRem + Saturating + PartialOrd + Ord @@ -109,6 +110,7 @@ impl< + CheckedSub + CheckedMul + CheckedDiv + + CheckedRem + Saturating + PartialOrd + Ord diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index ad2b3229a0881..ff4919876aff1 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -155,7 +155,8 @@ where + Debug + Copy + Clone - + Bounded; + + Bounded + + Encode; /// The target type. Needs to be an index (convert to usize). type Target: UniqueSaturatedInto @@ -164,7 +165,8 @@ where + Debug + Copy + Clone - + Bounded; + + Bounded + + Encode; /// The weight/accuracy type of each vote. type Accuracy: PerThing128; From 8706404fccd542ca157c347cbaa7f69f4aa2ebbf Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 27 Jul 2021 15:30:21 -0700 Subject: [PATCH 077/241] Update node runtime with VoterSnapshotPerBlock --- bin/node/runtime/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 9c7c8a1d022d9..b319bcab92eb6 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -548,6 +548,8 @@ parameter_types! { *RuntimeBlockLength::get() .max .get(DispatchClass::Normal); + + pub const VoterSnapshotPerBlock: u32 = u32::max_value(); } sp_npos_elections::generate_solution_type!( @@ -602,6 +604,7 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; type ForceOrigin = EnsureRootOrHalfCouncil; type BenchmarkingConfig = BenchmarkConfig; + type VoterSnapshotPerBlock = VoterSnapshotPerBlock; } parameter_types! { From ced60772e9dc3c71e014971d00a6471dc804a7d4 Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 28 Jul 2021 09:32:04 -0700 Subject: [PATCH 078/241] Unit test for pallet-staking unsorted bags feature (targets #9081) (#9422) * impl notional_bag_for_works * Add tests: insert_as_works & insert_works * Impl test: remove_works * Trivial cleaning * Add test: update_position_for_works * Write out edge case; probably can delete later * Add test: bags::get_works * Add test: remove_node_happy_path_works * Add test: remove_node_bad_paths_documented * WIP: voting_data_works * done * Improve test voting_data_works * Add comment * Fill out test basic_setup_works * Update: iteration_is_semi_sorted * Improve remove_works * Update update_position_for_works; create set_ledger_and_free_balance * Improve get_works * Improve storage clean up checks in remove test * Test: impl rebag_works + insert_and_remove_works * forgot file - Test: impl rebag_works + insert_and_remove_works * Small tweak * Update voter_bags test to reflect unused bags are removed * Unbond & Rebond: do_rebag * Prevent infinite loops with duplicate tail insert * Check iter.count on voter list in pre-migrate * undo strang fmt comment stuff * Add in todo Co-authored-by: kianenigma --- frame/staking/src/lib.rs | 4 +- frame/staking/src/mock.rs | 45 +- frame/staking/src/tests.rs | 151 +++--- frame/staking/src/voter_bags.rs | 845 +++++++++++++++++++++++++------- 4 files changed, 776 insertions(+), 269 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 08d162c9fcedc..6121d3a3affa8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -766,7 +766,7 @@ pub mod migrations { pub fn pre_migrate() -> Result<(), &'static str> { ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); ensure!( - VoterList::::decode_len().unwrap_or_default() == 0, + VoterList::::iter().count() == 0, "voter list already exists" ); Ok(()) @@ -1781,6 +1781,7 @@ pub mod pallet { let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); Self::deposit_event(Event::::Unbonded(ledger.stash, value)); } Ok(()) @@ -2273,6 +2274,7 @@ pub mod pallet { Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); Ok(Some( 35 * WEIGHT_PER_MICROS + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index c16d49f7dce16..a08b2e671042d 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -493,13 +493,23 @@ impl ExtBuilder { ext.execute_with(test); ext.execute_with(post_conditions); } + /// WARNING: This should only be use for testing `VoterList` api or lower. + pub fn build_and_execute_without_check_count(self, test: impl FnOnce() -> ()) { + let mut ext = self.build(); + ext.execute_with(test); + ext.execute_with(post_conditions_without_check_count); + } } fn post_conditions() { + post_conditions_without_check_count(); + check_count(); +} + +fn post_conditions_without_check_count() { check_nominators(); check_exposures(); check_ledgers(); - check_count(); } fn check_count() { @@ -602,10 +612,14 @@ pub(crate) fn current_era() -> EraIndex { Staking::current_era().unwrap() } -pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { +pub(crate) fn bond(stash: AccountId, ctrl: AccountId, val: Balance) { let _ = Balances::make_free_balance_be(&stash, val); let _ = Balances::make_free_balance_be(&ctrl, val); assert_ok!(Staking::bond(Origin::signed(stash), ctrl, val, RewardDestination::Controller)); +} + +pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { + bond(stash, ctrl, val); assert_ok!(Staking::validate(Origin::signed(ctrl), ValidatorPrefs::default())); } @@ -615,9 +629,7 @@ pub(crate) fn bond_nominator( val: Balance, target: Vec, ) { - let _ = Balances::make_free_balance_be(&stash, val); - let _ = Balances::make_free_balance_be(&ctrl, val); - assert_ok!(Staking::bond(Origin::signed(stash), ctrl, val, RewardDestination::Controller)); + bond(stash, ctrl, val); assert_ok!(Staking::nominate(Origin::signed(ctrl), target)); } @@ -833,3 +845,26 @@ pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { }) .collect::>() } + +pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { + bag.iter().map(|n| n.voter().id).collect::>() +} + +pub(crate) fn get_voter_list_as_ids() -> Vec { + VoterList::::iter().map(|n| n.voter().id).collect::>() +} + +pub(crate) fn get_voter_list_as_voters() -> Vec> { + VoterList::::iter().map(|node| node.voter().clone()).collect::>() +} + +// Useful for when you want to change the effectively bonded value but you don't want to use +// the bond extrinsics because they implicitly rebag. +pub(crate) fn set_ledger_and_free_balance(account: &AccountId, value: Balance) { + Balances::make_free_balance_be(account, value); + let controller = Staking::bonded(account).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = value; + ledger.active = value; + Staking::update_ledger(&controller, &ledger); +} diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 4ae38611492e5..11432e6d68e3d 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3827,88 +3827,87 @@ fn on_finalize_weight_is_nonzero() { // end-to-end nodes of the voter bags operation. mod voter_bags { + use super::Origin; + use crate::{mock::*, ValidatorPrefs}; + use frame_support::{assert_ok, traits::Currency}; #[test] - fn rebag_works() { - todo!() - } -} -/* -// TODO: this needs some love, retire it in favour of the one above. Use the mock data, don't make -// it complicated with data setup, use the simplest data possible, instead check multiple -// edge-cases. -#[test] -fn test_rebag() { - use crate::{ - testing_utils::create_stash_controller, - voter_bags::{Bag, Node}, - }; - use frame_system::RawOrigin; - - /// Make a validator and return its stash - fn make_validator(n: u32, balance_factor: u32) -> Result, &'static str> { - let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); - - // Bond the full value of the stash - // - // By default, `create_stash_controller` only bonds 10% of the stash. However, we're going - // to want to edit one account's bonded value to match another's, so it's simpler if 100% of - // the balance is bonded. - let balance = ::Currency::free_balance(&stash); - Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), balance).unwrap(); - Staking::validate( - RawOrigin::Signed(controller.clone()).into(), - ValidatorPrefs::default(), - ).unwrap(); - - Ok(stash) + fn insert_and_remove_works() { + // we test insert/remove indirectly via `validate`, `nominate`, and chill + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // `bond` + bond(42, 43, 2_000); + // does not insert the voter + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // `validate` + assert_ok!(Staking::validate(Origin::signed(43).into(), ValidatorPrefs::default())); + // moves the voter into a bag + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // `nominate`-ing, but not changing active stake (which implicitly calls remove) + assert_ok!(Staking::nominate(Origin::signed(43), vec![11])); + // does not change the voters position + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // `chill` + assert_ok!(Staking::chill(Origin::signed(43))); + // removes the voter + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + }); } - ExtBuilder::default().build_and_execute(|| { - // We want to have two validators: one, `stash`, is the one we will rebag. - // The other, `other_stash`, exists only so that the destination bag is not empty. - let stash = make_validator(0, 2000).unwrap(); - let other_stash = make_validator(1, 9000).unwrap(); - - // verify preconditions - let weight_of = Staking::weight_of_fn(); - let node = Node::::from_id(&stash).unwrap(); - assert_eq!( - { - let origin_bag = Bag::::get(node.bag_upper).unwrap(); - origin_bag.iter().count() - }, - 1, - "stash should be the only node in origin bag", - ); - let other_node = Node::::from_id(&other_stash).unwrap(); - assert!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); - assert_ne!( - { - let destination_bag = Bag::::get(other_node.bag_upper); - destination_bag.iter().count() - }, - 0, - "destination bag should not be empty", - ); + #[test] + fn rebag_works() { + ExtBuilder::default().build_and_execute(|| { + // add a nominator to genesis state + bond_nominator(42, 43, 20, vec![11]); + Balances::make_free_balance_be(&42, 2_000); - // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. - // - // This implicitly calls rebag, so the user stays in the best bag they qualify for. - let new_balance = ::Currency::free_balance(&other_stash); - ::Currency::make_free_balance_be(&stash, new_balance); - Staking::bond_extra( - RawOrigin::Signed(stash.clone()).into(), - new_balance, - ).unwrap(); - - // node should no longer be misplaced - // note that we refresh the node, in case the storage value has changed - let node = Node::::from_id(&stash).unwrap(); - assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); - }); + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); + + // increase stake and implicitly rebag with `bond_extra` to the level of non-existent bag + assert_ok!(Staking::bond_extra(Origin::signed(42), 1_980)); // 20 + 1_980 = 2_000 + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // decrease stake within the range of the current bag + assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 + // does not change bags + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // reduce stake to the level of a non-existent bag + assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 + // creates the bag and moves the voter into it + assert_eq!( + get_bags(), + vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] + ); + + // increase stake by `rebond`-ing to the level of a pre-existing bag + assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 + // moves the voter to that bag + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); + + // TODO test rebag directly + }); + } } -*/ mod election_data_provider { use super::*; diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 1927c4c6c049c..04d1943164bfb 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -400,6 +400,7 @@ impl VoterList { /// appearing within the voter set. #[derive(DefaultNoBound, Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq))] pub struct Bag { head: Option>, tail: Option>, @@ -430,9 +431,18 @@ impl Bag { Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } + /// `True` if self is empty. + pub fn is_empty(&self) -> bool { + self.head.is_none() && self.tail.is_none() + } + /// Put the bag back into storage. pub fn put(self) { - crate::VoterBags::::insert(self.bag_upper, self); + if self.is_empty() { + crate::VoterBags::::remove(self.bag_upper); + } else { + crate::VoterBags::::insert(self.bag_upper, self); + } } /// Get the head node in this bag. @@ -469,6 +479,15 @@ impl Bag { /// Storage note: this modifies storage, but only for the node. You still need to call /// `self.put()` after use. fn insert_node(&mut self, mut node: Node) { + if let Some(tail) = &self.tail { + if *tail == node.voter.id { + // this should never happen, but this check prevents a worst case infinite loop + debug_assert!(false, "system logic error: inserting a node who has the id of tail"); + crate::log!(warn, "system logic error: inserting a node who has the id of tail"); + return + }; + } + let id = node.voter.id.clone(); node.prev = self.tail.clone(); @@ -562,6 +581,7 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq, Clone))] pub struct Node { voter: Voter>, prev: Option>, @@ -670,8 +690,8 @@ impl Node { notional_bag_for::(current_weight) } - #[cfg(any(test, feature = "runtime-benchmarks"))] /// Get the underlying voter. + #[cfg(any(test, feature = "runtime-benchmarks"))] pub fn voter(&self) -> &Voter { &self.voter } @@ -924,28 +944,66 @@ pub mod make_bags { mod voter_list { use super::*; use crate::mock::*; + use frame_support::{assert_ok, assert_storage_noop, traits::Currency}; #[test] fn basic_setup_works() { + use crate::{ + CounterForNominators, CounterForValidators, CounterForVoters, VoterBags, VoterNodes, + }; + let node = |voter, prev, next| Node:: { voter, prev, next, bag_upper: 0 }; + // make sure ALL relevant data structures are setup correctly. - // TODO: we are not checking all of them yet. ExtBuilder::default().build_and_execute(|| { - assert_eq!(crate::CounterForVoters::::get(), 4); + assert_eq!(CounterForVoters::::get(), 4); + assert_eq!(VoterBagFor::::iter().count(), 4); + assert_eq!(VoterNodes::::iter().count(), 4); + assert_eq!(VoterBags::::iter().count(), 2); + assert_eq!(CounterForValidators::::get(), 3); + assert_eq!(CounterForNominators::::get(), 1); + + assert_eq!( + VoterBags::::get(10).unwrap(), + Bag:: { head: Some(31), tail: Some(31), bag_upper: 0 } + ); + assert_eq!( + VoterBags::::get(1_000).unwrap(), + Bag:: { head: Some(11), tail: Some(101), bag_upper: 0 } + ); let weight_of = Staking::weight_of_fn(); + assert_eq!(weight_of(&11), 1000); assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(11).unwrap(), + node(Voter::validator(11), None, Some(21)) + ); assert_eq!(weight_of(&21), 1000); assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(21).unwrap(), + node(Voter::validator(21), Some(11), Some(101)) + ); assert_eq!(weight_of(&31), 1); assert_eq!(VoterBagFor::::get(31).unwrap(), 10); + assert_eq!( + VoterNodes::::get(31).unwrap(), + node(Voter::validator(31), None, None) + ); + assert_eq!(weight_of(&41), 1000); assert_eq!(VoterBagFor::::get(41), None); // this staker is chilled! + assert_eq!(VoterNodes::::get(41), None); assert_eq!(weight_of(&101), 500); assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(101).unwrap(), + node(Voter::nominator(101), Some(21), None) + ); // iteration of the bags would yield: assert_eq!( @@ -954,13 +1012,48 @@ mod voter_list { // ^^ note the order of insertion in genesis! ); - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); }) } #[test] fn notional_bag_for_works() { - todo!(); + // under a threshold gives the next threshold. + assert_eq!(notional_bag_for::(0), 10); + assert_eq!(notional_bag_for::(9), 10); + assert_eq!(notional_bag_for::(11), 20); + + // at a threshold gives that threshold. + assert_eq!(notional_bag_for::(10), 10); + + let max_explicit_threshold = *::VoterBagThresholds::get().last().unwrap(); + assert_eq!(max_explicit_threshold, 10_000); + // if the max explicit threshold is less than VoteWeight::MAX, + assert!(VoteWeight::MAX > max_explicit_threshold); + // anything above it will belong to the VoteWeight::MAX bag. + assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); + } + + #[test] + fn remove_last_voter_in_bags_cleans_bag() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // give 31 more stake to bump it to a new bag. + Balances::make_free_balance_be(&31, 10000); + assert_ok!(Staking::bond_extra(Origin::signed(31), 10000 - 10)); + + // then the bag with bound 10 is wiped from storage. + assert_eq!(get_bags(), vec![(1000, vec![11, 21, 101]), (10_000, vec![31])]); + + // and can be recreated again as needed + bond_validator(77, 777, 10); + assert_eq!( + get_bags(), + vec![(10, vec![77]), (1000, vec![11, 21, 101]), (10_000, vec![31])] + ); + }); } #[test] @@ -976,18 +1069,29 @@ mod voter_list { vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], ); - // when - let iteration = VoterList::::iter().map(|node| node.voter.id).collect::>(); - // then assert_eq!( - iteration, + get_voter_list_as_ids(), vec![ 51, 61, // best bag 11, 21, 101, // middle bag 31, // last bag. ] ); + + // when adding a voter that has a higher weight than pre-existing voters in the bag + bond_validator(71, 70, 10); + + // then + assert_eq!( + get_voter_list_as_ids(), + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, + 71, // last bag; the new voter is last, because it is order of insertion + ] + ); }) } @@ -1020,239 +1124,606 @@ mod voter_list { }) } - #[test] - fn storage_is_cleaned_up_as_voters_are_removed() {} - #[test] fn insert_works() { - todo!() + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when inserting into an existing bag + bond(42, 43, 1_000); + VoterList::::insert(Voter::<_>::nominator(42), Pallet::::weight_of_fn()); + + // then + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 42, 31]); + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42])]); + + // when inserting into a non-existent bag + bond(422, 433, 1_001); + VoterList::::insert(Voter::<_>::nominator(422), Pallet::::weight_of_fn()); + + // then + assert_eq!(get_voter_list_as_ids(), vec![422, 11, 21, 101, 42, 31]); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42]), (2_000, vec![422])] + ); + }); } #[test] fn insert_as_works() { - // insert a new one with role - // update the status of already existing one. - todo!() + ExtBuilder::default().build_and_execute_without_check_count(|| { + // given + let actual = get_voter_list_as_voters(); + let mut expected: Vec> = vec![ + Voter::<_>::validator(11), + Voter::<_>::validator(21), + Voter::<_>::nominator(101), + Voter::<_>::validator(31), + ]; + assert_eq!(actual, expected); + + // when inserting a new voter + VoterList::::insert_as(&42, VoterType::Nominator); + + // then + let actual = get_voter_list_as_voters(); + expected.push(Voter::<_>::nominator(42)); + assert_eq!(actual, expected); + + // when updating the voter type of an already existing voter + VoterList::::insert_as(&42, VoterType::Validator); + + // then + let actual = get_voter_list_as_voters(); + expected[4] = Voter::<_>::validator(42); + assert_eq!(actual, expected); + }); } #[test] fn remove_works() { - todo!() + use crate::{CounterForVoters, VoterBags, VoterNodes}; + + let check_storage = |id, counter, voters, bags| { + assert!(!VoterBagFor::::contains_key(id)); + assert!(!VoterNodes::::contains_key(id)); + assert_eq!(CounterForVoters::::get(), counter); + assert_eq!(VoterBagFor::::iter().count() as u32, counter); + assert_eq!(VoterNodes::::iter().count() as u32, counter); + assert_eq!(get_voter_list_as_ids(), voters); + assert_eq!(get_bags(), bags); + }; + + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when removing a non-existent voter + VoterList::::remove(&42); + assert!(!VoterBagFor::::contains_key(42)); + assert!(!VoterNodes::::contains_key(42)); + + // then nothing changes + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(CounterForVoters::::get(), 4); + + // when removing a node from a bag with multiple nodes + VoterList::::remove(&11); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); + check_storage( + 11, + 3, + vec![21, 101, 31], // voter list + vec![(10, vec![31]), (1_000, vec![21, 101])], // bags + ); + + // when removing a node from a bag with only one node: + VoterList::::remove(&31); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101]); + check_storage( + 31, + 2, + vec![21, 101], // voter list + vec![(1_000, vec![21, 101])], // bags + ); + assert!(!VoterBags::::contains_key(10)); // bag 10 is removed + + // remove remaining voters to make sure storage cleans up as expected + VoterList::::remove(&21); + check_storage( + 21, + 1, + vec![101], // voter list + vec![(1_000, vec![101])], // bags + ); + + VoterList::::remove(&101); + check_storage( + 101, + 0, + Vec::::new(), // voter list + vec![], // bags + ); + assert!(!VoterBags::::contains_key(1_000)); // bag 1_000 is removed + + // bags are deleted via removals + assert_eq!(VoterBags::::iter().count(), 0); + // nominator and validator counters are not updated at this level of the api + assert_eq!(crate::CounterForValidators::::get(), 3); + assert_eq!(crate::CounterForNominators::::get(), 1); + }); } #[test] fn update_position_for_works() { - // alter the genesis state to require a re-bag, then ensure this fixes it. Might be similar - // `rebag_works()` - todo!(); + ExtBuilder::default().build_and_execute_without_check_count(|| { + let weight_of = Staking::weight_of_fn(); + + // given a correctly placed account 31 + let node_31 = Node::::from_id(&31).unwrap(); + assert!(!node_31.is_misplaced(&weight_of)); + + // when account 31 bonds extra and needs to be moved to a non-existing higher bag + // (we can't call bond_extra, because that implicitly calls update_position_for) + set_ledger_and_free_balance(&31, 11); + + assert!(node_31.is_misplaced(&weight_of)); + assert_eq!(weight_of(&31), 11); + + // then updating position moves it to the correct bag + assert_eq!(VoterList::::update_position_for(node_31, &weight_of), Some((10, 20))); + assert_eq!(get_bags(), vec![(20, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + + // and if you try and update the position with no change in active stake nothing changes + let node_31 = Node::::from_id(&31).unwrap(); + assert_storage_noop!(assert_eq!( + VoterList::::update_position_for(node_31, &weight_of), + None, + )); + + // when account 31 bonds extra and needs to be moved to an existing higher bag + set_ledger_and_free_balance(&31, 61); + + // then updating positions moves it to the correct bag + let node_31 = Node::::from_id(&31).unwrap(); + assert_eq!( + VoterList::::update_position_for(node_31, &weight_of), + Some((20, 1_000)) + ); + assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101, 31])]); + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + + // when account 31 bonds extra but should not change bags + set_ledger_and_free_balance(&31, 1_000); + + // then nothing changes + let node_31 = Node::::from_id(&31).unwrap(); + assert_storage_noop!(assert_eq!( + VoterList::::update_position_for(node_31, &weight_of), + None, + )); + }); } } #[cfg(test)] mod bags { + use super::*; + use crate::mock::*; + use frame_support::{assert_ok, assert_storage_noop}; + #[test] fn get_works() { - todo!() + use crate::VoterBags; + ExtBuilder::default().build_and_execute_without_check_count(|| { + let check_bag = |bag_upper, head, tail, ids| { + assert_storage_noop!(Bag::::get(bag_upper)); + + let bag = Bag::::get(bag_upper).unwrap(); + let bag_ids = bag.iter().map(|n| n.voter().id).collect::>(); + + assert_eq!(bag, Bag:: { head, tail, bag_upper }); + assert_eq!(bag_ids, ids); + }; + + // given uppers of bags that exist. + let existing_bag_uppers = vec![10, 1_000]; + + // we can fetch them + check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); + // (getting the same bag twice has the same results) + check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); + check_bag(existing_bag_uppers[1], Some(11), Some(101), vec![11, 21, 101]); + + // and all other uppers don't get bags. + ::VoterBagThresholds::get() + .iter() + .chain(iter::once(&VoteWeight::MAX)) + .filter(|bag_upper| !existing_bag_uppers.contains(bag_upper)) + .for_each(|bag_upper| { + assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); + assert!(!VoterBags::::contains_key(*bag_upper)); + }); + + // when we make a pre-existing bag empty + VoterList::::remove(&31); + + // then + assert_eq!(Bag::::get(existing_bag_uppers[0]), None) + }); } #[test] - fn insert_works() { - todo!() + #[should_panic] + fn get_panics_with_a_bad_threshold() { + // NOTE: panic is only expected with debug compilation + ExtBuilder::default().build_and_execute_without_check_count(|| { + Bag::::get(11); + }); } #[test] - fn remove_works() { - todo!() + fn insert_node_happy_paths_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let node = |voter, bag_upper| Node:: { voter, prev: None, next: None, bag_upper }; + + // when inserting into a bag with 1 node + let mut bag_10 = Bag::::get(10).unwrap(); + // (note: bags api does not care about balance or ledger) + bag_10.insert_node(node(Voter::nominator(42), bag_10.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_10), vec![31, 42]); + + // when inserting into a bag with 3 nodes + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.insert_node(node(Voter::nominator(52), bag_1000.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 52]); + + // when inserting into a new bag + let mut bag_20 = Bag::::get_or_make(20); + bag_20.insert_node(node(Voter::nominator(71), bag_20.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_20), vec![71]); + + // when inserting a node pointing to the accounts not in the bag + let voter_61 = Voter::validator(61); + let node_61 = Node:: { + voter: voter_61.clone(), + prev: Some(21), + next: Some(101), + bag_upper: 20, + }; + bag_20.insert_node(node_61); + // then ids are in order + assert_eq!(bag_as_ids(&bag_20), vec![71, 61]); + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(20, &61).unwrap(), + Node:: { voter: voter_61, prev: Some(71), next: None, bag_upper: 20 } + ); + + // state of all bags is as expected + bag_20.put(); // need to put this bag so its in the storage map + assert_eq!( + get_bags(), + vec![(10, vec![31, 42]), (20, vec![71, 61]), (1_000, vec![11, 21, 101, 52])] + ); + }); } -} -#[cfg(test)] -mod voter_node { + // Document improper ways `insert_node` may be getting used. #[test] - fn get_voting_data_works() { - todo!() + fn insert_node_bad_paths_documented() { + let node = |voter, prev, next, bag_upper| Node:: { voter, prev, next, bag_upper }; + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when inserting a node with both prev & next pointing at an account in the bag + // and an incorrect bag_upper + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let voter_42 = Voter::nominator(42); + bag_1000.insert_node(node(voter_42.clone(), Some(11), Some(11), 0)); + + // then the ids are in the correct order + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 42]); + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(1_000, &42).unwrap(), + node(voter_42, Some(101), None, bag_1000.bag_upper) + ); + + // given 21 is a validator in bag_1000 (and not a tail node) + let bag_1000_voter = + bag_1000.iter().map(|node| node.voter().clone()).collect::>(); + assert_eq!(bag_1000_voter[1], Voter::validator(21)); + + // when inserting a node with duplicate id 21 but as a nominator + let voter_21_nom = Voter::nominator(21); + bag_1000.insert_node(node(voter_21_nom.clone(), None, None, bag_1000.bag_upper)); + + // then all the nodes after the duplicate are lost (because it is set as the tail) + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21]); + // and the re-fetched node is a nominator with an **incorrect** prev pointer. + assert_eq!( + Node::::get(1_000, &21).unwrap(), + node(voter_21_nom, Some(42), None, bag_1000.bag_upper) + ); + }); + + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when inserting a duplicate id of the head + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let voter_11 = Voter::validator(11); + bag_1000.insert_node(node(voter_11.clone(), None, None, 0)); + // then all nodes after the head are lost + assert_eq!(bag_as_ids(&bag_1000), vec![11]); + // and the re-fetched node + assert_eq!( + Node::::get(1_000, &11).unwrap(), + node(voter_11, Some(101), None, bag_1000.bag_upper) + ); + + assert_eq!( + bag_1000, + Bag { + head: Some(11), tail: Some(11), bag_upper: 1_000 + } + ) + }); } #[test] - fn is_misplaced_works() { - todo!() + #[should_panic = "system logic error: inserting a node who has the id of tail"] + fn insert_node_duplicate_tail_panics_with_debug_assert() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let node = |voter, prev, next, bag_upper| Node:: { voter, prev, next, bag_upper }; + + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + + // when inserting a duplicate id that is already the tail + assert_eq!(bag_1000.tail, Some(101)); + let voter_101 = Voter::validator(101); + bag_1000.insert_node(node(voter_101, None, None, bag_1000.bag_upper)); // panics + }); + } + + #[test] + fn remove_node_happy_paths_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + // add some validators to genesis state + bond_validator(51, 50, 1_000); + bond_validator(61, 60, 1_000); + bond_validator(71, 70, 10); + bond_validator(81, 80, 10); + bond_validator(91, 90, 2_000); + bond_validator(161, 160, 2_000); + bond_validator(171, 170, 2_000); + bond_validator(181, 180, 2_000); + bond_validator(191, 190, 2_000); + + let mut bag_10 = Bag::::get(10).unwrap(); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let mut bag_2000 = Bag::::get(2_000).unwrap(); + + // given + assert_eq!(bag_as_ids(&bag_10), vec![31, 71, 81]); + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 51, 61]); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 161, 171, 181, 191]); + + // remove node that is not pointing at head or tail + let node_101 = Node::::get(bag_1000.bag_upper, &101).unwrap(); + let node_101_pre_remove = node_101.clone(); + bag_1000.remove_node(&node_101); + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 51, 61]); + assert_ok!(bag_1000.sanity_check()); + // node isn't mutated when its removed + assert_eq!(node_101, node_101_pre_remove); + + // remove head when its not pointing at tail + let node_11 = Node::::get(bag_1000.bag_upper, &11).unwrap(); + bag_1000.remove_node(&node_11); + assert_eq!(bag_as_ids(&bag_1000), vec![21, 51, 61]); + assert_ok!(bag_1000.sanity_check()); + + // remove tail when its not pointing at head + let node_61 = Node::::get(bag_1000.bag_upper, &61).unwrap(); + bag_1000.remove_node(&node_61); + assert_eq!(bag_as_ids(&bag_1000), vec![21, 51]); + assert_ok!(bag_1000.sanity_check()); + + // remove tail when its pointing at head + let node_51 = Node::::get(bag_1000.bag_upper, &51).unwrap(); + bag_1000.remove_node(&node_51); + assert_eq!(bag_as_ids(&bag_1000), vec![21]); + assert_ok!(bag_1000.sanity_check()); + + // remove node that is head & tail + let node_21 = Node::::get(bag_1000.bag_upper, &21).unwrap(); + bag_1000.remove_node(&node_21); + bag_1000.put(); // put into storage so get returns the updated bag + assert_eq!(Bag::::get(1_000), None); + + // remove node that is pointing at head and tail + let node_71 = Node::::get(bag_10.bag_upper, &71).unwrap(); + bag_10.remove_node(&node_71); + assert_eq!(bag_as_ids(&bag_10), vec![31, 81]); + assert_ok!(bag_10.sanity_check()); + + // remove head when pointing at tail + let node_31 = Node::::get(bag_10.bag_upper, &31).unwrap(); + bag_10.remove_node(&node_31); + assert_eq!(bag_as_ids(&bag_10), vec![81]); + assert_ok!(bag_10.sanity_check()); + bag_10.put(); // since we updated the bag's head/tail, we need to write this storage + + // remove node that is pointing at head, but not tail + let node_161 = Node::::get(bag_2000.bag_upper, &161).unwrap(); + bag_2000.remove_node(&node_161); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 181, 191]); + assert_ok!(bag_2000.sanity_check()); + + // remove node that is pointing at tail, but not head + let node_181 = Node::::get(bag_2000.bag_upper, &181).unwrap(); + bag_2000.remove_node(&node_181); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 191]); + assert_ok!(bag_2000.sanity_check()); + + // state of all bags is as expected + assert_eq!(get_bags(), vec![(10, vec![81]), (2_000, vec![91, 171, 191])]); + }); + } + + #[test] + fn remove_node_bad_paths_documented() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + // removing a node that is in the bag but has the wrong upper works. + + let bad_upper_node_11 = Node:: { + voter: Voter::<_>::validator(11), + prev: None, + next: Some(21), + bag_upper: 10, // should be 1_000 + }; + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.remove_node(&bad_upper_node_11); + bag_1000.put(); + + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![21, 101])]); + let bag_1000 = Bag::::get(1_000).unwrap(); + assert_ok!(bag_1000.sanity_check()); + assert_eq!(bag_1000.head, Some(21)); + assert_eq!(bag_1000.tail, Some(101)); + }); + + ExtBuilder::default().build_and_execute_without_check_count(|| { + // removing a node that is in another bag, will mess up the + // other bag. + + let node_101 = Node::::get(1_000, &101).unwrap(); + let mut bag_10 = Bag::::get(10).unwrap(); + bag_10.remove_node(&node_101); // node_101 is in bag 1_000 + bag_10.put(); + + // the node was removed from its actual bag, bag_1000. + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21])]); + + // the bag removed was called on is ok. + let bag_10 = Bag::::get(10).unwrap(); + assert_eq!(bag_10.tail, Some(31)); + assert_eq!(bag_10.head, Some(31)); + + // but the bag that the node belonged to is in an invalid state + let bag_1000 = Bag::::get(1_000).unwrap(); + // because it still has the removed node as its tail. + assert_eq!(bag_1000.tail, Some(101)); + assert_eq!(bag_1000.head, Some(11)); + assert_ok!(bag_1000.sanity_check()); + }); } } -// TODO: I've created simpler versions of these tests above. We can probably remove the ones below -// now. Peter was likely not very familiar with the staking mock and he came up with these rather -// complicated test setups. Please see my versions above, we can test the same properties, easily, -// without the need to alter the stakers so much. -/* #[cfg(test)] -mod tests { - use frame_support::traits::Currency; - +mod voter_node { use super::*; use crate::mock::*; - const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; - - /// This tests the property that when iterating through the `VoterList`, we iterate from higher - /// bags to lower. #[test] - fn iteration_is_semi_sorted() { - use rand::seq::SliceRandom; - let mut rng = rand::thread_rng(); - - // Randomly sort the list of voters. Later we'll give each of these a stake such that it - // fits into a different bag. - let voters = { - let mut v = vec![0; GENESIS_VOTER_IDS.len()]; - v.copy_from_slice(&GENESIS_VOTER_IDS); - v.shuffle(&mut rng); - v - }; + fn voting_data_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let weight_of = Staking::weight_of_fn(); - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - // initialize the voters' deposits - let mut balance = 10; - for voter_id in voters.iter().rev() { - ::Currency::make_free_balance_be(voter_id, balance); - let controller = Staking::bonded(voter_id).unwrap(); - let mut ledger = Staking::ledger(&controller).unwrap(); - ledger.total = balance; - ledger.active = balance; - Staking::update_ledger(&controller, &ledger); - Staking::do_rebag(voter_id); - - // Increase balance to the next threshold. - balance += 10; - } + // add nominator with no targets + bond_nominator(42, 43, 1_000, vec![11]); - let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); - assert_eq!(voters, have_voters); - }); - } + // given + assert_eq!( + get_voter_list_as_voters(), + vec![ + Voter::validator(11), + Voter::validator(21), + Voter::nominator(101), + Voter::nominator(42), + Voter::validator(31), + ] + ); + assert_eq!(active_era(), 0); - /// This tests that we can `take` x voters, even if that quantity ends midway through a list. - #[test] - fn take_works() { - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - // initialize the voters' deposits - let mut balance = 0; // This will be 10 on the first loop iteration because 0 % 3 == 0 - for (idx, voter_id) in GENESIS_VOTER_IDS.iter().enumerate() { - if idx % 3 == 0 { - // This increases the balance by 10, which is the amount each threshold - // increases by. Thus this will increase the balance by 1 bag. - // - // This will create 2 bags, the lower threshold bag having - // 3 voters with balance 10, and the higher threshold bag having - // 2 voters with balance 20. - balance += 10; - } + let slashing_spans = + ::SlashingSpans::iter().collect::>(); + assert_eq!(slashing_spans.keys().len(), 0); // no pre-existing slashing spans - ::Currency::make_free_balance_be(voter_id, balance); - let controller = Staking::bonded(voter_id).unwrap(); - let mut ledger = Staking::ledger(&controller).unwrap(); - ledger.total = balance; - ledger.active = balance; - Staking::update_ledger(&controller, &ledger); - Staking::do_rebag(voter_id); - } + let node_11 = Node::::get(10, &11).unwrap(); + assert_eq!( + node_11.voting_data(&weight_of, &slashing_spans).unwrap(), + (11, 1_000, vec![11]) + ); - let bag_thresh10 = Bag::::get(10) - .unwrap() - .iter() - .map(|node| node.voter.id) - .collect::>(); - assert_eq!(bag_thresh10, vec![11, 21, 31]); + // getting data for a nominators with 0 slashed targets + let node_101 = Node::::get(1_000, &101).unwrap(); + assert_eq!( + node_101.voting_data(&weight_of, &slashing_spans).unwrap(), + (101, 500, vec![11, 21]) + ); + let node_42 = Node::::get(10, &42).unwrap(); + assert_eq!( + node_42.voting_data(&weight_of, &slashing_spans).unwrap(), + (42, 1_000, vec![11]) + ); - let bag_thresh20 = Bag::::get(20) - .unwrap() - .iter() - .map(|node| node.voter.id) - .collect::>(); - assert_eq!(bag_thresh20, vec![41, 101]); + // roll ahead an era so any slashes will be after the previous nominations + start_active_era(1); - let voters: Vec<_> = VoterList::::iter() - // take 4/5 from [41, 101],[11, 21, 31], demonstrating that we can do a - // take that stops mid bag. - .take(4) - .map(|node| node.voter.id) - .collect(); + // when a validator gets a slash, + add_slash(&11); + let slashing_spans = + ::SlashingSpans::iter().collect::>(); + + assert_eq!(slashing_spans.keys().cloned().collect::>(), vec![11, 42, 101]); + // then its node no longer exists + assert_eq!( + get_voter_list_as_voters(), + vec![ + Voter::validator(21), + Voter::nominator(101), + Voter::nominator(42), + Voter::validator(31), + ] + ); + // and its nominators no longer have it as a target + let node_101 = Node::::get(10, &101).unwrap(); + assert_eq!( + node_101.voting_data(&weight_of, &slashing_spans), + Some((101, 475, vec![21])), + ); - assert_eq!(voters, vec![41, 101, 11, 21]); + let node_42 = Node::::get(10, &42).unwrap(); + assert_eq!( + node_42.voting_data(&weight_of, &slashing_spans), + None, // no voting data since its 1 target has been slashed since nominating + ); }); } #[test] - fn storage_is_cleaned_up_as_voters_are_removed() { - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - // Initialize voters deposits so there are 5 bags with one voter each. - let mut balance = 10; - for voter_id in GENESIS_VOTER_IDS.iter() { - ::Currency::make_free_balance_be(voter_id, balance); - let controller = Staking::bonded(voter_id).unwrap(); - let mut ledger = Staking::ledger(&controller).unwrap(); - ledger.total = balance; - ledger.active = balance; - Staking::update_ledger(&controller, &ledger); - Staking::do_rebag(voter_id); - - // Increase balance to the next threshold. - balance += 10; - } + fn is_misplaced_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let weight_of = Staking::weight_of_fn(); + let node_31 = Node::::get(10, &31).unwrap(); - let voter_list_storage_items_eq = |mut v: Vec| { - v.sort(); - let mut voters: Vec<_> = - VoterList::::iter().map(|node| node.voter.id).collect(); - voters.sort(); - assert_eq!(voters, v); - - let mut nodes: Vec<_> = - ::VoterNodes::iter_keys().collect(); - nodes.sort(); - assert_eq!(nodes, v); - - let mut flat_bags: Vec<_> = ::VoterBags::iter() - // We always get the bag with the Bag getter because the bag_upper - // is only initialized in the getter. - .flat_map(|(key, _bag)| Bag::::get(key).unwrap().iter()) - .map(|node| node.voter.id) - .collect(); - flat_bags.sort(); - assert_eq!(flat_bags, v); - - let mut bags_for: Vec<_> = - ::VoterBagFor::iter_keys().collect(); - bags_for.sort(); - assert_eq!(bags_for, v); - }; + // a node is properly placed if its slashable balance is in range + // of the threshold of the bag its in. + assert_eq!(Staking::slashable_balance_of(&31), 1); + assert!(!node_31.is_misplaced(&weight_of)); - let genesis_voters = vec![101, 41, 31, 21, 11]; - voter_list_storage_items_eq(genesis_voters); - assert_eq!(::CounterForVoters::get(), 5); + // and will become misplaced if its slashable balance does not + // correspond to the bag it is in. + set_ledger_and_free_balance(&31, 11); - // Remove 1 voter, - VoterList::::remove(&101); - let remaining_voters = vec![41, 31, 21, 11]; - // and assert they have been cleaned up. - voter_list_storage_items_eq(remaining_voters.clone()); - assert_eq!(::CounterForVoters::get(), 4); - - // Now remove the remaining voters so we have 0 left, - remaining_voters.iter().for_each(|v| VoterList::::remove(v)); - // and assert all of them have been cleaned up. - voter_list_storage_items_eq(vec![]); - assert_eq!(::CounterForVoters::get(), 0); - - - // TODO bags do not get cleaned up from storages - // - is this ok? I assume its ok if this is not cleaned just because voters are removed - // but it should be cleaned up if we migrate thresholds - assert_eq!(::VoterBags::iter().collect::>().len(), 6); - // and the voter list has no one in it. - assert_eq!(VoterList::::iter().collect::>().len(), 0); - assert_eq!(::VoterBagFor::iter().collect::>().len(), 0); - assert_eq!(::VoterNodes::iter().collect::>().len(), 0); + assert_eq!(Staking::slashable_balance_of(&31), 11); + assert!(node_31.is_misplaced(&weight_of)); }); } } -*/ From da4814c234a5a56fe08902c002ee69d7f30996f4 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 28 Jul 2021 10:40:49 -0700 Subject: [PATCH 079/241] Try prepare for master merge --- frame/staking/src/benchmarking.rs | 14 +- frame/staking/src/lib.rs | 2904 +--------------------------- frame/staking/src/migrations.rs | 117 ++ frame/staking/src/mock.rs | 7 +- frame/staking/src/pallet/impls.rs | 1158 +++++++++++ frame/staking/src/pallet/mod.rs | 1698 ++++++++++++++++ frame/staking/src/slashing.rs | 2 +- frame/staking/src/testing_utils.rs | 5 + frame/staking/src/tests.rs | 20 +- frame/staking/src/voter_bags.rs | 7 +- 10 files changed, 3024 insertions(+), 2908 deletions(-) create mode 100644 frame/staking/src/migrations.rs create mode 100644 frame/staking/src/pallet/impls.rs create mode 100644 frame/staking/src/pallet/mod.rs diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index e817cacda6f5b..ff4a8ba986b00 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -21,11 +21,23 @@ use super::*; use crate::Pallet as Staking; use testing_utils::*; +use frame_support::{ + pallet_prelude::*, + traits::{Currency, Get, Imbalance}, +}; +use sp_runtime::{ + traits::{CheckedSub, StaticLookup, Zero}, + Perbill, Percent, +}; +use sp_staking::SessionIndex; +use sp_std::prelude::*; + +use crate::voter_bags::VoterList; pub use frame_benchmarking::{ account, benchmarks, impl_benchmark_test_suite, whitelist_account, whitelisted_caller, }; use frame_system::RawOrigin; -use sp_runtime::traits::One; +use sp_runtime::traits::{Bounded, One}; const SEED: u32 = 0; const MAX_SPANS: u32 = 100; diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 6121d3a3affa8..993fe349a1045 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -284,43 +284,33 @@ pub(crate) mod mock; mod tests; pub mod inflation; +pub mod migrations; pub mod slashing; pub mod voter_bags; pub mod weights; +mod pallet; + use codec::{Decode, Encode, HasCompact}; -use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; +use frame_election_provider_support::VoteWeight; use frame_support::{ - pallet_prelude::*, - traits::{ - Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, Imbalance, - LockIdentifier, LockableCurrency, OnUnbalanced, UnixTime, WithdrawReasons, - }, - weights::{ - constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}, - Weight, WithPostDispatchInfo, - }, + traits::{Currency, Get}, + weights::Weight, }; -use frame_system::{ensure_root, ensure_signed, offchain::SendTransactionTypes, pallet_prelude::*}; -pub use pallet::*; -use pallet_session::historical; use sp_runtime::{ curve::PiecewiseLinear, - traits::{ - AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, - StaticLookup, Zero, - }, - DispatchError, Perbill, Percent, RuntimeDebug, + traits::{AtLeast32BitUnsigned, Convert, Saturating, Zero}, + Perbill, RuntimeDebug, }; use sp_staking::{ - offence::{Offence, OffenceDetails, OffenceError, OnOffenceHandler, ReportOffence}, + offence::{Offence, OffenceError, ReportOffence}, SessionIndex, }; -use sp_std::{collections::btree_map::BTreeMap, convert::From, prelude::*, result}; -use voter_bags::{VoterList, VoterType}; +use sp_std::{collections::btree_map::BTreeMap, convert::From, prelude::*}; pub use weights::WeightInfo; -const STAKING_ID: LockIdentifier = *b"staking "; +pub use pallet::{pallet::*, *}; + pub(crate) const LOG_TARGET: &'static str = "runtime::staking"; // syntactic sugar for logging. @@ -334,8 +324,6 @@ macro_rules! log { }; } -pub const MAX_UNLOCKING_CHUNKS: usize = 32; - /// Counter for the number of eras that have passed. pub type EraIndex = u32; @@ -757,2737 +745,6 @@ impl Default for Releases { } } -pub mod migrations { - use super::*; - - pub mod v8 { - use super::*; - - pub fn pre_migrate() -> Result<(), &'static str> { - ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); - ensure!( - VoterList::::iter().count() == 0, - "voter list already exists" - ); - Ok(()) - } - - pub fn migrate() -> Weight { - log!(info, "Migrating staking to Releases::V8_0_0"); - - let migrated = VoterList::::regenerate(); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - - StorageVersion::::put(Releases::V8_0_0); - log!( - info, - "Completed staking migration to Releases::V8_0_0 with {} voters migrated", - migrated, - ); - - T::WeightInfo::regenerate( - CounterForValidators::::get(), - CounterForNominators::::get(), - ) - .saturating_add(T::DbWeight::get().reads(2)) - } - } - - pub mod v7 { - use super::*; - - pub fn pre_migrate() -> Result<(), &'static str> { - assert!( - CounterForValidators::::get().is_zero(), - "CounterForValidators already set." - ); - assert!( - CounterForNominators::::get().is_zero(), - "CounterForNominators already set." - ); - assert!(StorageVersion::::get() == Releases::V6_0_0); - Ok(()) - } - - pub fn migrate() -> Weight { - log!(info, "Migrating staking to Releases::V7_0_0"); - let validator_count = Validators::::iter().count() as u32; - let nominator_count = Nominators::::iter().count() as u32; - - CounterForValidators::::put(validator_count); - CounterForNominators::::put(nominator_count); - - StorageVersion::::put(Releases::V7_0_0); - log!(info, "Completed staking migration to Releases::V7_0_0"); - - T::DbWeight::get() - .reads_writes(validator_count.saturating_add(nominator_count).into(), 2) - } - } - - pub mod v6 { - use super::*; - use frame_support::{generate_storage_alias, traits::Get, weights::Weight}; - - // NOTE: value type doesn't matter, we just set it to () here. - generate_storage_alias!(Staking, SnapshotValidators => Value<()>); - generate_storage_alias!(Staking, SnapshotNominators => Value<()>); - generate_storage_alias!(Staking, QueuedElected => Value<()>); - generate_storage_alias!(Staking, QueuedScore => Value<()>); - generate_storage_alias!(Staking, EraElectionStatus => Value<()>); - generate_storage_alias!(Staking, IsCurrentSessionFinal => Value<()>); - - /// check to execute prior to migration. - pub fn pre_migrate() -> Result<(), &'static str> { - // these may or may not exist. - log!(info, "SnapshotValidators.exits()? {:?}", SnapshotValidators::exists()); - log!(info, "SnapshotNominators.exits()? {:?}", SnapshotNominators::exists()); - log!(info, "QueuedElected.exits()? {:?}", QueuedElected::exists()); - log!(info, "QueuedScore.exits()? {:?}", QueuedScore::exists()); - // these must exist. - assert!( - IsCurrentSessionFinal::exists(), - "IsCurrentSessionFinal storage item not found!" - ); - assert!(EraElectionStatus::exists(), "EraElectionStatus storage item not found!"); - Ok(()) - } - - /// Migrate storage to v6. - pub fn migrate() -> Weight { - log!(info, "Migrating staking to Releases::V6_0_0"); - - SnapshotValidators::kill(); - SnapshotNominators::kill(); - QueuedElected::kill(); - QueuedScore::kill(); - EraElectionStatus::kill(); - IsCurrentSessionFinal::kill(); - - StorageVersion::::put(Releases::V6_0_0); - log!(info, "Done."); - T::DbWeight::get().writes(6 + 1) - } - } -} - -#[frame_support::pallet] -pub mod pallet { - use super::*; - - #[pallet::pallet] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: frame_system::Config + SendTransactionTypes> { - /// The staking balance. - type Currency: LockableCurrency; - - /// Time used for computing era duration. - /// - /// It is guaranteed to start being called from the first `on_finalize`. Thus value at genesis - /// is not used. - type UnixTime: UnixTime; - - /// Convert a balance into a number used for election calculation. This must fit into a `u64` - /// but is allowed to be sensibly lossy. The `u64` is used to communicate with the - /// [`sp_npos_elections`] crate which accepts u64 numbers and does operations in 128. - /// Consequently, the backward convert is used convert the u128s from sp-elections back to a - /// [`BalanceOf`]. - type CurrencyToVote: CurrencyToVote>; - - /// Something that provides the election functionality. - type ElectionProvider: frame_election_provider_support::ElectionProvider< - Self::AccountId, - Self::BlockNumber, - // we only accept an election provider that has staking as data provider. - DataProvider = Pallet, - >; - - /// Something that provides the election functionality at genesis. - type GenesisElectionProvider: frame_election_provider_support::ElectionProvider< - Self::AccountId, - Self::BlockNumber, - DataProvider = Pallet, - >; - - /// Maximum number of nominations per nominator. - const MAX_NOMINATIONS: u32; - - /// Tokens have been minted and are unused for validator-reward. - /// See [Era payout](./index.html#era-payout). - type RewardRemainder: OnUnbalanced>; - - /// The overarching event type. - type Event: From> + IsType<::Event>; - - /// Handler for the unbalanced reduction when slashing a staker. - type Slash: OnUnbalanced>; - - /// Handler for the unbalanced increment when rewarding a staker. - type Reward: OnUnbalanced>; - - /// Number of sessions per era. - #[pallet::constant] - type SessionsPerEra: Get; - - /// Number of eras that staked funds must remain bonded for. - #[pallet::constant] - type BondingDuration: Get; - - /// Number of eras that slashes are deferred by, after computation. - /// - /// This should be less than the bonding duration. Set to 0 if slashes - /// should be applied immediately, without opportunity for intervention. - #[pallet::constant] - type SlashDeferDuration: Get; - - /// The origin which can cancel a deferred slash. Root can always do this. - type SlashCancelOrigin: EnsureOrigin; - - /// Interface for interacting with a session pallet. - type SessionInterface: self::SessionInterface; - - /// The payout for validators and the system for the current era. - /// See [Era payout](./index.html#era-payout). - type EraPayout: EraPayout>; - - /// Something that can estimate the next session change, accurately or as a best effort guess. - type NextNewSession: EstimateNextNewSession; - - /// The maximum number of nominators rewarded for each validator. - /// - /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim - /// their reward. This used to limit the i/o cost for the nominator payout. - #[pallet::constant] - type MaxNominatorRewardedPerValidator: Get; - - /// Weight information for extrinsics in this pallet. - type WeightInfo: WeightInfo; - - /// The list of thresholds separating the various voter bags. - /// - /// Voters are separated into unsorted bags according to their vote weight. This specifies - /// the thresholds separating the bags. A voter's bag is the largest bag for which the - /// voter's weight is less than or equal to its upper threshold. - /// - /// When voters are iterated, higher bags are iterated completely before lower bags. This - /// means that iteration is _semi-sorted_: voters of higher weight tend to come before - /// voters of lower weight, but peer voters within a particular bag are sorted in insertion - /// order. - /// - /// # Expressing the constant - /// - /// This constant must be sorted in strictly increasing order. Duplicate items are not - /// permitted. - /// - /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be - /// specified within the bag. For any two threshold lists, if one ends with - /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists - /// will behave identically. - /// - /// # Calculation - /// - /// It is recommended to generate the set of thresholds in a geometric series, such that - /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * - /// constant_ratio).max(threshold[k] + 1)` for all `k`. - /// - /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use - /// them, the `make-bags` feature must be enabled. - /// - /// # Examples - /// - /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, - /// and iteration is strictly in insertion order. - /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is equal to 2. - /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is approximately equal - /// to 1.248. - /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will - /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. - /// - /// # Migration - /// - /// In the event that this list ever changes, a copy of the old bags list must be retained. - /// With that `VoterList::migrate` can be called, which will perform the appropriate - /// migration. - #[pallet::constant] - type VoterBagThresholds: Get<&'static [VoteWeight]>; - } - - #[pallet::extra_constants] - impl Pallet { - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - #[allow(non_snake_case)] - fn MaxNominations() -> u32 { - T::MAX_NOMINATIONS - } - } - - #[pallet::type_value] - pub(crate) fn HistoryDepthOnEmpty() -> u32 { - 84u32 - } - - /// Number of eras to keep in history. - /// - /// Information is kept for eras in `[current_era - history_depth; current_era]`. - /// - /// Must be more than the number of eras delayed by session otherwise. I.e. active era must - /// always be in history. I.e. `active_era > current_era - history_depth` must be - /// guaranteed. - #[pallet::storage] - #[pallet::getter(fn history_depth)] - pub(crate) type HistoryDepth = StorageValue<_, u32, ValueQuery, HistoryDepthOnEmpty>; - - /// The ideal number of staking participants. - #[pallet::storage] - #[pallet::getter(fn validator_count)] - pub type ValidatorCount = StorageValue<_, u32, ValueQuery>; - - /// Minimum number of staking participants before emergency conditions are imposed. - #[pallet::storage] - #[pallet::getter(fn minimum_validator_count)] - pub type MinimumValidatorCount = StorageValue<_, u32, ValueQuery>; - - /// Any validators that may never be slashed or forcibly kicked. It's a Vec since they're - /// easy to initialize and the performance hit is minimal (we expect no more than four - /// invulnerables) and restricted to testnets. - #[pallet::storage] - #[pallet::getter(fn invulnerables)] - pub type Invulnerables = StorageValue<_, Vec, ValueQuery>; - - /// Map from all locked "stash" accounts to the controller account. - #[pallet::storage] - #[pallet::getter(fn bonded)] - pub type Bonded = StorageMap<_, Twox64Concat, T::AccountId, T::AccountId>; - - /// The minimum active bond to become and maintain the role of a nominator. - #[pallet::storage] - pub type MinNominatorBond = StorageValue<_, BalanceOf, ValueQuery>; - - /// The minimum active bond to become and maintain the role of a validator. - #[pallet::storage] - pub type MinValidatorBond = StorageValue<_, BalanceOf, ValueQuery>; - - /// Map from all (unlocked) "controller" accounts to the info regarding the staking. - #[pallet::storage] - #[pallet::getter(fn ledger)] - pub type Ledger = - StorageMap<_, Blake2_128Concat, T::AccountId, StakingLedger>>; - - /// Where the reward payment should be made. Keyed by stash. - #[pallet::storage] - #[pallet::getter(fn payee)] - pub type Payee = - StorageMap<_, Twox64Concat, T::AccountId, RewardDestination, ValueQuery>; - - /// The map from (wannabe) validator stash key to the preferences of that validator. - /// - /// When updating this storage item, you must also update the `CounterForValidators`. - #[pallet::storage] - #[pallet::getter(fn validators)] - pub type Validators = - StorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; - - /// A tracker to keep count of the number of items in the `Validators` map. - #[pallet::storage] - pub type CounterForValidators = StorageValue<_, u32, ValueQuery>; - - /// The maximum validator count before we stop allowing new validators to join. - /// - /// When this value is not set, no limits are enforced. - #[pallet::storage] - pub type MaxValidatorsCount = StorageValue<_, u32, OptionQuery>; - - /// The map from nominator stash key to the set of stash keys of all validators to nominate. - /// - /// When updating this storage item, you must also update the `CounterForNominators`. - #[pallet::storage] - #[pallet::getter(fn nominators)] - pub type Nominators = - StorageMap<_, Twox64Concat, T::AccountId, Nominations>; - - /// A tracker to keep count of the number of items in the `Nominators` map. - #[pallet::storage] - pub type CounterForNominators = StorageValue<_, u32, ValueQuery>; - - /// The maximum nominator count before we stop allowing new validators to join. - /// - /// When this value is not set, no limits are enforced. - #[pallet::storage] - pub type MaxNominatorsCount = StorageValue<_, u32, OptionQuery>; - - /// The current era index. - /// - /// This is the latest planned era, depending on how the Session pallet queues the validator - /// set, it might be active or not. - #[pallet::storage] - #[pallet::getter(fn current_era)] - pub type CurrentEra = StorageValue<_, EraIndex>; - - /// The active era information, it holds index and start. - /// - /// The active era is the era being currently rewarded. Validator set of this era must be - /// equal to [`SessionInterface::validators`]. - #[pallet::storage] - #[pallet::getter(fn active_era)] - pub type ActiveEra = StorageValue<_, ActiveEraInfo>; - - /// The session index at which the era start for the last `HISTORY_DEPTH` eras. - /// - /// Note: This tracks the starting session (i.e. session index when era start being active) - /// for the eras in `[CurrentEra - HISTORY_DEPTH, CurrentEra]`. - #[pallet::storage] - #[pallet::getter(fn eras_start_session_index)] - pub type ErasStartSessionIndex = StorageMap<_, Twox64Concat, EraIndex, SessionIndex>; - - /// Exposure of validator at era. - /// - /// This is keyed first by the era index to allow bulk deletion and then the stash account. - /// - /// Is it removed after `HISTORY_DEPTH` eras. - /// If stakers hasn't been set or has been removed then empty exposure is returned. - #[pallet::storage] - #[pallet::getter(fn eras_stakers)] - pub type ErasStakers = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - Exposure>, - ValueQuery, - >; - - /// Clipped Exposure of validator at era. - /// - /// This is similar to [`ErasStakers`] but number of nominators exposed is reduced to the - /// `T::MaxNominatorRewardedPerValidator` biggest stakers. - /// (Note: the field `total` and `own` of the exposure remains unchanged). - /// This is used to limit the i/o cost for the nominator payout. - /// - /// This is keyed fist by the era index to allow bulk deletion and then the stash account. - /// - /// Is it removed after `HISTORY_DEPTH` eras. - /// If stakers hasn't been set or has been removed then empty exposure is returned. - #[pallet::storage] - #[pallet::getter(fn eras_stakers_clipped)] - pub type ErasStakersClipped = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - Exposure>, - ValueQuery, - >; - - /// Similar to `ErasStakers`, this holds the preferences of validators. - /// - /// This is keyed first by the era index to allow bulk deletion and then the stash account. - /// - /// Is it removed after `HISTORY_DEPTH` eras. - // If prefs hasn't been set or has been removed then 0 commission is returned. - #[pallet::storage] - #[pallet::getter(fn eras_validator_prefs)] - pub type ErasValidatorPrefs = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - ValidatorPrefs, - ValueQuery, - >; - - /// The total validator era payout for the last `HISTORY_DEPTH` eras. - /// - /// Eras that haven't finished yet or has been removed doesn't have reward. - #[pallet::storage] - #[pallet::getter(fn eras_validator_reward)] - pub type ErasValidatorReward = StorageMap<_, Twox64Concat, EraIndex, BalanceOf>; - - /// Rewards for the last `HISTORY_DEPTH` eras. - /// If reward hasn't been set or has been removed then 0 reward is returned. - #[pallet::storage] - #[pallet::getter(fn eras_reward_points)] - pub type ErasRewardPoints = - StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; - - /// The total amount staked for the last `HISTORY_DEPTH` eras. - /// If total hasn't been set or has been removed then 0 stake is returned. - #[pallet::storage] - #[pallet::getter(fn eras_total_stake)] - pub type ErasTotalStake = - StorageMap<_, Twox64Concat, EraIndex, BalanceOf, ValueQuery>; - - /// Mode of era forcing. - #[pallet::storage] - #[pallet::getter(fn force_era)] - pub type ForceEra = StorageValue<_, Forcing, ValueQuery>; - - /// The percentage of the slash that is distributed to reporters. - /// - /// The rest of the slashed value is handled by the `Slash`. - #[pallet::storage] - #[pallet::getter(fn slash_reward_fraction)] - pub type SlashRewardFraction = StorageValue<_, Perbill, ValueQuery>; - - /// The amount of currency given to reporters of a slash event which was - /// canceled by extraordinary circumstances (e.g. governance). - #[pallet::storage] - #[pallet::getter(fn canceled_payout)] - pub type CanceledSlashPayout = StorageValue<_, BalanceOf, ValueQuery>; - - /// All unapplied slashes that are queued for later. - #[pallet::storage] - pub type UnappliedSlashes = StorageMap< - _, - Twox64Concat, - EraIndex, - Vec>>, - ValueQuery, - >; - - /// A mapping from still-bonded eras to the first session index of that era. - /// - /// Must contains information for eras for the range: - /// `[active_era - bounding_duration; active_era]` - #[pallet::storage] - pub(crate) type BondedEras = - StorageValue<_, Vec<(EraIndex, SessionIndex)>, ValueQuery>; - - /// All slashing events on validators, mapped by era to the highest slash proportion - /// and slash value of the era. - #[pallet::storage] - pub(crate) type ValidatorSlashInEra = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - (Perbill, BalanceOf), - >; - - /// All slashing events on nominators, mapped by era to the highest slash value of the era. - #[pallet::storage] - pub(crate) type NominatorSlashInEra = - StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, BalanceOf>; - - /// Slashing spans for stash accounts. - #[pallet::storage] - pub(crate) type SlashingSpans = - StorageMap<_, Twox64Concat, T::AccountId, slashing::SlashingSpans>; - - /// Records information about the maximum slash of a stash within a slashing span, - /// as well as how much reward has been paid out. - #[pallet::storage] - pub(crate) type SpanSlash = StorageMap< - _, - Twox64Concat, - (T::AccountId, slashing::SpanIndex), - slashing::SpanRecord>, - ValueQuery, - >; - - /// The earliest era for which we have a pending, unapplied slash. - #[pallet::storage] - pub(crate) type EarliestUnappliedSlash = StorageValue<_, EraIndex>; - - /// The last planned session scheduled by the session pallet. - /// - /// This is basically in sync with the call to [`pallet_session::SessionManager::new_session`]. - #[pallet::storage] - #[pallet::getter(fn current_planned_session)] - pub type CurrentPlannedSession = StorageValue<_, SessionIndex, ValueQuery>; - - /// True if network has been upgraded to this version. - /// Storage version of the pallet. - /// - /// This is set to v7.0.0 for new networks. - #[pallet::storage] - pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; - - // The next storage items collectively comprise the voter bags: a composite data structure - // designed to allow efficient iteration of the top N voters by stake, mostly. See - // `mod voter_bags` for details. - // - // In each of these items, voter bags are indexed by their upper weight threshold. - - /// How many voters are registered. - #[pallet::storage] - pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; - - /// Which bag currently contains a particular voter. - /// - /// This may not be the appropriate bag for the voter's weight if they have been rewarded or - /// slashed. - #[pallet::storage] - pub(crate) type VoterBagFor = - StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; - - /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which - /// mainly exists to store head and tail pointers to the appropriate nodes. - #[pallet::storage] - pub(crate) type VoterBags = - StorageMap<_, Twox64Concat, VoteWeight, voter_bags::Bag>; - - /// Voter nodes store links forward and back within their respective bags, the stash id, and - /// whether the voter is a validator or nominator. - /// - /// There is nothing in this map directly identifying to which bag a particular node belongs. - /// However, the `Node` data structure has helpers which can provide that information. - #[pallet::storage] - pub(crate) type VoterNodes = - StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::Node>; - - // End of voter bags data. - - /// The threshold for when users can start calling `chill_other` for other validators / nominators. - /// The threshold is compared to the actual number of validators / nominators (`CountFor*`) in - /// the system compared to the configured max (`Max*Count`). - #[pallet::storage] - pub(crate) type ChillThreshold = StorageValue<_, Percent, OptionQuery>; - - #[pallet::genesis_config] - pub struct GenesisConfig { - pub history_depth: u32, - pub validator_count: u32, - pub minimum_validator_count: u32, - pub invulnerables: Vec, - pub force_era: Forcing, - pub slash_reward_fraction: Perbill, - pub canceled_payout: BalanceOf, - pub stakers: Vec<(T::AccountId, T::AccountId, BalanceOf, StakerStatus)>, - pub min_nominator_bond: BalanceOf, - pub min_validator_bond: BalanceOf, - } - - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - GenesisConfig { - history_depth: 84u32, - validator_count: Default::default(), - minimum_validator_count: Default::default(), - invulnerables: Default::default(), - force_era: Default::default(), - slash_reward_fraction: Default::default(), - canceled_payout: Default::default(), - stakers: Default::default(), - min_nominator_bond: Default::default(), - min_validator_bond: Default::default(), - } - } - } - - #[pallet::genesis_build] - impl GenesisBuild for GenesisConfig { - fn build(&self) { - HistoryDepth::::put(self.history_depth); - ValidatorCount::::put(self.validator_count); - MinimumValidatorCount::::put(self.minimum_validator_count); - Invulnerables::::put(&self.invulnerables); - ForceEra::::put(self.force_era); - CanceledSlashPayout::::put(self.canceled_payout); - SlashRewardFraction::::put(self.slash_reward_fraction); - StorageVersion::::put(Releases::V7_0_0); - MinNominatorBond::::put(self.min_nominator_bond); - MinValidatorBond::::put(self.min_validator_bond); - - let mut num_voters: u32 = 0; - for &(ref stash, ref controller, balance, ref status) in &self.stakers { - log!( - trace, - "inserting genesis staker: {:?} => {:?} => {:?}", - stash, - balance, - status - ); - assert!( - T::Currency::free_balance(&stash) >= balance, - "Stash does not have enough balance to bond." - ); - - if let Err(why) = >::bond( - T::Origin::from(Some(stash.clone()).into()), - T::Lookup::unlookup(controller.clone()), - balance, - RewardDestination::Staked, - ) { - // TODO: later on, fix all the tests that trigger these warnings, and - // make these assertions. Genesis stakers should all be correct! - log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue - } - match status { - StakerStatus::Validator => { - if let Err(why) = >::validate( - T::Origin::from(Some(controller.clone()).into()), - Default::default(), - ) { - log!(warn, "failed to validate staker at genesis: {:?}.", why); - } else { - num_voters += 1; - } - }, - StakerStatus::Nominator(votes) => { - if let Err(why) = >::nominate( - T::Origin::from(Some(controller.clone()).into()), - votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), - ) { - log!(warn, "failed to nominate staker at genesis: {:?}.", why); - } else { - num_voters += 1; - } - }, - _ => (), - }; - } - - // all voters are inserted sanely. - assert_eq!( - CounterForVoters::::get(), - num_voters, - "not all genesis stakers were inserted into bags, something is wrong." - ); - } - } - - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - #[pallet::metadata(T::AccountId = "AccountId", BalanceOf = "Balance")] - pub enum Event { - /// The era payout has been set; the first balance is the validator-payout; the second is - /// the remainder from the maximum amount of reward. - /// \[era_index, validator_payout, remainder\] - EraPayout(EraIndex, BalanceOf, BalanceOf), - /// The staker has been rewarded by this amount. \[stash, amount\] - Reward(T::AccountId, BalanceOf), - /// One validator (and its nominators) has been slashed by the given amount. - /// \[validator, amount\] - Slash(T::AccountId, BalanceOf), - /// An old slashing report from a prior era was discarded because it could - /// not be processed. \[session_index\] - OldSlashingReportDiscarded(SessionIndex), - /// A new set of stakers was elected. - StakingElection, - /// An account has bonded this amount. \[stash, amount\] - /// - /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, - /// it will not be emitted for staking rewards when they are added to stake. - Bonded(T::AccountId, BalanceOf), - /// An account has unbonded this amount. \[stash, amount\] - Unbonded(T::AccountId, BalanceOf), - /// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance` - /// from the unlocking queue. \[stash, amount\] - Withdrawn(T::AccountId, BalanceOf), - /// A nominator has been kicked from a validator. \[nominator, stash\] - Kicked(T::AccountId, T::AccountId), - /// The election failed. No new era is planned. - StakingElectionFailed, - /// An account has stopped participating as either a validator or nominator. - /// \[stash\] - Chilled(T::AccountId), - /// Moved an account from one bag to another. \[who, from, to\]. - Rebagged(T::AccountId, VoteWeight, VoteWeight), - } - - #[pallet::error] - pub enum Error { - /// Not a controller account. - NotController, - /// Not a stash account. - NotStash, - /// Stash is already bonded. - AlreadyBonded, - /// Controller is already paired. - AlreadyPaired, - /// Targets cannot be empty. - EmptyTargets, - /// Duplicate index. - DuplicateIndex, - /// Slash record index out of bounds. - InvalidSlashIndex, - /// Can not bond with value less than minimum required. - InsufficientBond, - /// Can not schedule more unlock chunks. - NoMoreChunks, - /// Can not rebond without unlocking chunks. - NoUnlockChunk, - /// Attempting to target a stash that still has funds. - FundedTarget, - /// Invalid era to reward. - InvalidEraToReward, - /// Invalid number of nominations. - InvalidNumberOfNominations, - /// Items are not sorted and unique. - NotSortedAndUnique, - /// Rewards for this era have already been claimed for this validator. - AlreadyClaimed, - /// Incorrect previous history depth input provided. - IncorrectHistoryDepth, - /// Incorrect number of slashing spans provided. - IncorrectSlashingSpans, - /// Internal state has become somehow corrupted and the operation cannot continue. - BadState, - /// Too many nomination targets supplied. - TooManyTargets, - /// A nomination target was supplied that was blocked or otherwise not a validator. - BadTarget, - /// The user has enough bond and thus cannot be chilled forcefully by an external person. - CannotChillOther, - /// There are too many nominators in the system. Governance needs to adjust the staking settings - /// to keep things safe for the runtime. - TooManyNominators, - /// There are too many validators in the system. Governance needs to adjust the staking settings - /// to keep things safe for the runtime. - TooManyValidators, - } - - #[pallet::hooks] - impl Hooks> for Pallet { - fn on_runtime_upgrade() -> Weight { - if StorageVersion::::get() == Releases::V6_0_0 { - migrations::v7::migrate::() - } else { - T::DbWeight::get().reads(1) - } - } - - #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result<(), &'static str> { - if StorageVersion::::get() == Releases::V6_0_0 { - migrations::v7::pre_migrate::() - } else { - Ok(()) - } - } - - fn on_initialize(_now: BlockNumberFor) -> Weight { - // just return the weight of the on_finalize. - T::DbWeight::get().reads(1) - } - - fn on_finalize(_n: BlockNumberFor) { - // Set the start of the first era. - if let Some(mut active_era) = Self::active_era() { - if active_era.start.is_none() { - let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); - active_era.start = Some(now_as_millis_u64); - // This write only ever happens once, we don't include it in the weight in general - ActiveEra::::put(active_era); - } - } - // `on_finalize` weight is tracked in `on_initialize` - } - - fn integrity_test() { - sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| { - assert!( - T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, - "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", - T::SlashDeferDuration::get(), - T::BondingDuration::get(), - ); - - assert!( - T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "Voter bag thresholds must strictly increase", - ); - - assert!( - { - let existential_weight = voter_bags::existential_weight::(); - T::VoterBagThresholds::get() - .first() - .map(|&lowest_threshold| lowest_threshold >= existential_weight) - .unwrap_or(true) - }, - "Smallest bag should not be smaller than existential weight", - ); - }); - } - } - } - - #[pallet::call] - impl Pallet { - /// Take the origin account as a stash and lock up `value` of its balance. `controller` will - /// be the account that controls it. - /// - /// `value` must be more than the `minimum_balance` specified by `T::Currency`. - /// - /// The dispatch origin for this call must be _Signed_ by the stash account. - /// - /// Emits `Bonded`. - /// # - /// - Independent of the arguments. Moderate complexity. - /// - O(1). - /// - Three extra DB entries. - /// - /// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned - /// unless the `origin` falls below _existential deposit_ and gets removed as dust. - /// ------------------ - /// # - #[pallet::weight(T::WeightInfo::bond())] - pub fn bond( - origin: OriginFor, - controller: ::Source, - #[pallet::compact] value: BalanceOf, - payee: RewardDestination, - ) -> DispatchResult { - let stash = ensure_signed(origin)?; - - if >::contains_key(&stash) { - Err(Error::::AlreadyBonded)? - } - - let controller = T::Lookup::lookup(controller)?; - - if >::contains_key(&controller) { - Err(Error::::AlreadyPaired)? - } - - // Reject a bond which is considered to be _dust_. - if value < T::Currency::minimum_balance() { - Err(Error::::InsufficientBond)? - } - - frame_system::Pallet::::inc_consumers(&stash).map_err(|_| Error::::BadState)?; - - // You're auto-bonded forever, here. We might improve this by only bonding when - // you actually validate/nominate and remove once you unbond __everything__. - >::insert(&stash, &controller); - >::insert(&stash, payee); - - let current_era = CurrentEra::::get().unwrap_or(0); - let history_depth = Self::history_depth(); - let last_reward_era = current_era.saturating_sub(history_depth); - - let stash_balance = T::Currency::free_balance(&stash); - let value = value.min(stash_balance); - Self::deposit_event(Event::::Bonded(stash.clone(), value)); - let item = StakingLedger { - stash, - total: value, - active: value, - unlocking: vec![], - claimed_rewards: (last_reward_era..current_era).collect(), - }; - Self::update_ledger(&controller, &item); - Ok(()) - } - - /// Add some extra amount that have appeared in the stash `free_balance` into the balance up - /// for staking. - /// - /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. - /// - /// Use this if there are additional funds in your stash account that you wish to bond. - /// Unlike [`bond`](Self::bond) or [`unbond`](Self::unbond) this function does not impose any limitation - /// on the amount that can be added. - /// - /// Emits `Bonded`. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - O(1). - /// # - #[pallet::weight(T::WeightInfo::bond_extra())] - pub fn bond_extra( - origin: OriginFor, - #[pallet::compact] max_additional: BalanceOf, - ) -> DispatchResult { - let stash = ensure_signed(origin)?; - - let controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; - let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - - let stash_balance = T::Currency::free_balance(&stash); - if let Some(extra) = stash_balance.checked_sub(&ledger.total) { - let extra = extra.min(max_additional); - ledger.total += extra; - ledger.active += extra; - // Last check: the new active amount of ledger must be more than ED. - ensure!( - ledger.active >= T::Currency::minimum_balance(), - Error::::InsufficientBond - ); - - Self::deposit_event(Event::::Bonded(stash.clone(), extra)); - Self::update_ledger(&controller, &ledger); - Self::do_rebag(&stash); - } - Ok(()) - } - - /// Schedule a portion of the stash to be unlocked ready for transfer out after the bond - /// period ends. If this leaves an amount actively bonded less than - /// T::Currency::minimum_balance(), then it is increased to the full amount. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// Once the unlock period is done, you can call `withdraw_unbonded` to actually move - /// the funds out of management ready for transfer. - /// - /// No more than a limited number of unlocking chunks (see `MAX_UNLOCKING_CHUNKS`) - /// can co-exists at the same time. In that case, [`Call::withdraw_unbonded`] need - /// to be called first to remove some of the chunks (if possible). - /// - /// If a user encounters the `InsufficientBond` error when calling this extrinsic, - /// they should call `chill` first in order to free up their bonded funds. - /// - /// Emits `Unbonded`. - /// - /// See also [`Call::withdraw_unbonded`]. - #[pallet::weight(T::WeightInfo::unbond())] - pub fn unbond( - origin: OriginFor, - #[pallet::compact] value: BalanceOf, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks); - - let mut value = value.min(ledger.active); - - if !value.is_zero() { - ledger.active -= value; - - // Avoid there being a dust balance left in the staking system. - if ledger.active < T::Currency::minimum_balance() { - value += ledger.active; - ledger.active = Zero::zero(); - } - - let min_active_bond = if Nominators::::contains_key(&ledger.stash) { - MinNominatorBond::::get() - } else if Validators::::contains_key(&ledger.stash) { - MinValidatorBond::::get() - } else { - Zero::zero() - }; - - // Make sure that the user maintains enough active bond for their role. - // If a user runs into this error, they should chill first. - ensure!(ledger.active >= min_active_bond, Error::::InsufficientBond); - - // Note: in case there is no current era it is fine to bond one era more. - let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); - ledger.unlocking.push(UnlockChunk { value, era }); - Self::update_ledger(&controller, &ledger); - Self::do_rebag(&ledger.stash); - Self::deposit_event(Event::::Unbonded(ledger.stash, value)); - } - Ok(()) - } - - /// Remove any unlocked chunks from the `unlocking` queue from our management. - /// - /// This essentially frees up that balance to be used by the stash account to do - /// whatever it wants. - /// - /// The dispatch origin for this call must be _Signed_ by the controller. - /// - /// Emits `Withdrawn`. - /// - /// See also [`Call::unbond`]. - /// - /// # - /// Complexity O(S) where S is the number of slashing spans to remove - /// NOTE: Weight annotation is the kill scenario, we refund otherwise. - /// # - #[pallet::weight(T::WeightInfo::withdraw_unbonded_kill(*num_slashing_spans))] - pub fn withdraw_unbonded( - origin: OriginFor, - num_slashing_spans: u32, - ) -> DispatchResultWithPostInfo { - let controller = ensure_signed(origin)?; - let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let (stash, old_total) = (ledger.stash.clone(), ledger.total); - if let Some(current_era) = Self::current_era() { - ledger = ledger.consolidate_unlocked(current_era) - } - - let post_info_weight = - if ledger.unlocking.is_empty() && ledger.active < T::Currency::minimum_balance() { - // This account must have called `unbond()` with some value that caused the active - // portion to fall below existential deposit + will have no more unlocking chunks - // left. We can now safely remove all staking-related information. - Self::kill_stash(&stash, num_slashing_spans)?; - // Remove the lock. - T::Currency::remove_lock(STAKING_ID, &stash); - // This is worst case scenario, so we use the full weight and return None - None - } else { - // This was the consequence of a partial unbond. just update the ledger and move on. - Self::update_ledger(&controller, &ledger); - - // This is only an update, so we use less overall weight. - Some(T::WeightInfo::withdraw_unbonded_update(num_slashing_spans)) - }; - - // `old_total` should never be less than the new total because - // `consolidate_unlocked` strictly subtracts balance. - if ledger.total < old_total { - // Already checked that this won't overflow by entry condition. - let value = old_total - ledger.total; - Self::deposit_event(Event::::Withdrawn(stash, value)); - } - - Ok(post_info_weight.into()) - } - - /// Declare the desire to validate for the origin controller. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - #[pallet::weight(T::WeightInfo::validate())] - pub fn validate(origin: OriginFor, prefs: ValidatorPrefs) -> DispatchResult { - let controller = ensure_signed(origin)?; - - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); - let stash = &ledger.stash; - - // Only check limits if they are not already a validator. - if !Validators::::contains_key(stash) { - // If this error is reached, we need to adjust the `MinValidatorBond` and start calling `chill_other`. - // Until then, we explicitly block new validators to protect the runtime. - if let Some(max_validators) = MaxValidatorsCount::::get() { - ensure!( - CounterForValidators::::get() < max_validators, - Error::::TooManyValidators - ); - } - } - - Self::do_remove_nominator(stash); - Self::do_add_validator(stash, prefs); - Ok(()) - } - - /// Declare the desire to nominate `targets` for the origin controller. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// # - /// - The transaction's complexity is proportional to the size of `targets` (N) - /// which is capped at CompactAssignments::LIMIT (MAX_NOMINATIONS). - /// - Both the reads and writes follow a similar pattern. - /// # - #[pallet::weight(T::WeightInfo::nominate(targets.len() as u32))] - pub fn nominate( - origin: OriginFor, - targets: Vec<::Source>, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.active >= MinNominatorBond::::get(), Error::::InsufficientBond); - let stash = &ledger.stash; - - // Only check limits if they are not already a nominator. - if !Nominators::::contains_key(stash) { - // If this error is reached, we need to adjust the `MinNominatorBond` and start calling `chill_other`. - // Until then, we explicitly block new nominators to protect the runtime. - if let Some(max_nominators) = MaxNominatorsCount::::get() { - ensure!( - CounterForNominators::::get() < max_nominators, - Error::::TooManyNominators - ); - } - } - - ensure!(!targets.is_empty(), Error::::EmptyTargets); - ensure!(targets.len() <= T::MAX_NOMINATIONS as usize, Error::::TooManyTargets); - - let old = Nominators::::get(stash).map_or_else(Vec::new, |x| x.targets); - - let targets = targets - .into_iter() - .map(|t| T::Lookup::lookup(t).map_err(DispatchError::from)) - .map(|n| { - n.and_then(|n| { - if old.contains(&n) || !Validators::::get(&n).blocked { - Ok(n) - } else { - Err(Error::::BadTarget.into()) - } - }) - }) - .collect::, _>>()?; - - let nominations = Nominations { - targets, - // Initial nominations are considered submitted at era 0. See `Nominations` doc - submitted_in: Self::current_era().unwrap_or(0), - suppressed: false, - }; - - Self::do_remove_validator(stash); - Self::do_add_nominator(stash, nominations); - Ok(()) - } - - /// Declare no desire to either validate or nominate. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - Contains one read. - /// - Writes are limited to the `origin` account key. - /// # - #[pallet::weight(T::WeightInfo::chill())] - pub fn chill(origin: OriginFor) -> DispatchResult { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - Self::chill_stash(&ledger.stash); - Ok(()) - } - - /// (Re-)set the payment target for a controller. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - Contains a limited number of reads. - /// - Writes are limited to the `origin` account key. - /// --------- - /// - Weight: O(1) - /// - DB Weight: - /// - Read: Ledger - /// - Write: Payee - /// # - #[pallet::weight(T::WeightInfo::set_payee())] - pub fn set_payee( - origin: OriginFor, - payee: RewardDestination, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let stash = &ledger.stash; - >::insert(stash, payee); - Ok(()) - } - - /// (Re-)set the controller of a stash. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - Contains a limited number of reads. - /// - Writes are limited to the `origin` account key. - /// ---------- - /// Weight: O(1) - /// DB Weight: - /// - Read: Bonded, Ledger New Controller, Ledger Old Controller - /// - Write: Bonded, Ledger New Controller, Ledger Old Controller - /// # - #[pallet::weight(T::WeightInfo::set_controller())] - pub fn set_controller( - origin: OriginFor, - controller: ::Source, - ) -> DispatchResult { - let stash = ensure_signed(origin)?; - let old_controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; - let controller = T::Lookup::lookup(controller)?; - if >::contains_key(&controller) { - Err(Error::::AlreadyPaired)? - } - if controller != old_controller { - >::insert(&stash, &controller); - if let Some(l) = >::take(&old_controller) { - >::insert(&controller, l); - } - } - Ok(()) - } - - /// Sets the ideal number of validators. - /// - /// The dispatch origin must be Root. - /// - /// # - /// Weight: O(1) - /// Write: Validator Count - /// # - #[pallet::weight(T::WeightInfo::set_validator_count())] - pub fn set_validator_count( - origin: OriginFor, - #[pallet::compact] new: u32, - ) -> DispatchResult { - ensure_root(origin)?; - ValidatorCount::::put(new); - Ok(()) - } - - /// Increments the ideal number of validators. - /// - /// The dispatch origin must be Root. - /// - /// # - /// Same as [`Self::set_validator_count`]. - /// # - #[pallet::weight(T::WeightInfo::set_validator_count())] - pub fn increase_validator_count( - origin: OriginFor, - #[pallet::compact] additional: u32, - ) -> DispatchResult { - ensure_root(origin)?; - ValidatorCount::::mutate(|n| *n += additional); - Ok(()) - } - - /// Scale up the ideal number of validators by a factor. - /// - /// The dispatch origin must be Root. - /// - /// # - /// Same as [`Self::set_validator_count`]. - /// # - #[pallet::weight(T::WeightInfo::set_validator_count())] - pub fn scale_validator_count(origin: OriginFor, factor: Percent) -> DispatchResult { - ensure_root(origin)?; - ValidatorCount::::mutate(|n| *n += factor * *n); - Ok(()) - } - - /// Force there to be no new eras indefinitely. - /// - /// The dispatch origin must be Root. - /// - /// # Warning - /// - /// The election process starts multiple blocks before the end of the era. - /// Thus the election process may be ongoing when this is called. In this case the - /// election will continue until the next era is triggered. - /// - /// # - /// - No arguments. - /// - Weight: O(1) - /// - Write: ForceEra - /// # - #[pallet::weight(T::WeightInfo::force_no_eras())] - pub fn force_no_eras(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - ForceEra::::put(Forcing::ForceNone); - Ok(()) - } - - /// Force there to be a new era at the end of the next session. After this, it will be - /// reset to normal (non-forced) behaviour. - /// - /// The dispatch origin must be Root. - /// - /// # Warning - /// - /// The election process starts multiple blocks before the end of the era. - /// If this is called just before a new era is triggered, the election process may not - /// have enough blocks to get a result. - /// - /// # - /// - No arguments. - /// - Weight: O(1) - /// - Write ForceEra - /// # - #[pallet::weight(T::WeightInfo::force_new_era())] - pub fn force_new_era(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - ForceEra::::put(Forcing::ForceNew); - Ok(()) - } - - /// Set the validators who cannot be slashed (if any). - /// - /// The dispatch origin must be Root. - /// - /// # - /// - O(V) - /// - Write: Invulnerables - /// # - #[pallet::weight(T::WeightInfo::set_invulnerables(invulnerables.len() as u32))] - pub fn set_invulnerables( - origin: OriginFor, - invulnerables: Vec, - ) -> DispatchResult { - ensure_root(origin)?; - >::put(invulnerables); - Ok(()) - } - - /// Force a current staker to become completely unstaked, immediately. - /// - /// The dispatch origin must be Root. - /// - /// # - /// O(S) where S is the number of slashing spans to be removed - /// Reads: Bonded, Slashing Spans, Account, Locks - /// Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Account, Locks - /// Writes Each: SpanSlash * S - /// # - #[pallet::weight(T::WeightInfo::force_unstake(*num_slashing_spans))] - pub fn force_unstake( - origin: OriginFor, - stash: T::AccountId, - num_slashing_spans: u32, - ) -> DispatchResult { - ensure_root(origin)?; - - // Remove all staking-related information. - Self::kill_stash(&stash, num_slashing_spans)?; - - // Remove the lock. - T::Currency::remove_lock(STAKING_ID, &stash); - Ok(()) - } - - /// Force there to be a new era at the end of sessions indefinitely. - /// - /// The dispatch origin must be Root. - /// - /// # Warning - /// - /// The election process starts multiple blocks before the end of the era. - /// If this is called just before a new era is triggered, the election process may not - /// have enough blocks to get a result. - /// - /// # - /// - Weight: O(1) - /// - Write: ForceEra - /// # - #[pallet::weight(T::WeightInfo::force_new_era_always())] - pub fn force_new_era_always(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - ForceEra::::put(Forcing::ForceAlways); - Ok(()) - } - - /// Cancel enactment of a deferred slash. - /// - /// Can be called by the `T::SlashCancelOrigin`. - /// - /// Parameters: era and indices of the slashes for that era to kill. - /// - /// # - /// Complexity: O(U + S) - /// with U unapplied slashes weighted with U=1000 - /// and S is the number of slash indices to be canceled. - /// - Read: Unapplied Slashes - /// - Write: Unapplied Slashes - /// # - #[pallet::weight(T::WeightInfo::cancel_deferred_slash(slash_indices.len() as u32))] - pub fn cancel_deferred_slash( - origin: OriginFor, - era: EraIndex, - slash_indices: Vec, - ) -> DispatchResult { - T::SlashCancelOrigin::ensure_origin(origin)?; - - ensure!(!slash_indices.is_empty(), Error::::EmptyTargets); - ensure!(is_sorted_and_unique(&slash_indices), Error::::NotSortedAndUnique); - - let mut unapplied = ::UnappliedSlashes::get(&era); - let last_item = slash_indices[slash_indices.len() - 1]; - ensure!((last_item as usize) < unapplied.len(), Error::::InvalidSlashIndex); - - for (removed, index) in slash_indices.into_iter().enumerate() { - let index = (index as usize) - removed; - unapplied.remove(index); - } - - ::UnappliedSlashes::insert(&era, &unapplied); - Ok(()) - } - - /// Pay out all the stakers behind a single validator for a single era. - /// - /// - `validator_stash` is the stash account of the validator. Their nominators, up to - /// `T::MaxNominatorRewardedPerValidator`, will also receive their rewards. - /// - `era` may be any era between `[current_era - history_depth; current_era]`. - /// - /// The origin of this call must be _Signed_. Any account can call this function, even if - /// it is not one of the stakers. - /// - /// # - /// - Time complexity: at most O(MaxNominatorRewardedPerValidator). - /// - Contains a limited number of reads and writes. - /// ----------- - /// N is the Number of payouts for the validator (including the validator) - /// Weight: - /// - Reward Destination Staked: O(N) - /// - Reward Destination Controller (Creating): O(N) - /// - /// NOTE: weights are assuming that payouts are made to alive stash account (Staked). - /// Paying even a dead controller is cheaper weight-wise. We don't do any refunds here. - /// # - #[pallet::weight(T::WeightInfo::payout_stakers_alive_staked( - T::MaxNominatorRewardedPerValidator::get() - ))] - pub fn payout_stakers( - origin: OriginFor, - validator_stash: T::AccountId, - era: EraIndex, - ) -> DispatchResultWithPostInfo { - ensure_signed(origin)?; - Self::do_payout_stakers(validator_stash, era) - } - - /// Rebond a portion of the stash scheduled to be unlocked. - /// - /// The dispatch origin must be signed by the controller. - /// - /// # - /// - Time complexity: O(L), where L is unlocking chunks - /// - Bounded by `MAX_UNLOCKING_CHUNKS`. - /// - Storage changes: Can't increase storage, only decrease it. - /// # - #[pallet::weight(T::WeightInfo::rebond(MAX_UNLOCKING_CHUNKS as u32))] - pub fn rebond( - origin: OriginFor, - #[pallet::compact] value: BalanceOf, - ) -> DispatchResultWithPostInfo { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(!ledger.unlocking.is_empty(), Error::::NoUnlockChunk); - - let ledger = ledger.rebond(value); - // Last check: the new active amount of ledger must be more than ED. - ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); - - Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); - Self::update_ledger(&controller, &ledger); - Self::do_rebag(&ledger.stash); - Ok(Some( - 35 * WEIGHT_PER_MICROS + - 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + - T::DbWeight::get().reads_writes(3, 2), - ) - .into()) - } - - /// Set `HistoryDepth` value. This function will delete any history information - /// when `HistoryDepth` is reduced. - /// - /// Parameters: - /// - `new_history_depth`: The new history depth you would like to set. - /// - `era_items_deleted`: The number of items that will be deleted by this dispatch. - /// This should report all the storage items that will be deleted by clearing old - /// era history. Needed to report an accurate weight for the dispatch. Trusted by - /// `Root` to report an accurate number. - /// - /// Origin must be root. - /// - /// # - /// - E: Number of history depths removed, i.e. 10 -> 7 = 3 - /// - Weight: O(E) - /// - DB Weight: - /// - Reads: Current Era, History Depth - /// - Writes: History Depth - /// - Clear Prefix Each: Era Stakers, EraStakersClipped, ErasValidatorPrefs - /// - Writes Each: ErasValidatorReward, ErasRewardPoints, ErasTotalStake, ErasStartSessionIndex - /// # - #[pallet::weight(T::WeightInfo::set_history_depth(*_era_items_deleted))] - pub fn set_history_depth( - origin: OriginFor, - #[pallet::compact] new_history_depth: EraIndex, - #[pallet::compact] _era_items_deleted: u32, - ) -> DispatchResult { - ensure_root(origin)?; - if let Some(current_era) = Self::current_era() { - HistoryDepth::::mutate(|history_depth| { - let last_kept = current_era.checked_sub(*history_depth).unwrap_or(0); - let new_last_kept = current_era.checked_sub(new_history_depth).unwrap_or(0); - for era_index in last_kept..new_last_kept { - Self::clear_era_information(era_index); - } - *history_depth = new_history_depth - }) - } - Ok(()) - } - - /// Remove all data structure concerning a staker/stash once its balance is at the minimum. - /// This is essentially equivalent to `withdraw_unbonded` except it can be called by anyone - /// and the target `stash` must have no funds left beyond the ED. - /// - /// This can be called from any origin. - /// - /// - `stash`: The stash account to reap. Its balance must be zero. - /// - /// # - /// Complexity: O(S) where S is the number of slashing spans on the account. - /// DB Weight: - /// - Reads: Stash Account, Bonded, Slashing Spans, Locks - /// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Stash Account, Locks - /// - Writes Each: SpanSlash * S - /// # - #[pallet::weight(T::WeightInfo::reap_stash(*num_slashing_spans))] - pub fn reap_stash( - _origin: OriginFor, - stash: T::AccountId, - num_slashing_spans: u32, - ) -> DispatchResult { - let at_minimum = T::Currency::total_balance(&stash) == T::Currency::minimum_balance(); - ensure!(at_minimum, Error::::FundedTarget); - Self::kill_stash(&stash, num_slashing_spans)?; - T::Currency::remove_lock(STAKING_ID, &stash); - Ok(()) - } - - /// Remove the given nominations from the calling validator. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// - `who`: A list of nominator stash accounts who are nominating this validator which - /// should no longer be nominating this validator. - /// - /// Note: Making this call only makes sense if you first set the validator preferences to - /// block any further nominations. - #[pallet::weight(T::WeightInfo::kick(who.len() as u32))] - pub fn kick( - origin: OriginFor, - who: Vec<::Source>, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let stash = &ledger.stash; - - for nom_stash in who - .into_iter() - .map(T::Lookup::lookup) - .collect::, _>>()? - .into_iter() - { - Nominators::::mutate(&nom_stash, |maybe_nom| { - if let Some(ref mut nom) = maybe_nom { - if let Some(pos) = nom.targets.iter().position(|v| v == stash) { - nom.targets.swap_remove(pos); - Self::deposit_event(Event::::Kicked( - nom_stash.clone(), - stash.clone(), - )); - } - } - }); - } - - Ok(()) - } - - /// Update the various staking limits this pallet. - /// - /// * `min_nominator_bond`: The minimum active bond needed to be a nominator. - /// * `min_validator_bond`: The minimum active bond needed to be a validator. - /// * `max_nominator_count`: The max number of users who can be a nominator at once. - /// When set to `None`, no limit is enforced. - /// * `max_validator_count`: The max number of users who can be a validator at once. - /// When set to `None`, no limit is enforced. - /// - /// Origin must be Root to call this function. - /// - /// NOTE: Existing nominators and validators will not be affected by this update. - /// to kick people under the new limits, `chill_other` should be called. - #[pallet::weight(T::WeightInfo::set_staking_limits())] - pub fn set_staking_limits( - origin: OriginFor, - min_nominator_bond: BalanceOf, - min_validator_bond: BalanceOf, - max_nominator_count: Option, - max_validator_count: Option, - threshold: Option, - ) -> DispatchResult { - ensure_root(origin)?; - MinNominatorBond::::set(min_nominator_bond); - MinValidatorBond::::set(min_validator_bond); - MaxNominatorsCount::::set(max_nominator_count); - MaxValidatorsCount::::set(max_validator_count); - ChillThreshold::::set(threshold); - Ok(()) - } - - /// Declare a `controller` to stop participating as either a validator or nominator. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_, but can be called by anyone. - /// - /// If the caller is the same as the controller being targeted, then no further checks are - /// enforced, and this function behaves just like `chill`. - /// - /// If the caller is different than the controller being targeted, the following conditions - /// must be met: - /// * A `ChillThreshold` must be set and checked which defines how close to the max - /// nominators or validators we must reach before users can start chilling one-another. - /// * A `MaxNominatorCount` and `MaxValidatorCount` must be set which is used to determine - /// how close we are to the threshold. - /// * A `MinNominatorBond` and `MinValidatorBond` must be set and checked, which determines - /// if this is a person that should be chilled because they have not met the threshold - /// bond required. - /// - /// This can be helpful if bond requirements are updated, and we need to remove old users - /// who do not satisfy these requirements. - // TODO: Maybe we can deprecate `chill` in the future. - // https://github.com/paritytech/substrate/issues/9111 - #[pallet::weight(T::WeightInfo::chill_other())] - pub fn chill_other(origin: OriginFor, controller: T::AccountId) -> DispatchResult { - // Anyone can call this function. - let caller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let stash = ledger.stash; - - // In order for one user to chill another user, the following conditions must be met: - // * A `ChillThreshold` is set which defines how close to the max nominators or - // validators we must reach before users can start chilling one-another. - // * A `MaxNominatorCount` and `MaxValidatorCount` which is used to determine how close - // we are to the threshold. - // * A `MinNominatorBond` and `MinValidatorBond` which is the final condition checked to - // determine this is a person that should be chilled because they have not met the - // threshold bond required. - // - // Otherwise, if caller is the same as the controller, this is just like `chill`. - if caller != controller { - let threshold = ChillThreshold::::get().ok_or(Error::::CannotChillOther)?; - let min_active_bond = if Nominators::::contains_key(&stash) { - let max_nominator_count = - MaxNominatorsCount::::get().ok_or(Error::::CannotChillOther)?; - let current_nominator_count = CounterForNominators::::get(); - ensure!( - threshold * max_nominator_count < current_nominator_count, - Error::::CannotChillOther - ); - MinNominatorBond::::get() - } else if Validators::::contains_key(&stash) { - let max_validator_count = - MaxValidatorsCount::::get().ok_or(Error::::CannotChillOther)?; - let current_validator_count = CounterForValidators::::get(); - ensure!( - threshold * max_validator_count < current_validator_count, - Error::::CannotChillOther - ); - MinValidatorBond::::get() - } else { - Zero::zero() - }; - - ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); - } - - Self::chill_stash(&stash); - Ok(()) - } - - /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its - /// stake that it should properly fall into a different bag than its current position. - /// - /// This will adjust its position into the appropriate bag. This will affect its position - /// among the nominator/validator set once the snapshot is prepared for the election. - /// - /// Anyone can call this function about any stash. - #[pallet::weight(T::WeightInfo::rebag())] - pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { - ensure_signed(origin)?; - Pallet::::do_rebag(&stash); - Ok(()) - } - } -} - -impl Pallet { - /// The total balance that can be slashed from a stash account as of right now. - pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { - // Weight note: consider making the stake accessible through stash. - Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() - } - - /// Internal impl of [`Self::slashable_balance_of`] that returns [`VoteWeight`]. - pub fn slashable_balance_of_vote_weight( - stash: &T::AccountId, - issuance: BalanceOf, - ) -> VoteWeight { - T::CurrencyToVote::to_vote(Self::slashable_balance_of(stash), issuance) - } - - /// Returns a closure around `slashable_balance_of_vote_weight` that can be passed around. - /// - /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is - /// important to be only used while the total issuance is not changing. - pub fn weight_of_fn() -> Box VoteWeight> { - // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still - // compile, while some types in mock fail to resolve. - let issuance = T::Currency::total_issuance(); - Box::new(move |who: &T::AccountId| -> VoteWeight { - Self::slashable_balance_of_vote_weight(who, issuance) - }) - } - - fn do_payout_stakers( - validator_stash: T::AccountId, - era: EraIndex, - ) -> DispatchResultWithPostInfo { - // Validate input data - let current_era = CurrentEra::::get().ok_or_else(|| { - Error::::InvalidEraToReward - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - })?; - let history_depth = Self::history_depth(); - ensure!( - era <= current_era && era >= current_era.saturating_sub(history_depth), - Error::::InvalidEraToReward - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - ); - - // Note: if era has no reward to be claimed, era may be future. better not to update - // `ledger.claimed_rewards` in this case. - let era_payout = >::get(&era).ok_or_else(|| { - Error::::InvalidEraToReward - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - })?; - - let controller = Self::bonded(&validator_stash).ok_or_else(|| { - Error::::NotStash.with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - })?; - let mut ledger = >::get(&controller).ok_or(Error::::NotController)?; - - ledger - .claimed_rewards - .retain(|&x| x >= current_era.saturating_sub(history_depth)); - match ledger.claimed_rewards.binary_search(&era) { - Ok(_) => Err(Error::::AlreadyClaimed - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)))?, - Err(pos) => ledger.claimed_rewards.insert(pos, era), - } - - let exposure = >::get(&era, &ledger.stash); - - // Input data seems good, no errors allowed after this point - - >::insert(&controller, &ledger); - - // Get Era reward points. It has TOTAL and INDIVIDUAL - // Find the fraction of the era reward that belongs to the validator - // Take that fraction of the eras rewards to split to nominator and validator - // - // Then look at the validator, figure out the proportion of their reward - // which goes to them and each of their nominators. - - let era_reward_points = >::get(&era); - let total_reward_points = era_reward_points.total; - let validator_reward_points = era_reward_points - .individual - .get(&ledger.stash) - .map(|points| *points) - .unwrap_or_else(|| Zero::zero()); - - // Nothing to do if they have no reward points. - if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) - } - - // This is the fraction of the total reward that the validator and the - // nominators will get. - let validator_total_reward_part = - Perbill::from_rational(validator_reward_points, total_reward_points); - - // This is how much validator + nominators are entitled to. - let validator_total_payout = validator_total_reward_part * era_payout; - - let validator_prefs = Self::eras_validator_prefs(&era, &validator_stash); - // Validator first gets a cut off the top. - let validator_commission = validator_prefs.commission; - let validator_commission_payout = validator_commission * validator_total_payout; - - let validator_leftover_payout = validator_total_payout - validator_commission_payout; - // Now let's calculate how this is split to the validator. - let validator_exposure_part = Perbill::from_rational(exposure.own, exposure.total); - let validator_staking_payout = validator_exposure_part * validator_leftover_payout; - - // We can now make total validator payout: - if let Some(imbalance) = - Self::make_payout(&ledger.stash, validator_staking_payout + validator_commission_payout) - { - Self::deposit_event(Event::::Reward(ledger.stash, imbalance.peek())); - } - - // Track the number of payout ops to nominators. Note: `WeightInfo::payout_stakers_alive_staked` - // always assumes at least a validator is paid out, so we do not need to count their payout op. - let mut nominator_payout_count: u32 = 0; - - // Lets now calculate how this is split to the nominators. - // Reward only the clipped exposures. Note this is not necessarily sorted. - for nominator in exposure.others.iter() { - let nominator_exposure_part = Perbill::from_rational(nominator.value, exposure.total); - - let nominator_reward: BalanceOf = - nominator_exposure_part * validator_leftover_payout; - // We can now make nominator payout: - if let Some(imbalance) = Self::make_payout(&nominator.who, nominator_reward) { - // Note: this logic does not count payouts for `RewardDestination::None`. - nominator_payout_count += 1; - Self::deposit_event(Event::::Reward(nominator.who.clone(), imbalance.peek())); - } - } - - debug_assert!(nominator_payout_count <= T::MaxNominatorRewardedPerValidator::get()); - Ok(Some(T::WeightInfo::payout_stakers_alive_staked(nominator_payout_count)).into()) - } - - /// Update the ledger for a controller. - /// - /// This will also update the stash lock. - fn update_ledger( - controller: &T::AccountId, - ledger: &StakingLedger>, - ) { - T::Currency::set_lock(STAKING_ID, &ledger.stash, ledger.total, WithdrawReasons::all()); - >::insert(controller, ledger); - } - - /// Chill a stash account. - fn chill_stash(stash: &T::AccountId) { - let chilled_as_validator = Self::do_remove_validator(stash); - let chilled_as_nominator = Self::do_remove_nominator(stash); - if chilled_as_validator || chilled_as_nominator { - Self::deposit_event(Event::::Chilled(stash.clone())); - } - } - - /// Actually make a payment to a staker. This uses the currency's reward function - /// to pay the right payee for the given staker account. - fn make_payout(stash: &T::AccountId, amount: BalanceOf) -> Option> { - let dest = Self::payee(stash); - match dest { - RewardDestination::Controller => Self::bonded(stash) - .and_then(|controller| Some(T::Currency::deposit_creating(&controller, amount))), - RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), - RewardDestination::Staked => Self::bonded(stash) - .and_then(|c| Self::ledger(&c).map(|l| (c, l))) - .and_then(|(controller, mut l)| { - l.active += amount; - l.total += amount; - let r = T::Currency::deposit_into_existing(stash, amount).ok(); - Self::update_ledger(&controller, &l); - r - }), - RewardDestination::Account(dest_account) => - Some(T::Currency::deposit_creating(&dest_account, amount)), - RewardDestination::None => None, - } - } - - /// Plan a new session potentially trigger a new era. - fn new_session(session_index: SessionIndex, is_genesis: bool) -> Option> { - if let Some(current_era) = Self::current_era() { - // Initial era has been set. - let current_era_start_session_index = Self::eras_start_session_index(current_era) - .unwrap_or_else(|| { - frame_support::print("Error: start_session_index must be set for current_era"); - 0 - }); - - let era_length = - session_index.checked_sub(current_era_start_session_index).unwrap_or(0); // Must never happen. - - match ForceEra::::get() { - // Will be set to `NotForcing` again if a new era has been triggered. - Forcing::ForceNew => (), - // Short circuit to `try_trigger_new_era`. - Forcing::ForceAlways => (), - // Only go to `try_trigger_new_era` if deadline reached. - Forcing::NotForcing if era_length >= T::SessionsPerEra::get() => (), - _ => { - // Either `Forcing::ForceNone`, - // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None - }, - } - - // New era. - let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() && - matches!(ForceEra::::get(), Forcing::ForceNew) - { - ForceEra::::put(Forcing::NotForcing); - } - - maybe_new_era_validators - } else { - // Set initial era. - log!(debug, "Starting the first era."); - Self::try_trigger_new_era(session_index, is_genesis) - } - } - - /// Start a session potentially starting an era. - fn start_session(start_session: SessionIndex) { - let next_active_era = Self::active_era().map(|e| e.index + 1).unwrap_or(0); - // This is only `Some` when current era has already progressed to the next era, while the - // active era is one behind (i.e. in the *last session of the active era*, or *first session - // of the new current era*, depending on how you look at it). - if let Some(next_active_era_start_session_index) = - Self::eras_start_session_index(next_active_era) - { - if next_active_era_start_session_index == start_session { - Self::start_era(start_session); - } else if next_active_era_start_session_index < start_session { - // This arm should never happen, but better handle it than to stall the staking - // pallet. - frame_support::print("Warning: A session appears to have been skipped."); - Self::start_era(start_session); - } - } - } - - /// End a session potentially ending an era. - fn end_session(session_index: SessionIndex) { - if let Some(active_era) = Self::active_era() { - if let Some(next_active_era_start_session_index) = - Self::eras_start_session_index(active_era.index + 1) - { - if next_active_era_start_session_index == session_index + 1 { - Self::end_era(active_era, session_index); - } - } - } - } - - /// * Increment `active_era.index`, - /// * reset `active_era.start`, - /// * update `BondedEras` and apply slashes. - fn start_era(start_session: SessionIndex) { - let active_era = ActiveEra::::mutate(|active_era| { - let new_index = active_era.as_ref().map(|info| info.index + 1).unwrap_or(0); - *active_era = Some(ActiveEraInfo { - index: new_index, - // Set new active era start in next `on_finalize`. To guarantee usage of `Time` - start: None, - }); - new_index - }); - - let bonding_duration = T::BondingDuration::get(); - - BondedEras::::mutate(|bonded| { - bonded.push((active_era, start_session)); - - if active_era > bonding_duration { - let first_kept = active_era - bonding_duration; - - // Prune out everything that's from before the first-kept index. - let n_to_prune = - bonded.iter().take_while(|&&(era_idx, _)| era_idx < first_kept).count(); - - // Kill slashing metadata. - for (pruned_era, _) in bonded.drain(..n_to_prune) { - slashing::clear_era_metadata::(pruned_era); - } - - if let Some(&(_, first_session)) = bonded.first() { - T::SessionInterface::prune_historical_up_to(first_session); - } - } - }); - - Self::apply_unapplied_slashes(active_era); - } - - /// Compute payout for era. - fn end_era(active_era: ActiveEraInfo, _session_index: SessionIndex) { - // Note: active_era_start can be None if end era is called during genesis config. - if let Some(active_era_start) = active_era.start { - let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); - - let era_duration = (now_as_millis_u64 - active_era_start).saturated_into::(); - let staked = Self::eras_total_stake(&active_era.index); - let issuance = T::Currency::total_issuance(); - let (validator_payout, rest) = T::EraPayout::era_payout(staked, issuance, era_duration); - - Self::deposit_event(Event::::EraPayout(active_era.index, validator_payout, rest)); - - // Set ending era reward. - >::insert(&active_era.index, validator_payout); - T::RewardRemainder::on_unbalanced(T::Currency::issue(rest)); - } - } - - /// Plan a new era. - /// - /// * Bump the current era storage (which holds the latest planned era). - /// * Store start session index for the new planned era. - /// * Clean old era information. - /// * Store staking information for the new planned era - /// - /// Returns the new validator set. - pub fn trigger_new_era( - start_session_index: SessionIndex, - exposures: Vec<(T::AccountId, Exposure>)>, - ) -> Vec { - // Increment or set current era. - let new_planned_era = CurrentEra::::mutate(|s| { - *s = Some(s.map(|s| s + 1).unwrap_or(0)); - s.unwrap() - }); - ErasStartSessionIndex::::insert(&new_planned_era, &start_session_index); - - // Clean old era information. - if let Some(old_era) = new_planned_era.checked_sub(Self::history_depth() + 1) { - Self::clear_era_information(old_era); - } - - // Set staking information for the new era. - Self::store_stakers_info(exposures, new_planned_era) - } - - /// Potentially plan a new era. - /// - /// Get election result from `T::ElectionProvider`. - /// In case election result has more than [`MinimumValidatorCount`] validator trigger a new era. - /// - /// In case a new era is planned, the new validator set is returned. - fn try_trigger_new_era( - start_session_index: SessionIndex, - is_genesis: bool, - ) -> Option> { - let (election_result, weight) = if is_genesis { - T::GenesisElectionProvider::elect().map_err(|e| { - log!(warn, "genesis election provider failed due to {:?}", e); - Self::deposit_event(Event::StakingElectionFailed); - }) - } else { - T::ElectionProvider::elect().map_err(|e| { - log!(warn, "election provider failed due to {:?}", e); - Self::deposit_event(Event::StakingElectionFailed); - }) - } - .ok()?; - - >::register_extra_weight_unchecked( - weight, - frame_support::weights::DispatchClass::Mandatory, - ); - - let exposures = Self::collect_exposures(election_result); - - if (exposures.len() as u32) < Self::minimum_validator_count().max(1) { - // Session will panic if we ever return an empty validator set, thus max(1) ^^. - match CurrentEra::::get() { - Some(current_era) if current_era > 0 => log!( - warn, - "chain does not have enough staking candidates to operate for era {:?} ({} \ - elected, minimum is {})", - CurrentEra::::get().unwrap_or(0), - exposures.len(), - Self::minimum_validator_count(), - ), - None => { - // The initial era is allowed to have no exposures. - // In this case the SessionManager is expected to choose a sensible validator - // set. - // TODO: this should be simplified #8911 - CurrentEra::::put(0); - ErasStartSessionIndex::::insert(&0, &start_session_index); - }, - _ => (), - } - - Self::deposit_event(Event::StakingElectionFailed); - return None - } - - Self::deposit_event(Event::StakingElection); - Some(Self::trigger_new_era(start_session_index, exposures)) - } - - /// Process the output of the election. - /// - /// Store staking information for the new planned era - pub fn store_stakers_info( - exposures: Vec<(T::AccountId, Exposure>)>, - new_planned_era: EraIndex, - ) -> Vec { - let elected_stashes = exposures.iter().cloned().map(|(x, _)| x).collect::>(); - - // Populate stakers, exposures, and the snapshot of validator prefs. - let mut total_stake: BalanceOf = Zero::zero(); - exposures.into_iter().for_each(|(stash, exposure)| { - total_stake = total_stake.saturating_add(exposure.total); - >::insert(new_planned_era, &stash, &exposure); - - let mut exposure_clipped = exposure; - let clipped_max_len = T::MaxNominatorRewardedPerValidator::get() as usize; - if exposure_clipped.others.len() > clipped_max_len { - exposure_clipped.others.sort_by(|a, b| a.value.cmp(&b.value).reverse()); - exposure_clipped.others.truncate(clipped_max_len); - } - >::insert(&new_planned_era, &stash, exposure_clipped); - }); - - // Insert current era staking information - >::insert(&new_planned_era, total_stake); - - // Collect the pref of all winners. - for stash in &elected_stashes { - let pref = Self::validators(stash); - >::insert(&new_planned_era, stash, pref); - } - - if new_planned_era > 0 { - log!( - info, - "new validator set of size {:?} has been processed for era {:?}", - elected_stashes.len(), - new_planned_era, - ); - } - - elected_stashes - } - - /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a - /// [`Exposure`]. - fn collect_exposures( - supports: Supports, - ) -> Vec<(T::AccountId, Exposure>)> { - let total_issuance = T::Currency::total_issuance(); - let to_currency = |e: frame_election_provider_support::ExtendedBalance| { - T::CurrencyToVote::to_currency(e, total_issuance) - }; - - supports - .into_iter() - .map(|(validator, support)| { - // Build `struct exposure` from `support`. - let mut others = Vec::with_capacity(support.voters.len()); - let mut own: BalanceOf = Zero::zero(); - let mut total: BalanceOf = Zero::zero(); - support - .voters - .into_iter() - .map(|(nominator, weight)| (nominator, to_currency(weight))) - .for_each(|(nominator, stake)| { - if nominator == validator { - own = own.saturating_add(stake); - } else { - others.push(IndividualExposure { who: nominator, value: stake }); - } - total = total.saturating_add(stake); - }); - - let exposure = Exposure { own, others, total }; - (validator, exposure) - }) - .collect::)>>() - } - - /// Remove all associated data of a stash account from the staking system. - /// - /// Assumes storage is upgraded before calling. - /// - /// This is called: - /// - after a `withdraw_unbonded()` call that frees all of a stash's bonded balance. - /// - through `reap_stash()` if the balance has fallen to zero (through slashing). - fn kill_stash(stash: &T::AccountId, num_slashing_spans: u32) -> DispatchResult { - let controller = >::get(stash).ok_or(Error::::NotStash)?; - - slashing::clear_stash_metadata::(stash, num_slashing_spans)?; - - >::remove(stash); - >::remove(&controller); - - >::remove(stash); - Self::do_remove_validator(stash); - Self::do_remove_nominator(stash); - - frame_system::Pallet::::dec_consumers(stash); - - Ok(()) - } - - /// Clear all era information for given era. - fn clear_era_information(era_index: EraIndex) { - >::remove_prefix(era_index, None); - >::remove_prefix(era_index, None); - >::remove_prefix(era_index, None); - >::remove(era_index); - >::remove(era_index); - >::remove(era_index); - ErasStartSessionIndex::::remove(era_index); - } - - /// Apply previously-unapplied slashes on the beginning of a new era, after a delay. - fn apply_unapplied_slashes(active_era: EraIndex) { - let slash_defer_duration = T::SlashDeferDuration::get(); - ::EarliestUnappliedSlash::mutate(|earliest| { - if let Some(ref mut earliest) = earliest { - let keep_from = active_era.saturating_sub(slash_defer_duration); - for era in (*earliest)..keep_from { - let era_slashes = ::UnappliedSlashes::take(&era); - for slash in era_slashes { - slashing::apply_slash::(slash); - } - } - - *earliest = (*earliest).max(keep_from) - } - }) - } - - /// Add reward points to validators using their stash account ID. - /// - /// Validators are keyed by stash account ID and must be in the current elected set. - /// - /// For each element in the iterator the given number of points in u32 is added to the - /// validator, thus duplicates are handled. - /// - /// At the end of the era each the total payout will be distributed among validator - /// relatively to their points. - /// - /// COMPLEXITY: Complexity is `number_of_validator_to_reward x current_elected_len`. - pub fn reward_by_ids(validators_points: impl IntoIterator) { - if let Some(active_era) = Self::active_era() { - >::mutate(active_era.index, |era_rewards| { - for (validator, points) in validators_points.into_iter() { - *era_rewards.individual.entry(validator).or_default() += points; - era_rewards.total += points; - } - }); - } - } - - /// Ensures that at the end of the current session there will be a new era. - fn ensure_new_era() { - match ForceEra::::get() { - Forcing::ForceAlways | Forcing::ForceNew => (), - _ => ForceEra::::put(Forcing::ForceNew), - } - } - - #[cfg(feature = "runtime-benchmarks")] - pub fn add_era_stakers( - current_era: EraIndex, - controller: T::AccountId, - exposure: Exposure>, - ) { - >::insert(¤t_era, &controller, &exposure); - } - - #[cfg(feature = "runtime-benchmarks")] - pub fn set_slash_reward_fraction(fraction: Perbill) { - SlashRewardFraction::::put(fraction); - } - - /// Get all of the voters that are eligible for the npos election. - /// - /// `voter_count` imposes an implicit cap on the number of voters returned; care should be taken - /// to ensure that it is accurate. - /// - /// This will use all on-chain nominators, and all the validators will inject a self vote. - /// - /// ### Slashing - /// - /// All nominations that have been submitted before the last non-zero slash of the validator are - /// auto-chilled. - /// - /// Note that this is fairly expensive: it must iterate over the min of `maybe_max_len` and - /// `voter_count` voters. Use with care. - pub fn get_npos_voters( - maybe_max_len: Option, - voter_count: usize, - ) -> Vec> { - let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); - - let weight_of = Self::weight_of_fn(); - // collect all slashing spans into a BTreeMap for further queries. - let slashing_spans = >::iter().collect::>(); - - VoterList::::iter() - .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) - .take(wanted_voters) - .collect() - } - - /// This is a very expensive function and result should be cached versus being called multiple times. - pub fn get_npos_targets() -> Vec { - Validators::::iter().map(|(v, _)| v).collect::>() - } - - /// This function will add a nominator to the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. - /// - /// If the nominator already exists, their nominations will be updated. - /// - /// NOTE: you must ALWAYS use this function to add a nominator to the system. Any access to - /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { - if !Nominators::::contains_key(who) { - CounterForNominators::::mutate(|x| x.saturating_inc()) - } - Nominators::::insert(who, nominations); - VoterList::::insert_as(who, VoterType::Nominator); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - } - - /// This function will remove a nominator from the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. - /// - /// Returns true if `who` was removed from `Nominators`, otherwise false. - /// - /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to - /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_remove_nominator(who: &T::AccountId) -> bool { - if Nominators::::contains_key(who) { - Nominators::::remove(who); - CounterForNominators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - true - } else { - false - } - } - - /// This function will add a validator to the `Validators` storage map, and keep track of the - /// `CounterForValidators`. - /// - /// If the validator already exists, their preferences will be updated. - /// - /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to - /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { - if !Validators::::contains_key(who) { - CounterForValidators::::mutate(|x| x.saturating_inc()) - } - Validators::::insert(who, prefs); - VoterList::::insert_as(who, VoterType::Validator); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - } - - /// This function will remove a validator from the `Validators` storage map, - /// and keep track of the `CounterForValidators`. - /// - /// Returns true if `who` was removed from `Validators`, otherwise false. - /// - /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to - /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_remove_validator(who: &T::AccountId) -> bool { - if Validators::::contains_key(who) { - Validators::::remove(who); - CounterForValidators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - true - } else { - false - } - } - - /// Move a stash account from one bag to another, depositing an event on success. - /// - /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { - // if no voter at that node, don't do anything. - // the caller just wasted the fee to call this. - let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { - let weight_of = Self::weight_of_fn(); - VoterList::update_position_for(node, weight_of) - }); - if let Some((from, to)) = maybe_movement { - Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); - }; - maybe_movement - } -} - -impl - frame_election_provider_support::ElectionDataProvider> - for Pallet -{ - const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; - fn desired_targets() -> data_provider::Result<(u32, Weight)> { - Ok((Self::validator_count(), ::DbWeight::get().reads(1))) - } - - fn voters( - maybe_max_len: Option, - ) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec)>, Weight)> { - let nominator_count = CounterForNominators::::get(); - let validator_count = CounterForValidators::::get(); - let voter_count = nominator_count.saturating_add(validator_count) as usize; - - // check a few counters one last time... - debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); - debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); - debug_assert_eq!( - voter_count, - VoterList::::decode_len().unwrap_or_default(), - "voter_count must be accurate", - ); - - let slashing_span_count = >::iter().count(); - let weight = T::WeightInfo::get_npos_voters( - nominator_count, - validator_count, - slashing_span_count as u32, - ); - - Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) - } - - fn targets(maybe_max_len: Option) -> data_provider::Result<(Vec, Weight)> { - let target_count = CounterForValidators::::get() as usize; - - if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big") - } - - let weight = ::DbWeight::get().reads(target_count as u64); - Ok((Self::get_npos_targets(), weight)) - } - - fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber { - let current_era = Self::current_era().unwrap_or(0); - let current_session = Self::current_planned_session(); - let current_era_start_session_index = - Self::eras_start_session_index(current_era).unwrap_or(0); - // Number of session in the current era or the maximum session per era if reached. - let era_progress = current_session - .saturating_sub(current_era_start_session_index) - .min(T::SessionsPerEra::get()); - - let until_this_session_end = T::NextNewSession::estimate_next_new_session(now) - .0 - .unwrap_or_default() - .saturating_sub(now); - - let session_length = T::NextNewSession::average_session_length(); - - let sessions_left: T::BlockNumber = match ForceEra::::get() { - Forcing::ForceNone => Bounded::max_value(), - Forcing::ForceNew | Forcing::ForceAlways => Zero::zero(), - Forcing::NotForcing if era_progress >= T::SessionsPerEra::get() => Zero::zero(), - Forcing::NotForcing => T::SessionsPerEra::get() - .saturating_sub(era_progress) - // One session is computed in this_session_end. - .saturating_sub(1) - .into(), - }; - - now.saturating_add( - until_this_session_end.saturating_add(sessions_left.saturating_mul(session_length)), - ) - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn add_voter(voter: T::AccountId, weight: VoteWeight, targets: Vec) { - use sp_std::convert::TryFrom; - let stake = >::try_from(weight).unwrap_or_else(|_| { - panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") - }); - >::insert(voter.clone(), voter.clone()); - >::insert( - voter.clone(), - StakingLedger { - stash: voter.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_nominator(&voter, Nominations { targets, submitted_in: 0, suppressed: false }); - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn add_target(target: T::AccountId) { - let stake = MinValidatorBond::::get() * 100u32.into(); - >::insert(target.clone(), target.clone()); - >::insert( - target.clone(), - StakingLedger { - stash: target.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_validator( - &target, - ValidatorPrefs { commission: Perbill::zero(), blocked: false }, - ); - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn clear() { - >::remove_all(None); - >::remove_all(None); - >::remove_all(None); - >::remove_all(None); - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn put_snapshot( - voters: Vec<(T::AccountId, VoteWeight, Vec)>, - targets: Vec, - target_stake: Option, - ) { - use sp_std::convert::TryFrom; - targets.into_iter().for_each(|v| { - let stake: BalanceOf = target_stake - .and_then(|w| >::try_from(w).ok()) - .unwrap_or_else(|| MinNominatorBond::::get() * 100u32.into()); - >::insert(v.clone(), v.clone()); - >::insert( - v.clone(), - StakingLedger { - stash: v.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_validator( - &v, - ValidatorPrefs { commission: Perbill::zero(), blocked: false }, - ); - }); - - voters.into_iter().for_each(|(v, s, t)| { - let stake = >::try_from(s).unwrap_or_else(|_| { - panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") - }); - >::insert(v.clone(), v.clone()); - >::insert( - v.clone(), - StakingLedger { - stash: v.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_nominator( - &v, - Nominations { targets: t, submitted_in: 0, suppressed: false }, - ); - }); - } -} - -/// In this implementation `new_session(session)` must be called before `end_session(session-1)` -/// i.e. the new session must be planned before the ending of the previous session. -/// -/// Once the first new_session is planned, all session must start and then end in order, though -/// some session can lag in between the newest session planned and the latest session started. -impl pallet_session::SessionManager for Pallet { - fn new_session(new_index: SessionIndex) -> Option> { - log!(trace, "planning new session {}", new_index); - CurrentPlannedSession::::put(new_index); - Self::new_session(new_index, false) - } - fn new_session_genesis(new_index: SessionIndex) -> Option> { - log!(trace, "planning new session {} at genesis", new_index); - CurrentPlannedSession::::put(new_index); - Self::new_session(new_index, true) - } - fn start_session(start_index: SessionIndex) { - log!(trace, "starting session {}", start_index); - Self::start_session(start_index) - } - fn end_session(end_index: SessionIndex) { - log!(trace, "ending session {}", end_index); - Self::end_session(end_index) - } -} - -impl historical::SessionManager>> - for Pallet -{ - fn new_session( - new_index: SessionIndex, - ) -> Option>)>> { - >::new_session(new_index).map(|validators| { - let current_era = Self::current_era() - // Must be some as a new era has been created. - .unwrap_or(0); - - validators - .into_iter() - .map(|v| { - let exposure = Self::eras_stakers(current_era, &v); - (v, exposure) - }) - .collect() - }) - } - fn new_session_genesis( - new_index: SessionIndex, - ) -> Option>)>> { - >::new_session_genesis(new_index).map( - |validators| { - let current_era = Self::current_era() - // Must be some as a new era has been created. - .unwrap_or(0); - - validators - .into_iter() - .map(|v| { - let exposure = Self::eras_stakers(current_era, &v); - (v, exposure) - }) - .collect() - }, - ) - } - fn start_session(start_index: SessionIndex) { - >::start_session(start_index) - } - fn end_session(end_index: SessionIndex) { - >::end_session(end_index) - } -} - -/// Add reward points to block authors: -/// * 20 points to the block producer for producing a (non-uncle) block in the relay chain, -/// * 2 points to the block producer for each reference to a previously unreferenced uncle, and -/// * 1 point to the producer of each referenced uncle block. -impl pallet_authorship::EventHandler for Pallet -where - T: Config + pallet_authorship::Config + pallet_session::Config, -{ - fn note_author(author: T::AccountId) { - Self::reward_by_ids(vec![(author, 20)]) - } - fn note_uncle(author: T::AccountId, _age: T::BlockNumber) { - Self::reward_by_ids(vec![(>::author(), 2), (author, 1)]) - } -} - /// A `Convert` implementation that finds the stash of the given controller account, /// if any. pub struct StashOf(sp_std::marker::PhantomData); @@ -3514,138 +771,6 @@ impl Convert } } -/// This is intended to be used with `FilterHistoricalOffences`. -impl - OnOffenceHandler, Weight> - for Pallet -where - T: pallet_session::Config::AccountId>, - T: pallet_session::historical::Config< - FullIdentification = Exposure<::AccountId, BalanceOf>, - FullIdentificationOf = ExposureOf, - >, - T::SessionHandler: pallet_session::SessionHandler<::AccountId>, - T::SessionManager: pallet_session::SessionManager<::AccountId>, - T::ValidatorIdOf: Convert< - ::AccountId, - Option<::AccountId>, - >, -{ - fn on_offence( - offenders: &[OffenceDetails< - T::AccountId, - pallet_session::historical::IdentificationTuple, - >], - slash_fraction: &[Perbill], - slash_session: SessionIndex, - ) -> Weight { - let reward_proportion = SlashRewardFraction::::get(); - let mut consumed_weight: Weight = 0; - let mut add_db_reads_writes = |reads, writes| { - consumed_weight += T::DbWeight::get().reads_writes(reads, writes); - }; - - let active_era = { - let active_era = Self::active_era(); - add_db_reads_writes(1, 0); - if active_era.is_none() { - // This offence need not be re-submitted. - return consumed_weight - } - active_era.expect("value checked not to be `None`; qed").index - }; - let active_era_start_session_index = Self::eras_start_session_index(active_era) - .unwrap_or_else(|| { - frame_support::print("Error: start_session_index must be set for current_era"); - 0 - }); - add_db_reads_writes(1, 0); - - let window_start = active_era.saturating_sub(T::BondingDuration::get()); - - // Fast path for active-era report - most likely. - // `slash_session` cannot be in a future active era. It must be in `active_era` or before. - let slash_era = if slash_session >= active_era_start_session_index { - active_era - } else { - let eras = BondedEras::::get(); - add_db_reads_writes(1, 0); - - // Reverse because it's more likely to find reports from recent eras. - match eras.iter().rev().filter(|&&(_, ref sesh)| sesh <= &slash_session).next() { - Some(&(ref slash_era, _)) => *slash_era, - // Before bonding period. defensive - should be filtered out. - None => return consumed_weight, - } - }; - - ::EarliestUnappliedSlash::mutate(|earliest| { - if earliest.is_none() { - *earliest = Some(active_era) - } - }); - add_db_reads_writes(1, 1); - - let slash_defer_duration = T::SlashDeferDuration::get(); - - let invulnerables = Self::invulnerables(); - add_db_reads_writes(1, 0); - - for (details, slash_fraction) in offenders.iter().zip(slash_fraction) { - let (stash, exposure) = &details.offender; - - // Skip if the validator is invulnerable. - if invulnerables.contains(stash) { - continue - } - - let unapplied = slashing::compute_slash::(slashing::SlashParams { - stash, - slash: *slash_fraction, - exposure, - slash_era, - window_start, - now: active_era, - reward_proportion, - }); - - if let Some(mut unapplied) = unapplied { - let nominators_len = unapplied.others.len() as u64; - let reporters_len = details.reporters.len() as u64; - - { - let upper_bound = 1 /* Validator/NominatorSlashInEra */ + 2 /* fetch_spans */; - let rw = upper_bound + nominators_len * upper_bound; - add_db_reads_writes(rw, rw); - } - unapplied.reporters = details.reporters.clone(); - if slash_defer_duration == 0 { - // Apply right away. - slashing::apply_slash::(unapplied); - { - let slash_cost = (6, 5); - let reward_cost = (2, 2); - add_db_reads_writes( - (1 + nominators_len) * slash_cost.0 + reward_cost.0 * reporters_len, - (1 + nominators_len) * slash_cost.1 + reward_cost.1 * reporters_len, - ); - } - } else { - // Defer to end of some `slash_defer_duration` from now. - ::UnappliedSlashes::mutate(active_era, move |for_later| { - for_later.push(unapplied) - }); - add_db_reads_writes(1, 1); - } - } else { - add_db_reads_writes(4 /* fetch_spans */, 5 /* kick_out_if_recent */) - } - } - - consumed_weight - } -} - /// Filter historical offences out and only allow those from the bonding period. pub struct FilterHistoricalOffences { _inner: sp_std::marker::PhantomData<(T, R)>, @@ -3675,8 +800,3 @@ where R::is_known_offence(offenders, time_slot) } } - -/// Check that list is sorted and has no duplicates. -fn is_sorted_and_unique(list: &[u32]) -> bool { - list.windows(2).all(|w| w[0] < w[1]) -} diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs new file mode 100644 index 0000000000000..83f3ae9b4d83e --- /dev/null +++ b/frame/staking/src/migrations.rs @@ -0,0 +1,117 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and + +//! Storage migrations for the Staking pallet. + +use super::*; + +pub mod v8 { + use super::{voter_bags::VoterList, *}; + use frame_support::ensure; + + pub fn pre_migrate() -> Result<(), &'static str> { + ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); + ensure!(VoterList::::iter().count() == 0, "voter list already exists"); + Ok(()) + } + + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V8_0_0"); + + let migrated = VoterList::::regenerate(); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + + StorageVersion::::put(Releases::V8_0_0); + log!( + info, + "Completed staking migration to Releases::V8_0_0 with {} voters migrated", + migrated, + ); + + T::WeightInfo::regenerate( + CounterForValidators::::get(), + CounterForNominators::::get(), + ) + .saturating_add(T::DbWeight::get().reads(2)) + } +} + +pub mod v7 { + use super::*; + + pub fn pre_migrate() -> Result<(), &'static str> { + assert!(CounterForValidators::::get().is_zero(), "CounterForValidators already set."); + assert!(CounterForNominators::::get().is_zero(), "CounterForNominators already set."); + assert!(StorageVersion::::get() == Releases::V6_0_0); + Ok(()) + } + + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V7_0_0"); + let validator_count = Validators::::iter().count() as u32; + let nominator_count = Nominators::::iter().count() as u32; + + CounterForValidators::::put(validator_count); + CounterForNominators::::put(nominator_count); + + StorageVersion::::put(Releases::V7_0_0); + log!(info, "Completed staking migration to Releases::V7_0_0"); + + T::DbWeight::get().reads_writes(validator_count.saturating_add(nominator_count).into(), 2) + } +} + +pub mod v6 { + use super::*; + use frame_support::{generate_storage_alias, traits::Get, weights::Weight}; + + // NOTE: value type doesn't matter, we just set it to () here. + generate_storage_alias!(Staking, SnapshotValidators => Value<()>); + generate_storage_alias!(Staking, SnapshotNominators => Value<()>); + generate_storage_alias!(Staking, QueuedElected => Value<()>); + generate_storage_alias!(Staking, QueuedScore => Value<()>); + generate_storage_alias!(Staking, EraElectionStatus => Value<()>); + generate_storage_alias!(Staking, IsCurrentSessionFinal => Value<()>); + + /// check to execute prior to migration. + pub fn pre_migrate() -> Result<(), &'static str> { + // these may or may not exist. + log!(info, "SnapshotValidators.exits()? {:?}", SnapshotValidators::exists()); + log!(info, "SnapshotNominators.exits()? {:?}", SnapshotNominators::exists()); + log!(info, "QueuedElected.exits()? {:?}", QueuedElected::exists()); + log!(info, "QueuedScore.exits()? {:?}", QueuedScore::exists()); + // these must exist. + assert!(IsCurrentSessionFinal::exists(), "IsCurrentSessionFinal storage item not found!"); + assert!(EraElectionStatus::exists(), "EraElectionStatus storage item not found!"); + Ok(()) + } + + /// Migrate storage to v6. + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V6_0_0"); + + SnapshotValidators::kill(); + SnapshotNominators::kill(); + QueuedElected::kill(); + QueuedScore::kill(); + EraElectionStatus::kill(); + IsCurrentSessionFinal::kill(); + + StorageVersion::::put(Releases::V6_0_0); + log!(info, "Done."); + T::DbWeight::get().writes(6 + 1) + } +} diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index a08b2e671042d..970b8a0f6e981 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -18,11 +18,14 @@ //! Test utilities use crate as staking; -use crate::*; +use crate::{voter_bags::VoterList, *}; use frame_election_provider_support::onchain; use frame_support::{ assert_ok, parameter_types, - traits::{Currency, FindAuthor, Get, OnInitialize, OneSessionHandler}, + traits::{ + Currency, FindAuthor, GenesisBuild, Get, Hooks, Imbalance, OnInitialize, OnUnbalanced, + OneSessionHandler, + }, weights::constants::RocksDbWeight, }; use sp_core::H256; diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs new file mode 100644 index 0000000000000..84f7317ee9091 --- /dev/null +++ b/frame/staking/src/pallet/impls.rs @@ -0,0 +1,1158 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementations for the Staking FRAME Pallet. + +use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; +use frame_support::{ + pallet_prelude::*, + traits::{ + Currency, CurrencyToVote, EstimateNextNewSession, Get, Imbalance, LockableCurrency, + OnUnbalanced, UnixTime, WithdrawReasons, + }, + weights::{Weight, WithPostDispatchInfo}, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use pallet_session::historical; +use sp_runtime::{ + traits::{Bounded, Convert, SaturatedConversion, Saturating, Zero}, + Perbill, +}; +use sp_staking::{ + offence::{OffenceDetails, OnOffenceHandler}, + SessionIndex, +}; +use sp_std::{collections::btree_map::BTreeMap, prelude::*}; + +use crate::{ + log, slashing, + voter_bags::{self, VoterList}, + weights::WeightInfo, + ActiveEraInfo, BalanceOf, EraIndex, EraPayout, Exposure, ExposureOf, Forcing, + IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, SessionInterface, + StakingLedger, ValidatorPrefs, VotingDataOf, +}; + +use super::{pallet::*, STAKING_ID}; + +impl Pallet { + /// The total balance that can be slashed from a stash account as of right now. + pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { + // Weight note: consider making the stake accessible through stash. + Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() + } + + /// Internal impl of [`Self::slashable_balance_of`] that returns [`VoteWeight`]. + pub fn slashable_balance_of_vote_weight( + stash: &T::AccountId, + issuance: BalanceOf, + ) -> VoteWeight { + T::CurrencyToVote::to_vote(Self::slashable_balance_of(stash), issuance) + } + + /// Returns a closure around `slashable_balance_of_vote_weight` that can be passed around. + /// + /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is + /// important to be only used while the total issuance is not changing. + pub fn weight_of_fn() -> Box VoteWeight> { + // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still + // compile, while some types in mock fail to resolve. + let issuance = T::Currency::total_issuance(); + Box::new(move |who: &T::AccountId| -> VoteWeight { + Self::slashable_balance_of_vote_weight(who, issuance) + }) + } + + pub(super) fn do_payout_stakers( + validator_stash: T::AccountId, + era: EraIndex, + ) -> DispatchResultWithPostInfo { + // Validate input data + let current_era = CurrentEra::::get().ok_or_else(|| { + Error::::InvalidEraToReward + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + })?; + let history_depth = Self::history_depth(); + ensure!( + era <= current_era && era >= current_era.saturating_sub(history_depth), + Error::::InvalidEraToReward + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + ); + + // Note: if era has no reward to be claimed, era may be future. better not to update + // `ledger.claimed_rewards` in this case. + let era_payout = >::get(&era).ok_or_else(|| { + Error::::InvalidEraToReward + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + })?; + + let controller = Self::bonded(&validator_stash).ok_or_else(|| { + Error::::NotStash.with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + })?; + let mut ledger = >::get(&controller).ok_or(Error::::NotController)?; + + ledger + .claimed_rewards + .retain(|&x| x >= current_era.saturating_sub(history_depth)); + match ledger.claimed_rewards.binary_search(&era) { + Ok(_) => Err(Error::::AlreadyClaimed + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)))?, + Err(pos) => ledger.claimed_rewards.insert(pos, era), + } + + let exposure = >::get(&era, &ledger.stash); + + // Input data seems good, no errors allowed after this point + + >::insert(&controller, &ledger); + + // Get Era reward points. It has TOTAL and INDIVIDUAL + // Find the fraction of the era reward that belongs to the validator + // Take that fraction of the eras rewards to split to nominator and validator + // + // Then look at the validator, figure out the proportion of their reward + // which goes to them and each of their nominators. + + let era_reward_points = >::get(&era); + let total_reward_points = era_reward_points.total; + let validator_reward_points = era_reward_points + .individual + .get(&ledger.stash) + .map(|points| *points) + .unwrap_or_else(|| Zero::zero()); + + // Nothing to do if they have no reward points. + if validator_reward_points.is_zero() { + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) + } + + // This is the fraction of the total reward that the validator and the + // nominators will get. + let validator_total_reward_part = + Perbill::from_rational(validator_reward_points, total_reward_points); + + // This is how much validator + nominators are entitled to. + let validator_total_payout = validator_total_reward_part * era_payout; + + let validator_prefs = Self::eras_validator_prefs(&era, &validator_stash); + // Validator first gets a cut off the top. + let validator_commission = validator_prefs.commission; + let validator_commission_payout = validator_commission * validator_total_payout; + + let validator_leftover_payout = validator_total_payout - validator_commission_payout; + // Now let's calculate how this is split to the validator. + let validator_exposure_part = Perbill::from_rational(exposure.own, exposure.total); + let validator_staking_payout = validator_exposure_part * validator_leftover_payout; + + // We can now make total validator payout: + if let Some(imbalance) = + Self::make_payout(&ledger.stash, validator_staking_payout + validator_commission_payout) + { + Self::deposit_event(Event::::Reward(ledger.stash, imbalance.peek())); + } + + // Track the number of payout ops to nominators. Note: `WeightInfo::payout_stakers_alive_staked` + // always assumes at least a validator is paid out, so we do not need to count their payout op. + let mut nominator_payout_count: u32 = 0; + + // Lets now calculate how this is split to the nominators. + // Reward only the clipped exposures. Note this is not necessarily sorted. + for nominator in exposure.others.iter() { + let nominator_exposure_part = Perbill::from_rational(nominator.value, exposure.total); + + let nominator_reward: BalanceOf = + nominator_exposure_part * validator_leftover_payout; + // We can now make nominator payout: + if let Some(imbalance) = Self::make_payout(&nominator.who, nominator_reward) { + // Note: this logic does not count payouts for `RewardDestination::None`. + nominator_payout_count += 1; + Self::deposit_event(Event::::Reward(nominator.who.clone(), imbalance.peek())); + } + } + + debug_assert!(nominator_payout_count <= T::MaxNominatorRewardedPerValidator::get()); + Ok(Some(T::WeightInfo::payout_stakers_alive_staked(nominator_payout_count)).into()) + } + + /// Update the ledger for a controller. + /// + /// This will also update the stash lock. + pub(crate) fn update_ledger( + controller: &T::AccountId, + ledger: &StakingLedger>, + ) { + T::Currency::set_lock(STAKING_ID, &ledger.stash, ledger.total, WithdrawReasons::all()); + >::insert(controller, ledger); + } + + /// Chill a stash account. + pub(crate) fn chill_stash(stash: &T::AccountId) { + let chilled_as_validator = Self::do_remove_validator(stash); + let chilled_as_nominator = Self::do_remove_nominator(stash); + if chilled_as_validator || chilled_as_nominator { + Self::deposit_event(Event::::Chilled(stash.clone())); + } + } + + /// Actually make a payment to a staker. This uses the currency's reward function + /// to pay the right payee for the given staker account. + fn make_payout(stash: &T::AccountId, amount: BalanceOf) -> Option> { + let dest = Self::payee(stash); + match dest { + RewardDestination::Controller => Self::bonded(stash) + .and_then(|controller| Some(T::Currency::deposit_creating(&controller, amount))), + RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), + RewardDestination::Staked => Self::bonded(stash) + .and_then(|c| Self::ledger(&c).map(|l| (c, l))) + .and_then(|(controller, mut l)| { + l.active += amount; + l.total += amount; + let r = T::Currency::deposit_into_existing(stash, amount).ok(); + Self::update_ledger(&controller, &l); + r + }), + RewardDestination::Account(dest_account) => + Some(T::Currency::deposit_creating(&dest_account, amount)), + RewardDestination::None => None, + } + } + + /// Plan a new session potentially trigger a new era. + fn new_session(session_index: SessionIndex, is_genesis: bool) -> Option> { + if let Some(current_era) = Self::current_era() { + // Initial era has been set. + let current_era_start_session_index = Self::eras_start_session_index(current_era) + .unwrap_or_else(|| { + frame_support::print("Error: start_session_index must be set for current_era"); + 0 + }); + + let era_length = + session_index.checked_sub(current_era_start_session_index).unwrap_or(0); // Must never happen. + + match ForceEra::::get() { + // Will be set to `NotForcing` again if a new era has been triggered. + Forcing::ForceNew => (), + // Short circuit to `try_trigger_new_era`. + Forcing::ForceAlways => (), + // Only go to `try_trigger_new_era` if deadline reached. + Forcing::NotForcing if era_length >= T::SessionsPerEra::get() => (), + _ => { + // Either `Forcing::ForceNone`, + // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. + return None + }, + } + + // New era. + let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); + if maybe_new_era_validators.is_some() && + matches!(ForceEra::::get(), Forcing::ForceNew) + { + ForceEra::::put(Forcing::NotForcing); + } + + maybe_new_era_validators + } else { + // Set initial era. + log!(debug, "Starting the first era."); + Self::try_trigger_new_era(session_index, is_genesis) + } + } + + /// Start a session potentially starting an era. + fn start_session(start_session: SessionIndex) { + let next_active_era = Self::active_era().map(|e| e.index + 1).unwrap_or(0); + // This is only `Some` when current era has already progressed to the next era, while the + // active era is one behind (i.e. in the *last session of the active era*, or *first session + // of the new current era*, depending on how you look at it). + if let Some(next_active_era_start_session_index) = + Self::eras_start_session_index(next_active_era) + { + if next_active_era_start_session_index == start_session { + Self::start_era(start_session); + } else if next_active_era_start_session_index < start_session { + // This arm should never happen, but better handle it than to stall the staking + // pallet. + frame_support::print("Warning: A session appears to have been skipped."); + Self::start_era(start_session); + } + } + } + + /// End a session potentially ending an era. + fn end_session(session_index: SessionIndex) { + if let Some(active_era) = Self::active_era() { + if let Some(next_active_era_start_session_index) = + Self::eras_start_session_index(active_era.index + 1) + { + if next_active_era_start_session_index == session_index + 1 { + Self::end_era(active_era, session_index); + } + } + } + } + + /// * Increment `active_era.index`, + /// * reset `active_era.start`, + /// * update `BondedEras` and apply slashes. + fn start_era(start_session: SessionIndex) { + let active_era = ActiveEra::::mutate(|active_era| { + let new_index = active_era.as_ref().map(|info| info.index + 1).unwrap_or(0); + *active_era = Some(ActiveEraInfo { + index: new_index, + // Set new active era start in next `on_finalize`. To guarantee usage of `Time` + start: None, + }); + new_index + }); + + let bonding_duration = T::BondingDuration::get(); + + BondedEras::::mutate(|bonded| { + bonded.push((active_era, start_session)); + + if active_era > bonding_duration { + let first_kept = active_era - bonding_duration; + + // Prune out everything that's from before the first-kept index. + let n_to_prune = + bonded.iter().take_while(|&&(era_idx, _)| era_idx < first_kept).count(); + + // Kill slashing metadata. + for (pruned_era, _) in bonded.drain(..n_to_prune) { + slashing::clear_era_metadata::(pruned_era); + } + + if let Some(&(_, first_session)) = bonded.first() { + T::SessionInterface::prune_historical_up_to(first_session); + } + } + }); + + Self::apply_unapplied_slashes(active_era); + } + + /// Compute payout for era. + fn end_era(active_era: ActiveEraInfo, _session_index: SessionIndex) { + // Note: active_era_start can be None if end era is called during genesis config. + if let Some(active_era_start) = active_era.start { + let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); + + let era_duration = (now_as_millis_u64 - active_era_start).saturated_into::(); + let staked = Self::eras_total_stake(&active_era.index); + let issuance = T::Currency::total_issuance(); + let (validator_payout, rest) = T::EraPayout::era_payout(staked, issuance, era_duration); + + Self::deposit_event(Event::::EraPayout(active_era.index, validator_payout, rest)); + + // Set ending era reward. + >::insert(&active_era.index, validator_payout); + T::RewardRemainder::on_unbalanced(T::Currency::issue(rest)); + } + } + + /// Plan a new era. + /// + /// * Bump the current era storage (which holds the latest planned era). + /// * Store start session index for the new planned era. + /// * Clean old era information. + /// * Store staking information for the new planned era + /// + /// Returns the new validator set. + pub fn trigger_new_era( + start_session_index: SessionIndex, + exposures: Vec<(T::AccountId, Exposure>)>, + ) -> Vec { + // Increment or set current era. + let new_planned_era = CurrentEra::::mutate(|s| { + *s = Some(s.map(|s| s + 1).unwrap_or(0)); + s.unwrap() + }); + ErasStartSessionIndex::::insert(&new_planned_era, &start_session_index); + + // Clean old era information. + if let Some(old_era) = new_planned_era.checked_sub(Self::history_depth() + 1) { + Self::clear_era_information(old_era); + } + + // Set staking information for the new era. + Self::store_stakers_info(exposures, new_planned_era) + } + + /// Potentially plan a new era. + /// + /// Get election result from `T::ElectionProvider`. + /// In case election result has more than [`MinimumValidatorCount`] validator trigger a new era. + /// + /// In case a new era is planned, the new validator set is returned. + pub(crate) fn try_trigger_new_era( + start_session_index: SessionIndex, + is_genesis: bool, + ) -> Option> { + let (election_result, weight) = if is_genesis { + T::GenesisElectionProvider::elect().map_err(|e| { + log!(warn, "genesis election provider failed due to {:?}", e); + Self::deposit_event(Event::StakingElectionFailed); + }) + } else { + T::ElectionProvider::elect().map_err(|e| { + log!(warn, "election provider failed due to {:?}", e); + Self::deposit_event(Event::StakingElectionFailed); + }) + } + .ok()?; + + >::register_extra_weight_unchecked( + weight, + frame_support::weights::DispatchClass::Mandatory, + ); + + let exposures = Self::collect_exposures(election_result); + + if (exposures.len() as u32) < Self::minimum_validator_count().max(1) { + // Session will panic if we ever return an empty validator set, thus max(1) ^^. + match CurrentEra::::get() { + Some(current_era) if current_era > 0 => log!( + warn, + "chain does not have enough staking candidates to operate for era {:?} ({} \ + elected, minimum is {})", + CurrentEra::::get().unwrap_or(0), + exposures.len(), + Self::minimum_validator_count(), + ), + None => { + // The initial era is allowed to have no exposures. + // In this case the SessionManager is expected to choose a sensible validator + // set. + // TODO: this should be simplified #8911 + CurrentEra::::put(0); + ErasStartSessionIndex::::insert(&0, &start_session_index); + }, + _ => (), + } + + Self::deposit_event(Event::StakingElectionFailed); + return None + } + + Self::deposit_event(Event::StakingElection); + Some(Self::trigger_new_era(start_session_index, exposures)) + } + + /// Process the output of the election. + /// + /// Store staking information for the new planned era + pub fn store_stakers_info( + exposures: Vec<(T::AccountId, Exposure>)>, + new_planned_era: EraIndex, + ) -> Vec { + let elected_stashes = exposures.iter().cloned().map(|(x, _)| x).collect::>(); + + // Populate stakers, exposures, and the snapshot of validator prefs. + let mut total_stake: BalanceOf = Zero::zero(); + exposures.into_iter().for_each(|(stash, exposure)| { + total_stake = total_stake.saturating_add(exposure.total); + >::insert(new_planned_era, &stash, &exposure); + + let mut exposure_clipped = exposure; + let clipped_max_len = T::MaxNominatorRewardedPerValidator::get() as usize; + if exposure_clipped.others.len() > clipped_max_len { + exposure_clipped.others.sort_by(|a, b| a.value.cmp(&b.value).reverse()); + exposure_clipped.others.truncate(clipped_max_len); + } + >::insert(&new_planned_era, &stash, exposure_clipped); + }); + + // Insert current era staking information + >::insert(&new_planned_era, total_stake); + + // Collect the pref of all winners. + for stash in &elected_stashes { + let pref = Self::validators(stash); + >::insert(&new_planned_era, stash, pref); + } + + if new_planned_era > 0 { + log!( + info, + "new validator set of size {:?} has been processed for era {:?}", + elected_stashes.len(), + new_planned_era, + ); + } + + elected_stashes + } + + /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a + /// [`Exposure`]. + fn collect_exposures( + supports: Supports, + ) -> Vec<(T::AccountId, Exposure>)> { + let total_issuance = T::Currency::total_issuance(); + let to_currency = |e: frame_election_provider_support::ExtendedBalance| { + T::CurrencyToVote::to_currency(e, total_issuance) + }; + + supports + .into_iter() + .map(|(validator, support)| { + // Build `struct exposure` from `support`. + let mut others = Vec::with_capacity(support.voters.len()); + let mut own: BalanceOf = Zero::zero(); + let mut total: BalanceOf = Zero::zero(); + support + .voters + .into_iter() + .map(|(nominator, weight)| (nominator, to_currency(weight))) + .for_each(|(nominator, stake)| { + if nominator == validator { + own = own.saturating_add(stake); + } else { + others.push(IndividualExposure { who: nominator, value: stake }); + } + total = total.saturating_add(stake); + }); + + let exposure = Exposure { own, others, total }; + (validator, exposure) + }) + .collect::)>>() + } + + /// Remove all associated data of a stash account from the staking system. + /// + /// Assumes storage is upgraded before calling. + /// + /// This is called: + /// - after a `withdraw_unbonded()` call that frees all of a stash's bonded balance. + /// - through `reap_stash()` if the balance has fallen to zero (through slashing). + pub(crate) fn kill_stash(stash: &T::AccountId, num_slashing_spans: u32) -> DispatchResult { + let controller = >::get(stash).ok_or(Error::::NotStash)?; + + slashing::clear_stash_metadata::(stash, num_slashing_spans)?; + + >::remove(stash); + >::remove(&controller); + + >::remove(stash); + Self::do_remove_validator(stash); + Self::do_remove_nominator(stash); + + frame_system::Pallet::::dec_consumers(stash); + + Ok(()) + } + + /// Clear all era information for given era. + pub(super) fn clear_era_information(era_index: EraIndex) { + >::remove_prefix(era_index, None); + >::remove_prefix(era_index, None); + >::remove_prefix(era_index, None); + >::remove(era_index); + >::remove(era_index); + >::remove(era_index); + ErasStartSessionIndex::::remove(era_index); + } + + /// Apply previously-unapplied slashes on the beginning of a new era, after a delay. + fn apply_unapplied_slashes(active_era: EraIndex) { + let slash_defer_duration = T::SlashDeferDuration::get(); + ::EarliestUnappliedSlash::mutate(|earliest| { + if let Some(ref mut earliest) = earliest { + let keep_from = active_era.saturating_sub(slash_defer_duration); + for era in (*earliest)..keep_from { + let era_slashes = ::UnappliedSlashes::take(&era); + for slash in era_slashes { + slashing::apply_slash::(slash); + } + } + + *earliest = (*earliest).max(keep_from) + } + }) + } + + /// Add reward points to validators using their stash account ID. + /// + /// Validators are keyed by stash account ID and must be in the current elected set. + /// + /// For each element in the iterator the given number of points in u32 is added to the + /// validator, thus duplicates are handled. + /// + /// At the end of the era each the total payout will be distributed among validator + /// relatively to their points. + /// + /// COMPLEXITY: Complexity is `number_of_validator_to_reward x current_elected_len`. + pub fn reward_by_ids(validators_points: impl IntoIterator) { + if let Some(active_era) = Self::active_era() { + >::mutate(active_era.index, |era_rewards| { + for (validator, points) in validators_points.into_iter() { + *era_rewards.individual.entry(validator).or_default() += points; + era_rewards.total += points; + } + }); + } + } + + /// Ensures that at the end of the current session there will be a new era. + pub(crate) fn ensure_new_era() { + match ForceEra::::get() { + Forcing::ForceAlways | Forcing::ForceNew => (), + _ => ForceEra::::put(Forcing::ForceNew), + } + } + + #[cfg(feature = "runtime-benchmarks")] + pub fn add_era_stakers( + current_era: EraIndex, + controller: T::AccountId, + exposure: Exposure>, + ) { + >::insert(¤t_era, &controller, &exposure); + } + + #[cfg(feature = "runtime-benchmarks")] + pub fn set_slash_reward_fraction(fraction: Perbill) { + SlashRewardFraction::::put(fraction); + } + + /// Get all of the voters that are eligible for the npos election. + /// + /// `voter_count` imposes an implicit cap on the number of voters returned; care should be taken + /// to ensure that it is accurate. + /// + /// This will use all on-chain nominators, and all the validators will inject a self vote. + /// + /// ### Slashing + /// + /// All nominations that have been submitted before the last non-zero slash of the validator are + /// auto-chilled. + /// + /// Note that this is fairly expensive: it must iterate over the min of `maybe_max_len` and + /// `voter_count` voters. Use with care. + pub fn get_npos_voters( + maybe_max_len: Option, + voter_count: usize, + ) -> Vec> { + let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); + + let weight_of = Self::weight_of_fn(); + // collect all slashing spans into a BTreeMap for further queries. + let slashing_spans = >::iter().collect::>(); + + VoterList::::iter() + .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) + .take(wanted_voters) + .collect() + } + + /// This is a very expensive function and result should be cached versus being called multiple times. + pub fn get_npos_targets() -> Vec { + Validators::::iter().map(|(v, _)| v).collect::>() + } + + /// This function will add a nominator to the `Nominators` storage map, + /// and keep track of the `CounterForNominators`. + /// + /// If the nominator already exists, their nominations will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a nominator to the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { + if !Nominators::::contains_key(who) { + CounterForNominators::::mutate(|x| x.saturating_inc()) + } + Nominators::::insert(who, nominations); + VoterList::::insert_as(who, voter_bags::VoterType::Nominator); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + } + + /// This function will remove a nominator from the `Nominators` storage map, + /// and keep track of the `CounterForNominators`. + /// + /// Returns true if `who` was removed from `Nominators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_remove_nominator(who: &T::AccountId) -> bool { + if Nominators::::contains_key(who) { + Nominators::::remove(who); + CounterForNominators::::mutate(|x| x.saturating_dec()); + VoterList::::remove(who); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + true + } else { + false + } + } + + /// This function will add a validator to the `Validators` storage map, and keep track of the + /// `CounterForValidators`. + /// + /// If the validator already exists, their preferences will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { + if !Validators::::contains_key(who) { + CounterForValidators::::mutate(|x| x.saturating_inc()) + } + Validators::::insert(who, prefs); + VoterList::::insert_as(who, voter_bags::VoterType::Validator); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + } + + /// This function will remove a validator from the `Validators` storage map, + /// and keep track of the `CounterForValidators`. + /// + /// Returns true if `who` was removed from `Validators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_remove_validator(who: &T::AccountId) -> bool { + if Validators::::contains_key(who) { + Validators::::remove(who); + CounterForValidators::::mutate(|x| x.saturating_dec()); + VoterList::::remove(who); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + true + } else { + false + } + } + + /// Move a stash account from one bag to another, depositing an event on success. + /// + /// If the stash changed bags, returns `Some((from, to))`. + pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { + let weight_of = Self::weight_of_fn(); + VoterList::update_position_for(node, weight_of) + }); + if let Some((from, to)) = maybe_movement { + Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); + }; + maybe_movement + } +} + +impl + frame_election_provider_support::ElectionDataProvider> + for Pallet +{ + const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; + fn desired_targets() -> data_provider::Result<(u32, Weight)> { + Ok((Self::validator_count(), ::DbWeight::get().reads(1))) + } + + fn voters( + maybe_max_len: Option, + ) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec)>, Weight)> { + let nominator_count = CounterForNominators::::get(); + let validator_count = CounterForValidators::::get(); + let voter_count = nominator_count.saturating_add(validator_count) as usize; + + // check a few counters one last time... + debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); + debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); + debug_assert_eq!( + voter_count, + VoterList::::decode_len().unwrap_or_default(), + "voter_count must be accurate", + ); + + let slashing_span_count = >::iter().count(); + let weight = T::WeightInfo::get_npos_voters( + nominator_count, + validator_count, + slashing_span_count as u32, + ); + + Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) + } + + fn targets(maybe_max_len: Option) -> data_provider::Result<(Vec, Weight)> { + let target_count = CounterForValidators::::get() as usize; + + if maybe_max_len.map_or(false, |max_len| target_count > max_len) { + return Err("Target snapshot too big") + } + + let weight = ::DbWeight::get().reads(target_count as u64); + Ok((Self::get_npos_targets(), weight)) + } + + fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber { + let current_era = Self::current_era().unwrap_or(0); + let current_session = Self::current_planned_session(); + let current_era_start_session_index = + Self::eras_start_session_index(current_era).unwrap_or(0); + // Number of session in the current era or the maximum session per era if reached. + let era_progress = current_session + .saturating_sub(current_era_start_session_index) + .min(T::SessionsPerEra::get()); + + let until_this_session_end = T::NextNewSession::estimate_next_new_session(now) + .0 + .unwrap_or_default() + .saturating_sub(now); + + let session_length = T::NextNewSession::average_session_length(); + + let sessions_left: T::BlockNumber = match ForceEra::::get() { + Forcing::ForceNone => Bounded::max_value(), + Forcing::ForceNew | Forcing::ForceAlways => Zero::zero(), + Forcing::NotForcing if era_progress >= T::SessionsPerEra::get() => Zero::zero(), + Forcing::NotForcing => T::SessionsPerEra::get() + .saturating_sub(era_progress) + // One session is computed in this_session_end. + .saturating_sub(1) + .into(), + }; + + now.saturating_add( + until_this_session_end.saturating_add(sessions_left.saturating_mul(session_length)), + ) + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn add_voter(voter: T::AccountId, weight: VoteWeight, targets: Vec) { + use sp_std::convert::TryFrom; + let stake = >::try_from(weight).unwrap_or_else(|_| { + panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") + }); + >::insert(voter.clone(), voter.clone()); + >::insert( + voter.clone(), + StakingLedger { + stash: voter.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_nominator(&voter, Nominations { targets, submitted_in: 0, suppressed: false }); + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn add_target(target: T::AccountId) { + let stake = MinValidatorBond::::get() * 100u32.into(); + >::insert(target.clone(), target.clone()); + >::insert( + target.clone(), + StakingLedger { + stash: target.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_validator( + &target, + ValidatorPrefs { commission: Perbill::zero(), blocked: false }, + ); + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn clear() { + >::remove_all(None); + >::remove_all(None); + >::remove_all(None); + >::remove_all(None); + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn put_snapshot( + voters: Vec<(T::AccountId, VoteWeight, Vec)>, + targets: Vec, + target_stake: Option, + ) { + use sp_std::convert::TryFrom; + targets.into_iter().for_each(|v| { + let stake: BalanceOf = target_stake + .and_then(|w| >::try_from(w).ok()) + .unwrap_or_else(|| MinNominatorBond::::get() * 100u32.into()); + >::insert(v.clone(), v.clone()); + >::insert( + v.clone(), + StakingLedger { + stash: v.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_validator( + &v, + ValidatorPrefs { commission: Perbill::zero(), blocked: false }, + ); + }); + + voters.into_iter().for_each(|(v, s, t)| { + let stake = >::try_from(s).unwrap_or_else(|_| { + panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") + }); + >::insert(v.clone(), v.clone()); + >::insert( + v.clone(), + StakingLedger { + stash: v.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_nominator( + &v, + Nominations { targets: t, submitted_in: 0, suppressed: false }, + ); + }); + } +} + +/// In this implementation `new_session(session)` must be called before `end_session(session-1)` +/// i.e. the new session must be planned before the ending of the previous session. +/// +/// Once the first new_session is planned, all session must start and then end in order, though +/// some session can lag in between the newest session planned and the latest session started. +impl pallet_session::SessionManager for Pallet { + fn new_session(new_index: SessionIndex) -> Option> { + log!(trace, "planning new session {}", new_index); + CurrentPlannedSession::::put(new_index); + Self::new_session(new_index, false) + } + fn new_session_genesis(new_index: SessionIndex) -> Option> { + log!(trace, "planning new session {} at genesis", new_index); + CurrentPlannedSession::::put(new_index); + Self::new_session(new_index, true) + } + fn start_session(start_index: SessionIndex) { + log!(trace, "starting session {}", start_index); + Self::start_session(start_index) + } + fn end_session(end_index: SessionIndex) { + log!(trace, "ending session {}", end_index); + Self::end_session(end_index) + } +} + +impl historical::SessionManager>> + for Pallet +{ + fn new_session( + new_index: SessionIndex, + ) -> Option>)>> { + >::new_session(new_index).map(|validators| { + let current_era = Self::current_era() + // Must be some as a new era has been created. + .unwrap_or(0); + + validators + .into_iter() + .map(|v| { + let exposure = Self::eras_stakers(current_era, &v); + (v, exposure) + }) + .collect() + }) + } + fn new_session_genesis( + new_index: SessionIndex, + ) -> Option>)>> { + >::new_session_genesis(new_index).map( + |validators| { + let current_era = Self::current_era() + // Must be some as a new era has been created. + .unwrap_or(0); + + validators + .into_iter() + .map(|v| { + let exposure = Self::eras_stakers(current_era, &v); + (v, exposure) + }) + .collect() + }, + ) + } + fn start_session(start_index: SessionIndex) { + >::start_session(start_index) + } + fn end_session(end_index: SessionIndex) { + >::end_session(end_index) + } +} + +/// Add reward points to block authors: +/// * 20 points to the block producer for producing a (non-uncle) block in the relay chain, +/// * 2 points to the block producer for each reference to a previously unreferenced uncle, and +/// * 1 point to the producer of each referenced uncle block. +impl pallet_authorship::EventHandler for Pallet +where + T: Config + pallet_authorship::Config + pallet_session::Config, +{ + fn note_author(author: T::AccountId) { + Self::reward_by_ids(vec![(author, 20)]) + } + fn note_uncle(author: T::AccountId, _age: T::BlockNumber) { + Self::reward_by_ids(vec![(>::author(), 2), (author, 1)]) + } +} + +/// This is intended to be used with `FilterHistoricalOffences`. +impl + OnOffenceHandler, Weight> + for Pallet +where + T: pallet_session::Config::AccountId>, + T: pallet_session::historical::Config< + FullIdentification = Exposure<::AccountId, BalanceOf>, + FullIdentificationOf = ExposureOf, + >, + T::SessionHandler: pallet_session::SessionHandler<::AccountId>, + T::SessionManager: pallet_session::SessionManager<::AccountId>, + T::ValidatorIdOf: Convert< + ::AccountId, + Option<::AccountId>, + >, +{ + fn on_offence( + offenders: &[OffenceDetails< + T::AccountId, + pallet_session::historical::IdentificationTuple, + >], + slash_fraction: &[Perbill], + slash_session: SessionIndex, + ) -> Weight { + let reward_proportion = SlashRewardFraction::::get(); + let mut consumed_weight: Weight = 0; + let mut add_db_reads_writes = |reads, writes| { + consumed_weight += T::DbWeight::get().reads_writes(reads, writes); + }; + + let active_era = { + let active_era = Self::active_era(); + add_db_reads_writes(1, 0); + if active_era.is_none() { + // This offence need not be re-submitted. + return consumed_weight + } + active_era.expect("value checked not to be `None`; qed").index + }; + let active_era_start_session_index = Self::eras_start_session_index(active_era) + .unwrap_or_else(|| { + frame_support::print("Error: start_session_index must be set for current_era"); + 0 + }); + add_db_reads_writes(1, 0); + + let window_start = active_era.saturating_sub(T::BondingDuration::get()); + + // Fast path for active-era report - most likely. + // `slash_session` cannot be in a future active era. It must be in `active_era` or before. + let slash_era = if slash_session >= active_era_start_session_index { + active_era + } else { + let eras = BondedEras::::get(); + add_db_reads_writes(1, 0); + + // Reverse because it's more likely to find reports from recent eras. + match eras.iter().rev().filter(|&&(_, ref sesh)| sesh <= &slash_session).next() { + Some(&(ref slash_era, _)) => *slash_era, + // Before bonding period. defensive - should be filtered out. + None => return consumed_weight, + } + }; + + ::EarliestUnappliedSlash::mutate(|earliest| { + if earliest.is_none() { + *earliest = Some(active_era) + } + }); + add_db_reads_writes(1, 1); + + let slash_defer_duration = T::SlashDeferDuration::get(); + + let invulnerables = Self::invulnerables(); + add_db_reads_writes(1, 0); + + for (details, slash_fraction) in offenders.iter().zip(slash_fraction) { + let (stash, exposure) = &details.offender; + + // Skip if the validator is invulnerable. + if invulnerables.contains(stash) { + continue + } + + let unapplied = slashing::compute_slash::(slashing::SlashParams { + stash, + slash: *slash_fraction, + exposure, + slash_era, + window_start, + now: active_era, + reward_proportion, + }); + + if let Some(mut unapplied) = unapplied { + let nominators_len = unapplied.others.len() as u64; + let reporters_len = details.reporters.len() as u64; + + { + let upper_bound = 1 /* Validator/NominatorSlashInEra */ + 2 /* fetch_spans */; + let rw = upper_bound + nominators_len * upper_bound; + add_db_reads_writes(rw, rw); + } + unapplied.reporters = details.reporters.clone(); + if slash_defer_duration == 0 { + // Apply right away. + slashing::apply_slash::(unapplied); + { + let slash_cost = (6, 5); + let reward_cost = (2, 2); + add_db_reads_writes( + (1 + nominators_len) * slash_cost.0 + reward_cost.0 * reporters_len, + (1 + nominators_len) * slash_cost.1 + reward_cost.1 * reporters_len, + ); + } + } else { + // Defer to end of some `slash_defer_duration` from now. + ::UnappliedSlashes::mutate(active_era, move |for_later| { + for_later.push(unapplied) + }); + add_db_reads_writes(1, 1); + } + } else { + add_db_reads_writes(4 /* fetch_spans */, 5 /* kick_out_if_recent */) + } + } + + consumed_weight + } +} diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs new file mode 100644 index 0000000000000..8150d9f7cf2a5 --- /dev/null +++ b/frame/staking/src/pallet/mod.rs @@ -0,0 +1,1698 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Staking FRAME Pallet. + +use frame_election_provider_support::VoteWeight; +use frame_support::{ + pallet_prelude::*, + traits::{ + Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, LockIdentifier, + LockableCurrency, OnUnbalanced, UnixTime, + }, + weights::{ + constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}, + Weight, + }, +}; +use frame_system::{ensure_root, ensure_signed, offchain::SendTransactionTypes, pallet_prelude::*}; +use sp_runtime::{ + traits::{CheckedSub, SaturatedConversion, StaticLookup, Zero}, + DispatchError, Perbill, Percent, +}; +use sp_staking::SessionIndex; +use sp_std::{convert::From, prelude::*, result}; + +mod impls; + +pub use impls::*; + +use crate::{ + log, migrations, slashing, voter_bags, weights::WeightInfo, AccountIdOf, ActiveEraInfo, + BalanceOf, EraIndex, EraPayout, EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, + Nominations, PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakerStatus, + StakingLedger, UnappliedSlash, UnlockChunk, ValidatorPrefs, +}; + +pub const MAX_UNLOCKING_CHUNKS: usize = 32; +const STAKING_ID: LockIdentifier = *b"staking "; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(crate) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + SendTransactionTypes> { + /// The staking balance. + type Currency: LockableCurrency; + + /// Time used for computing era duration. + /// + /// It is guaranteed to start being called from the first `on_finalize`. Thus value at genesis + /// is not used. + type UnixTime: UnixTime; + + /// Convert a balance into a number used for election calculation. This must fit into a `u64` + /// but is allowed to be sensibly lossy. The `u64` is used to communicate with the + /// [`sp_npos_elections`] crate which accepts u64 numbers and does operations in 128. + /// Consequently, the backward convert is used convert the u128s from sp-elections back to a + /// [`BalanceOf`]. + type CurrencyToVote: CurrencyToVote>; + + /// Something that provides the election functionality. + type ElectionProvider: frame_election_provider_support::ElectionProvider< + Self::AccountId, + Self::BlockNumber, + // we only accept an election provider that has staking as data provider. + DataProvider = Pallet, + >; + + /// Something that provides the election functionality at genesis. + type GenesisElectionProvider: frame_election_provider_support::ElectionProvider< + Self::AccountId, + Self::BlockNumber, + DataProvider = Pallet, + >; + + /// Maximum number of nominations per nominator. + const MAX_NOMINATIONS: u32; + + /// Tokens have been minted and are unused for validator-reward. + /// See [Era payout](./index.html#era-payout). + type RewardRemainder: OnUnbalanced>; + + /// The overarching event type. + type Event: From> + IsType<::Event>; + + /// Handler for the unbalanced reduction when slashing a staker. + type Slash: OnUnbalanced>; + + /// Handler for the unbalanced increment when rewarding a staker. + type Reward: OnUnbalanced>; + + /// Number of sessions per era. + #[pallet::constant] + type SessionsPerEra: Get; + + /// Number of eras that staked funds must remain bonded for. + #[pallet::constant] + type BondingDuration: Get; + + /// Number of eras that slashes are deferred by, after computation. + /// + /// This should be less than the bonding duration. Set to 0 if slashes + /// should be applied immediately, without opportunity for intervention. + #[pallet::constant] + type SlashDeferDuration: Get; + + /// The origin which can cancel a deferred slash. Root can always do this. + type SlashCancelOrigin: EnsureOrigin; + + /// Interface for interacting with a session pallet. + type SessionInterface: self::SessionInterface; + + /// The payout for validators and the system for the current era. + /// See [Era payout](./index.html#era-payout). + type EraPayout: EraPayout>; + + /// Something that can estimate the next session change, accurately or as a best effort guess. + type NextNewSession: EstimateNextNewSession; + + /// The maximum number of nominators rewarded for each validator. + /// + /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim + /// their reward. This used to limit the i/o cost for the nominator payout. + #[pallet::constant] + type MaxNominatorRewardedPerValidator: Get; + + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + + /// The list of thresholds separating the various voter bags. + /// + /// Voters are separated into unsorted bags according to their vote weight. This specifies + /// the thresholds separating the bags. A voter's bag is the largest bag for which the + /// voter's weight is less than or equal to its upper threshold. + /// + /// When voters are iterated, higher bags are iterated completely before lower bags. This + /// means that iteration is _semi-sorted_: voters of higher weight tend to come before + /// voters of lower weight, but peer voters within a particular bag are sorted in insertion + /// order. + /// + /// # Expressing the constant + /// + /// This constant must be sorted in strictly increasing order. Duplicate items are not + /// permitted. + /// + /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// specified within the bag. For any two threshold lists, if one ends with + /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists + /// will behave identically. + /// + /// # Calculation + /// + /// It is recommended to generate the set of thresholds in a geometric series, such that + /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * + /// constant_ratio).max(threshold[k] + 1)` for all `k`. + /// + /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use + /// them, the `make-bags` feature must be enabled. + /// + /// # Examples + /// + /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, + /// and iteration is strictly in insertion order. + /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is equal to 2. + /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is approximately equal + /// to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will + /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. + /// + /// # Migration + /// + /// In the event that this list ever changes, a copy of the old bags list must be retained. + /// With that `VoterList::migrate` can be called, which will perform the appropriate + /// migration. + #[pallet::constant] + type VoterBagThresholds: Get<&'static [VoteWeight]>; + } + + #[pallet::extra_constants] + impl Pallet { + // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. + #[allow(non_snake_case)] + fn MaxNominations() -> u32 { + T::MAX_NOMINATIONS + } + } + + #[pallet::type_value] + pub(crate) fn HistoryDepthOnEmpty() -> u32 { + 84u32 + } + + /// Number of eras to keep in history. + /// + /// Information is kept for eras in `[current_era - history_depth; current_era]`. + /// + /// Must be more than the number of eras delayed by session otherwise. I.e. active era must + /// always be in history. I.e. `active_era > current_era - history_depth` must be + /// guaranteed. + #[pallet::storage] + #[pallet::getter(fn history_depth)] + pub(crate) type HistoryDepth = StorageValue<_, u32, ValueQuery, HistoryDepthOnEmpty>; + + /// The ideal number of staking participants. + #[pallet::storage] + #[pallet::getter(fn validator_count)] + pub type ValidatorCount = StorageValue<_, u32, ValueQuery>; + + /// Minimum number of staking participants before emergency conditions are imposed. + #[pallet::storage] + #[pallet::getter(fn minimum_validator_count)] + pub type MinimumValidatorCount = StorageValue<_, u32, ValueQuery>; + + /// Any validators that may never be slashed or forcibly kicked. It's a Vec since they're + /// easy to initialize and the performance hit is minimal (we expect no more than four + /// invulnerables) and restricted to testnets. + #[pallet::storage] + #[pallet::getter(fn invulnerables)] + pub type Invulnerables = StorageValue<_, Vec, ValueQuery>; + + /// Map from all locked "stash" accounts to the controller account. + #[pallet::storage] + #[pallet::getter(fn bonded)] + pub type Bonded = StorageMap<_, Twox64Concat, T::AccountId, T::AccountId>; + + /// The minimum active bond to become and maintain the role of a nominator. + #[pallet::storage] + pub type MinNominatorBond = StorageValue<_, BalanceOf, ValueQuery>; + + /// The minimum active bond to become and maintain the role of a validator. + #[pallet::storage] + pub type MinValidatorBond = StorageValue<_, BalanceOf, ValueQuery>; + + /// Map from all (unlocked) "controller" accounts to the info regarding the staking. + #[pallet::storage] + #[pallet::getter(fn ledger)] + pub type Ledger = + StorageMap<_, Blake2_128Concat, T::AccountId, StakingLedger>>; + + /// Where the reward payment should be made. Keyed by stash. + #[pallet::storage] + #[pallet::getter(fn payee)] + pub type Payee = + StorageMap<_, Twox64Concat, T::AccountId, RewardDestination, ValueQuery>; + + /// The map from (wannabe) validator stash key to the preferences of that validator. + /// + /// When updating this storage item, you must also update the `CounterForValidators`. + #[pallet::storage] + #[pallet::getter(fn validators)] + pub type Validators = + StorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; + + /// A tracker to keep count of the number of items in the `Validators` map. + #[pallet::storage] + pub type CounterForValidators = StorageValue<_, u32, ValueQuery>; + + /// The maximum validator count before we stop allowing new validators to join. + /// + /// When this value is not set, no limits are enforced. + #[pallet::storage] + pub type MaxValidatorsCount = StorageValue<_, u32, OptionQuery>; + + /// The map from nominator stash key to the set of stash keys of all validators to nominate. + /// + /// When updating this storage item, you must also update the `CounterForNominators`. + #[pallet::storage] + #[pallet::getter(fn nominators)] + pub type Nominators = + StorageMap<_, Twox64Concat, T::AccountId, Nominations>; + + /// A tracker to keep count of the number of items in the `Nominators` map. + #[pallet::storage] + pub type CounterForNominators = StorageValue<_, u32, ValueQuery>; + + /// The maximum nominator count before we stop allowing new validators to join. + /// + /// When this value is not set, no limits are enforced. + #[pallet::storage] + pub type MaxNominatorsCount = StorageValue<_, u32, OptionQuery>; + + /// The current era index. + /// + /// This is the latest planned era, depending on how the Session pallet queues the validator + /// set, it might be active or not. + #[pallet::storage] + #[pallet::getter(fn current_era)] + pub type CurrentEra = StorageValue<_, EraIndex>; + + /// The active era information, it holds index and start. + /// + /// The active era is the era being currently rewarded. Validator set of this era must be + /// equal to [`SessionInterface::validators`]. + #[pallet::storage] + #[pallet::getter(fn active_era)] + pub type ActiveEra = StorageValue<_, ActiveEraInfo>; + + /// The session index at which the era start for the last `HISTORY_DEPTH` eras. + /// + /// Note: This tracks the starting session (i.e. session index when era start being active) + /// for the eras in `[CurrentEra - HISTORY_DEPTH, CurrentEra]`. + #[pallet::storage] + #[pallet::getter(fn eras_start_session_index)] + pub type ErasStartSessionIndex = StorageMap<_, Twox64Concat, EraIndex, SessionIndex>; + + /// Exposure of validator at era. + /// + /// This is keyed first by the era index to allow bulk deletion and then the stash account. + /// + /// Is it removed after `HISTORY_DEPTH` eras. + /// If stakers hasn't been set or has been removed then empty exposure is returned. + #[pallet::storage] + #[pallet::getter(fn eras_stakers)] + pub type ErasStakers = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + Exposure>, + ValueQuery, + >; + + /// Clipped Exposure of validator at era. + /// + /// This is similar to [`ErasStakers`] but number of nominators exposed is reduced to the + /// `T::MaxNominatorRewardedPerValidator` biggest stakers. + /// (Note: the field `total` and `own` of the exposure remains unchanged). + /// This is used to limit the i/o cost for the nominator payout. + /// + /// This is keyed fist by the era index to allow bulk deletion and then the stash account. + /// + /// Is it removed after `HISTORY_DEPTH` eras. + /// If stakers hasn't been set or has been removed then empty exposure is returned. + #[pallet::storage] + #[pallet::getter(fn eras_stakers_clipped)] + pub type ErasStakersClipped = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + Exposure>, + ValueQuery, + >; + + /// Similar to `ErasStakers`, this holds the preferences of validators. + /// + /// This is keyed first by the era index to allow bulk deletion and then the stash account. + /// + /// Is it removed after `HISTORY_DEPTH` eras. + // If prefs hasn't been set or has been removed then 0 commission is returned. + #[pallet::storage] + #[pallet::getter(fn eras_validator_prefs)] + pub type ErasValidatorPrefs = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + ValidatorPrefs, + ValueQuery, + >; + + /// The total validator era payout for the last `HISTORY_DEPTH` eras. + /// + /// Eras that haven't finished yet or has been removed doesn't have reward. + #[pallet::storage] + #[pallet::getter(fn eras_validator_reward)] + pub type ErasValidatorReward = StorageMap<_, Twox64Concat, EraIndex, BalanceOf>; + + /// Rewards for the last `HISTORY_DEPTH` eras. + /// If reward hasn't been set or has been removed then 0 reward is returned. + #[pallet::storage] + #[pallet::getter(fn eras_reward_points)] + pub type ErasRewardPoints = + StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; + + /// The total amount staked for the last `HISTORY_DEPTH` eras. + /// If total hasn't been set or has been removed then 0 stake is returned. + #[pallet::storage] + #[pallet::getter(fn eras_total_stake)] + pub type ErasTotalStake = + StorageMap<_, Twox64Concat, EraIndex, BalanceOf, ValueQuery>; + + /// Mode of era forcing. + #[pallet::storage] + #[pallet::getter(fn force_era)] + pub type ForceEra = StorageValue<_, Forcing, ValueQuery>; + + /// The percentage of the slash that is distributed to reporters. + /// + /// The rest of the slashed value is handled by the `Slash`. + #[pallet::storage] + #[pallet::getter(fn slash_reward_fraction)] + pub type SlashRewardFraction = StorageValue<_, Perbill, ValueQuery>; + + /// The amount of currency given to reporters of a slash event which was + /// canceled by extraordinary circumstances (e.g. governance). + #[pallet::storage] + #[pallet::getter(fn canceled_payout)] + pub type CanceledSlashPayout = StorageValue<_, BalanceOf, ValueQuery>; + + /// All unapplied slashes that are queued for later. + #[pallet::storage] + pub type UnappliedSlashes = StorageMap< + _, + Twox64Concat, + EraIndex, + Vec>>, + ValueQuery, + >; + + /// A mapping from still-bonded eras to the first session index of that era. + /// + /// Must contains information for eras for the range: + /// `[active_era - bounding_duration; active_era]` + #[pallet::storage] + pub(crate) type BondedEras = + StorageValue<_, Vec<(EraIndex, SessionIndex)>, ValueQuery>; + + /// All slashing events on validators, mapped by era to the highest slash proportion + /// and slash value of the era. + #[pallet::storage] + pub(crate) type ValidatorSlashInEra = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + (Perbill, BalanceOf), + >; + + /// All slashing events on nominators, mapped by era to the highest slash value of the era. + #[pallet::storage] + pub(crate) type NominatorSlashInEra = + StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, BalanceOf>; + + /// Slashing spans for stash accounts. + #[pallet::storage] + pub(crate) type SlashingSpans = + StorageMap<_, Twox64Concat, T::AccountId, slashing::SlashingSpans>; + + /// Records information about the maximum slash of a stash within a slashing span, + /// as well as how much reward has been paid out. + #[pallet::storage] + pub(crate) type SpanSlash = StorageMap< + _, + Twox64Concat, + (T::AccountId, slashing::SpanIndex), + slashing::SpanRecord>, + ValueQuery, + >; + + /// The earliest era for which we have a pending, unapplied slash. + #[pallet::storage] + pub(crate) type EarliestUnappliedSlash = StorageValue<_, EraIndex>; + + /// The last planned session scheduled by the session pallet. + /// + /// This is basically in sync with the call to [`pallet_session::SessionManager::new_session`]. + #[pallet::storage] + #[pallet::getter(fn current_planned_session)] + pub type CurrentPlannedSession = StorageValue<_, SessionIndex, ValueQuery>; + + /// True if network has been upgraded to this version. + /// Storage version of the pallet. + /// + /// This is set to v7.0.0 for new networks. + #[pallet::storage] + pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; + + // The next storage items collectively comprise the voter bags: a composite data structure + // designed to allow efficient iteration of the top N voters by stake, mostly. See + // `mod voter_bags` for details. + // + // In each of these items, voter bags are indexed by their upper weight threshold. + + /// How many voters are registered. + #[pallet::storage] + pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; + + /// Which bag currently contains a particular voter. + /// + /// This may not be the appropriate bag for the voter's weight if they have been rewarded or + /// slashed. + #[pallet::storage] + pub(crate) type VoterBagFor = + StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; + + /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which + /// mainly exists to store head and tail pointers to the appropriate nodes. + #[pallet::storage] + pub(crate) type VoterBags = + StorageMap<_, Twox64Concat, VoteWeight, voter_bags::Bag>; + + /// Voter nodes store links forward and back within their respective bags, the stash id, and + /// whether the voter is a validator or nominator. + /// + /// There is nothing in this map directly identifying to which bag a particular node belongs. + /// However, the `Node` data structure has helpers which can provide that information. + #[pallet::storage] + pub(crate) type VoterNodes = + StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::Node>; + + // End of voter bags data. + + /// The threshold for when users can start calling `chill_other` for other validators / nominators. + /// The threshold is compared to the actual number of validators / nominators (`CountFor*`) in + /// the system compared to the configured max (`Max*Count`). + #[pallet::storage] + pub(crate) type ChillThreshold = StorageValue<_, Percent, OptionQuery>; + + #[pallet::genesis_config] + pub struct GenesisConfig { + pub history_depth: u32, + pub validator_count: u32, + pub minimum_validator_count: u32, + pub invulnerables: Vec, + pub force_era: Forcing, + pub slash_reward_fraction: Perbill, + pub canceled_payout: BalanceOf, + pub stakers: Vec<(T::AccountId, T::AccountId, BalanceOf, StakerStatus)>, + pub min_nominator_bond: BalanceOf, + pub min_validator_bond: BalanceOf, + } + + #[cfg(feature = "std")] + impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + history_depth: 84u32, + validator_count: Default::default(), + minimum_validator_count: Default::default(), + invulnerables: Default::default(), + force_era: Default::default(), + slash_reward_fraction: Default::default(), + canceled_payout: Default::default(), + stakers: Default::default(), + min_nominator_bond: Default::default(), + min_validator_bond: Default::default(), + } + } + } + + #[pallet::genesis_build] + impl GenesisBuild for GenesisConfig { + fn build(&self) { + HistoryDepth::::put(self.history_depth); + ValidatorCount::::put(self.validator_count); + MinimumValidatorCount::::put(self.minimum_validator_count); + Invulnerables::::put(&self.invulnerables); + ForceEra::::put(self.force_era); + CanceledSlashPayout::::put(self.canceled_payout); + SlashRewardFraction::::put(self.slash_reward_fraction); + StorageVersion::::put(Releases::V7_0_0); + MinNominatorBond::::put(self.min_nominator_bond); + MinValidatorBond::::put(self.min_validator_bond); + + let mut num_voters: u32 = 0; + for &(ref stash, ref controller, balance, ref status) in &self.stakers { + log!( + trace, + "inserting genesis staker: {:?} => {:?} => {:?}", + stash, + balance, + status + ); + assert!( + T::Currency::free_balance(&stash) >= balance, + "Stash does not have enough balance to bond." + ); + + if let Err(why) = >::bond( + T::Origin::from(Some(stash.clone()).into()), + T::Lookup::unlookup(controller.clone()), + balance, + RewardDestination::Staked, + ) { + // TODO: later on, fix all the tests that trigger these warnings, and + // make these assertions. Genesis stakers should all be correct! + log!(warn, "failed to bond staker at genesis: {:?}.", why); + continue + } + match status { + StakerStatus::Validator => { + if let Err(why) = >::validate( + T::Origin::from(Some(controller.clone()).into()), + Default::default(), + ) { + log!(warn, "failed to validate staker at genesis: {:?}.", why); + } else { + num_voters += 1; + } + }, + StakerStatus::Nominator(votes) => { + if let Err(why) = >::nominate( + T::Origin::from(Some(controller.clone()).into()), + votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), + ) { + log!(warn, "failed to nominate staker at genesis: {:?}.", why); + } else { + num_voters += 1; + } + }, + _ => (), + }; + } + + // all voters are inserted sanely. + assert_eq!( + CounterForVoters::::get(), + num_voters, + "not all genesis stakers were inserted into bags, something is wrong." + ); + } + } + + #[pallet::event] + #[pallet::generate_deposit(pub(crate) fn deposit_event)] + #[pallet::metadata(T::AccountId = "AccountId", BalanceOf = "Balance")] + pub enum Event { + /// The era payout has been set; the first balance is the validator-payout; the second is + /// the remainder from the maximum amount of reward. + /// \[era_index, validator_payout, remainder\] + EraPayout(EraIndex, BalanceOf, BalanceOf), + /// The staker has been rewarded by this amount. \[stash, amount\] + Reward(T::AccountId, BalanceOf), + /// One validator (and its nominators) has been slashed by the given amount. + /// \[validator, amount\] + Slash(T::AccountId, BalanceOf), + /// An old slashing report from a prior era was discarded because it could + /// not be processed. \[session_index\] + OldSlashingReportDiscarded(SessionIndex), + /// A new set of stakers was elected. + StakingElection, + /// An account has bonded this amount. \[stash, amount\] + /// + /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, + /// it will not be emitted for staking rewards when they are added to stake. + Bonded(T::AccountId, BalanceOf), + /// An account has unbonded this amount. \[stash, amount\] + Unbonded(T::AccountId, BalanceOf), + /// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance` + /// from the unlocking queue. \[stash, amount\] + Withdrawn(T::AccountId, BalanceOf), + /// A nominator has been kicked from a validator. \[nominator, stash\] + Kicked(T::AccountId, T::AccountId), + /// The election failed. No new era is planned. + StakingElectionFailed, + /// An account has stopped participating as either a validator or nominator. + /// \[stash\] + Chilled(T::AccountId), + /// Moved an account from one bag to another. \[who, from, to\]. + Rebagged(T::AccountId, VoteWeight, VoteWeight), + } + + #[pallet::error] + pub enum Error { + /// Not a controller account. + NotController, + /// Not a stash account. + NotStash, + /// Stash is already bonded. + AlreadyBonded, + /// Controller is already paired. + AlreadyPaired, + /// Targets cannot be empty. + EmptyTargets, + /// Duplicate index. + DuplicateIndex, + /// Slash record index out of bounds. + InvalidSlashIndex, + /// Can not bond with value less than minimum required. + InsufficientBond, + /// Can not schedule more unlock chunks. + NoMoreChunks, + /// Can not rebond without unlocking chunks. + NoUnlockChunk, + /// Attempting to target a stash that still has funds. + FundedTarget, + /// Invalid era to reward. + InvalidEraToReward, + /// Invalid number of nominations. + InvalidNumberOfNominations, + /// Items are not sorted and unique. + NotSortedAndUnique, + /// Rewards for this era have already been claimed for this validator. + AlreadyClaimed, + /// Incorrect previous history depth input provided. + IncorrectHistoryDepth, + /// Incorrect number of slashing spans provided. + IncorrectSlashingSpans, + /// Internal state has become somehow corrupted and the operation cannot continue. + BadState, + /// Too many nomination targets supplied. + TooManyTargets, + /// A nomination target was supplied that was blocked or otherwise not a validator. + BadTarget, + /// The user has enough bond and thus cannot be chilled forcefully by an external person. + CannotChillOther, + /// There are too many nominators in the system. Governance needs to adjust the staking settings + /// to keep things safe for the runtime. + TooManyNominators, + /// There are too many validators in the system. Governance needs to adjust the staking settings + /// to keep things safe for the runtime. + TooManyValidators, + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_runtime_upgrade() -> Weight { + if StorageVersion::::get() == Releases::V6_0_0 { + migrations::v7::migrate::() + } else { + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + if StorageVersion::::get() == Releases::V6_0_0 { + migrations::v7::pre_migrate::() + } else { + Ok(()) + } + } + + fn on_initialize(_now: BlockNumberFor) -> Weight { + // just return the weight of the on_finalize. + T::DbWeight::get().reads(1) + } + + fn on_finalize(_n: BlockNumberFor) { + // Set the start of the first era. + if let Some(mut active_era) = Self::active_era() { + if active_era.start.is_none() { + let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); + active_era.start = Some(now_as_millis_u64); + // This write only ever happens once, we don't include it in the weight in general + ActiveEra::::put(active_era); + } + } + // `on_finalize` weight is tracked in `on_initialize` + } + + fn integrity_test() { + sp_std::if_std! { + sp_io::TestExternalities::new_empty().execute_with(|| { + assert!( + T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, + "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", + T::SlashDeferDuration::get(), + T::BondingDuration::get(), + ); + + assert!( + T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "Voter bag thresholds must strictly increase", + ); + + assert!( + { + let existential_weight = voter_bags::existential_weight::(); + T::VoterBagThresholds::get() + .first() + .map(|&lowest_threshold| lowest_threshold >= existential_weight) + .unwrap_or(true) + }, + "Smallest bag should not be smaller than existential weight", + ); + }); + } + } + } + + #[pallet::call] + impl Pallet { + /// Take the origin account as a stash and lock up `value` of its balance. `controller` will + /// be the account that controls it. + /// + /// `value` must be more than the `minimum_balance` specified by `T::Currency`. + /// + /// The dispatch origin for this call must be _Signed_ by the stash account. + /// + /// Emits `Bonded`. + /// # + /// - Independent of the arguments. Moderate complexity. + /// - O(1). + /// - Three extra DB entries. + /// + /// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned + /// unless the `origin` falls below _existential deposit_ and gets removed as dust. + /// ------------------ + /// # + #[pallet::weight(T::WeightInfo::bond())] + pub fn bond( + origin: OriginFor, + controller: ::Source, + #[pallet::compact] value: BalanceOf, + payee: RewardDestination, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + + if >::contains_key(&stash) { + Err(Error::::AlreadyBonded)? + } + + let controller = T::Lookup::lookup(controller)?; + + if >::contains_key(&controller) { + Err(Error::::AlreadyPaired)? + } + + // Reject a bond which is considered to be _dust_. + if value < T::Currency::minimum_balance() { + Err(Error::::InsufficientBond)? + } + + frame_system::Pallet::::inc_consumers(&stash).map_err(|_| Error::::BadState)?; + + // You're auto-bonded forever, here. We might improve this by only bonding when + // you actually validate/nominate and remove once you unbond __everything__. + >::insert(&stash, &controller); + >::insert(&stash, payee); + + let current_era = CurrentEra::::get().unwrap_or(0); + let history_depth = Self::history_depth(); + let last_reward_era = current_era.saturating_sub(history_depth); + + let stash_balance = T::Currency::free_balance(&stash); + let value = value.min(stash_balance); + Self::deposit_event(Event::::Bonded(stash.clone(), value)); + let item = StakingLedger { + stash, + total: value, + active: value, + unlocking: vec![], + claimed_rewards: (last_reward_era..current_era).collect(), + }; + Self::update_ledger(&controller, &item); + Ok(()) + } + + /// Add some extra amount that have appeared in the stash `free_balance` into the balance up + /// for staking. + /// + /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. + /// + /// Use this if there are additional funds in your stash account that you wish to bond. + /// Unlike [`bond`](Self::bond) or [`unbond`](Self::unbond) this function does not impose any limitation + /// on the amount that can be added. + /// + /// Emits `Bonded`. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - O(1). + /// # + #[pallet::weight(T::WeightInfo::bond_extra())] + pub fn bond_extra( + origin: OriginFor, + #[pallet::compact] max_additional: BalanceOf, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + + let controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + + let stash_balance = T::Currency::free_balance(&stash); + if let Some(extra) = stash_balance.checked_sub(&ledger.total) { + let extra = extra.min(max_additional); + ledger.total += extra; + ledger.active += extra; + // Last check: the new active amount of ledger must be more than ED. + ensure!( + ledger.active >= T::Currency::minimum_balance(), + Error::::InsufficientBond + ); + + Self::deposit_event(Event::::Bonded(stash.clone(), extra)); + Self::update_ledger(&controller, &ledger); + Self::do_rebag(&stash); + } + Ok(()) + } + + /// Schedule a portion of the stash to be unlocked ready for transfer out after the bond + /// period ends. If this leaves an amount actively bonded less than + /// T::Currency::minimum_balance(), then it is increased to the full amount. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// Once the unlock period is done, you can call `withdraw_unbonded` to actually move + /// the funds out of management ready for transfer. + /// + /// No more than a limited number of unlocking chunks (see `MAX_UNLOCKING_CHUNKS`) + /// can co-exists at the same time. In that case, [`Call::withdraw_unbonded`] need + /// to be called first to remove some of the chunks (if possible). + /// + /// If a user encounters the `InsufficientBond` error when calling this extrinsic, + /// they should call `chill` first in order to free up their bonded funds. + /// + /// Emits `Unbonded`. + /// + /// See also [`Call::withdraw_unbonded`]. + #[pallet::weight(T::WeightInfo::unbond())] + pub fn unbond( + origin: OriginFor, + #[pallet::compact] value: BalanceOf, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks); + + let mut value = value.min(ledger.active); + + if !value.is_zero() { + ledger.active -= value; + + // Avoid there being a dust balance left in the staking system. + if ledger.active < T::Currency::minimum_balance() { + value += ledger.active; + ledger.active = Zero::zero(); + } + + let min_active_bond = if Nominators::::contains_key(&ledger.stash) { + MinNominatorBond::::get() + } else if Validators::::contains_key(&ledger.stash) { + MinValidatorBond::::get() + } else { + Zero::zero() + }; + + // Make sure that the user maintains enough active bond for their role. + // If a user runs into this error, they should chill first. + ensure!(ledger.active >= min_active_bond, Error::::InsufficientBond); + + // Note: in case there is no current era it is fine to bond one era more. + let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); + ledger.unlocking.push(UnlockChunk { value, era }); + Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); + Self::deposit_event(Event::::Unbonded(ledger.stash, value)); + } + Ok(()) + } + + /// Remove any unlocked chunks from the `unlocking` queue from our management. + /// + /// This essentially frees up that balance to be used by the stash account to do + /// whatever it wants. + /// + /// The dispatch origin for this call must be _Signed_ by the controller. + /// + /// Emits `Withdrawn`. + /// + /// See also [`Call::unbond`]. + /// + /// # + /// Complexity O(S) where S is the number of slashing spans to remove + /// NOTE: Weight annotation is the kill scenario, we refund otherwise. + /// # + #[pallet::weight(T::WeightInfo::withdraw_unbonded_kill(*num_slashing_spans))] + pub fn withdraw_unbonded( + origin: OriginFor, + num_slashing_spans: u32, + ) -> DispatchResultWithPostInfo { + let controller = ensure_signed(origin)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let (stash, old_total) = (ledger.stash.clone(), ledger.total); + if let Some(current_era) = Self::current_era() { + ledger = ledger.consolidate_unlocked(current_era) + } + + let post_info_weight = + if ledger.unlocking.is_empty() && ledger.active < T::Currency::minimum_balance() { + // This account must have called `unbond()` with some value that caused the active + // portion to fall below existential deposit + will have no more unlocking chunks + // left. We can now safely remove all staking-related information. + Self::kill_stash(&stash, num_slashing_spans)?; + // Remove the lock. + T::Currency::remove_lock(STAKING_ID, &stash); + // This is worst case scenario, so we use the full weight and return None + None + } else { + // This was the consequence of a partial unbond. just update the ledger and move on. + Self::update_ledger(&controller, &ledger); + + // This is only an update, so we use less overall weight. + Some(T::WeightInfo::withdraw_unbonded_update(num_slashing_spans)) + }; + + // `old_total` should never be less than the new total because + // `consolidate_unlocked` strictly subtracts balance. + if ledger.total < old_total { + // Already checked that this won't overflow by entry condition. + let value = old_total - ledger.total; + Self::deposit_event(Event::::Withdrawn(stash, value)); + } + + Ok(post_info_weight.into()) + } + + /// Declare the desire to validate for the origin controller. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + #[pallet::weight(T::WeightInfo::validate())] + pub fn validate(origin: OriginFor, prefs: ValidatorPrefs) -> DispatchResult { + let controller = ensure_signed(origin)?; + + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); + let stash = &ledger.stash; + + // Only check limits if they are not already a validator. + if !Validators::::contains_key(stash) { + // If this error is reached, we need to adjust the `MinValidatorBond` and start calling `chill_other`. + // Until then, we explicitly block new validators to protect the runtime. + if let Some(max_validators) = MaxValidatorsCount::::get() { + ensure!( + CounterForValidators::::get() < max_validators, + Error::::TooManyValidators + ); + } + } + + Self::do_remove_nominator(stash); + Self::do_add_validator(stash, prefs); + Ok(()) + } + + /// Declare the desire to nominate `targets` for the origin controller. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// # + /// - The transaction's complexity is proportional to the size of `targets` (N) + /// which is capped at CompactAssignments::LIMIT (MAX_NOMINATIONS). + /// - Both the reads and writes follow a similar pattern. + /// # + #[pallet::weight(T::WeightInfo::nominate(targets.len() as u32))] + pub fn nominate( + origin: OriginFor, + targets: Vec<::Source>, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(ledger.active >= MinNominatorBond::::get(), Error::::InsufficientBond); + let stash = &ledger.stash; + + // Only check limits if they are not already a nominator. + if !Nominators::::contains_key(stash) { + // If this error is reached, we need to adjust the `MinNominatorBond` and start calling `chill_other`. + // Until then, we explicitly block new nominators to protect the runtime. + if let Some(max_nominators) = MaxNominatorsCount::::get() { + ensure!( + CounterForNominators::::get() < max_nominators, + Error::::TooManyNominators + ); + } + } + + ensure!(!targets.is_empty(), Error::::EmptyTargets); + ensure!(targets.len() <= T::MAX_NOMINATIONS as usize, Error::::TooManyTargets); + + let old = Nominators::::get(stash).map_or_else(Vec::new, |x| x.targets); + + let targets = targets + .into_iter() + .map(|t| T::Lookup::lookup(t).map_err(DispatchError::from)) + .map(|n| { + n.and_then(|n| { + if old.contains(&n) || !Validators::::get(&n).blocked { + Ok(n) + } else { + Err(Error::::BadTarget.into()) + } + }) + }) + .collect::, _>>()?; + + let nominations = Nominations { + targets, + // Initial nominations are considered submitted at era 0. See `Nominations` doc + submitted_in: Self::current_era().unwrap_or(0), + suppressed: false, + }; + + Self::do_remove_validator(stash); + Self::do_add_nominator(stash, nominations); + Ok(()) + } + + /// Declare no desire to either validate or nominate. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - Contains one read. + /// - Writes are limited to the `origin` account key. + /// # + #[pallet::weight(T::WeightInfo::chill())] + pub fn chill(origin: OriginFor) -> DispatchResult { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + Self::chill_stash(&ledger.stash); + Ok(()) + } + + /// (Re-)set the payment target for a controller. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - Contains a limited number of reads. + /// - Writes are limited to the `origin` account key. + /// --------- + /// - Weight: O(1) + /// - DB Weight: + /// - Read: Ledger + /// - Write: Payee + /// # + #[pallet::weight(T::WeightInfo::set_payee())] + pub fn set_payee( + origin: OriginFor, + payee: RewardDestination, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let stash = &ledger.stash; + >::insert(stash, payee); + Ok(()) + } + + /// (Re-)set the controller of a stash. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - Contains a limited number of reads. + /// - Writes are limited to the `origin` account key. + /// ---------- + /// Weight: O(1) + /// DB Weight: + /// - Read: Bonded, Ledger New Controller, Ledger Old Controller + /// - Write: Bonded, Ledger New Controller, Ledger Old Controller + /// # + #[pallet::weight(T::WeightInfo::set_controller())] + pub fn set_controller( + origin: OriginFor, + controller: ::Source, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + let old_controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; + let controller = T::Lookup::lookup(controller)?; + if >::contains_key(&controller) { + Err(Error::::AlreadyPaired)? + } + if controller != old_controller { + >::insert(&stash, &controller); + if let Some(l) = >::take(&old_controller) { + >::insert(&controller, l); + } + } + Ok(()) + } + + /// Sets the ideal number of validators. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Weight: O(1) + /// Write: Validator Count + /// # + #[pallet::weight(T::WeightInfo::set_validator_count())] + pub fn set_validator_count( + origin: OriginFor, + #[pallet::compact] new: u32, + ) -> DispatchResult { + ensure_root(origin)?; + ValidatorCount::::put(new); + Ok(()) + } + + /// Increments the ideal number of validators. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Same as [`Self::set_validator_count`]. + /// # + #[pallet::weight(T::WeightInfo::set_validator_count())] + pub fn increase_validator_count( + origin: OriginFor, + #[pallet::compact] additional: u32, + ) -> DispatchResult { + ensure_root(origin)?; + ValidatorCount::::mutate(|n| *n += additional); + Ok(()) + } + + /// Scale up the ideal number of validators by a factor. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Same as [`Self::set_validator_count`]. + /// # + #[pallet::weight(T::WeightInfo::set_validator_count())] + pub fn scale_validator_count(origin: OriginFor, factor: Percent) -> DispatchResult { + ensure_root(origin)?; + ValidatorCount::::mutate(|n| *n += factor * *n); + Ok(()) + } + + /// Force there to be no new eras indefinitely. + /// + /// The dispatch origin must be Root. + /// + /// # Warning + /// + /// The election process starts multiple blocks before the end of the era. + /// Thus the election process may be ongoing when this is called. In this case the + /// election will continue until the next era is triggered. + /// + /// # + /// - No arguments. + /// - Weight: O(1) + /// - Write: ForceEra + /// # + #[pallet::weight(T::WeightInfo::force_no_eras())] + pub fn force_no_eras(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + ForceEra::::put(Forcing::ForceNone); + Ok(()) + } + + /// Force there to be a new era at the end of the next session. After this, it will be + /// reset to normal (non-forced) behaviour. + /// + /// The dispatch origin must be Root. + /// + /// # Warning + /// + /// The election process starts multiple blocks before the end of the era. + /// If this is called just before a new era is triggered, the election process may not + /// have enough blocks to get a result. + /// + /// # + /// - No arguments. + /// - Weight: O(1) + /// - Write ForceEra + /// # + #[pallet::weight(T::WeightInfo::force_new_era())] + pub fn force_new_era(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + ForceEra::::put(Forcing::ForceNew); + Ok(()) + } + + /// Set the validators who cannot be slashed (if any). + /// + /// The dispatch origin must be Root. + /// + /// # + /// - O(V) + /// - Write: Invulnerables + /// # + #[pallet::weight(T::WeightInfo::set_invulnerables(invulnerables.len() as u32))] + pub fn set_invulnerables( + origin: OriginFor, + invulnerables: Vec, + ) -> DispatchResult { + ensure_root(origin)?; + >::put(invulnerables); + Ok(()) + } + + /// Force a current staker to become completely unstaked, immediately. + /// + /// The dispatch origin must be Root. + /// + /// # + /// O(S) where S is the number of slashing spans to be removed + /// Reads: Bonded, Slashing Spans, Account, Locks + /// Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Account, Locks + /// Writes Each: SpanSlash * S + /// # + #[pallet::weight(T::WeightInfo::force_unstake(*num_slashing_spans))] + pub fn force_unstake( + origin: OriginFor, + stash: T::AccountId, + num_slashing_spans: u32, + ) -> DispatchResult { + ensure_root(origin)?; + + // Remove all staking-related information. + Self::kill_stash(&stash, num_slashing_spans)?; + + // Remove the lock. + T::Currency::remove_lock(STAKING_ID, &stash); + Ok(()) + } + + /// Force there to be a new era at the end of sessions indefinitely. + /// + /// The dispatch origin must be Root. + /// + /// # Warning + /// + /// The election process starts multiple blocks before the end of the era. + /// If this is called just before a new era is triggered, the election process may not + /// have enough blocks to get a result. + /// + /// # + /// - Weight: O(1) + /// - Write: ForceEra + /// # + #[pallet::weight(T::WeightInfo::force_new_era_always())] + pub fn force_new_era_always(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + ForceEra::::put(Forcing::ForceAlways); + Ok(()) + } + + /// Cancel enactment of a deferred slash. + /// + /// Can be called by the `T::SlashCancelOrigin`. + /// + /// Parameters: era and indices of the slashes for that era to kill. + /// + /// # + /// Complexity: O(U + S) + /// with U unapplied slashes weighted with U=1000 + /// and S is the number of slash indices to be canceled. + /// - Read: Unapplied Slashes + /// - Write: Unapplied Slashes + /// # + #[pallet::weight(T::WeightInfo::cancel_deferred_slash(slash_indices.len() as u32))] + pub fn cancel_deferred_slash( + origin: OriginFor, + era: EraIndex, + slash_indices: Vec, + ) -> DispatchResult { + T::SlashCancelOrigin::ensure_origin(origin)?; + + ensure!(!slash_indices.is_empty(), Error::::EmptyTargets); + ensure!(is_sorted_and_unique(&slash_indices), Error::::NotSortedAndUnique); + + let mut unapplied = ::UnappliedSlashes::get(&era); + let last_item = slash_indices[slash_indices.len() - 1]; + ensure!((last_item as usize) < unapplied.len(), Error::::InvalidSlashIndex); + + for (removed, index) in slash_indices.into_iter().enumerate() { + let index = (index as usize) - removed; + unapplied.remove(index); + } + + ::UnappliedSlashes::insert(&era, &unapplied); + Ok(()) + } + + /// Pay out all the stakers behind a single validator for a single era. + /// + /// - `validator_stash` is the stash account of the validator. Their nominators, up to + /// `T::MaxNominatorRewardedPerValidator`, will also receive their rewards. + /// - `era` may be any era between `[current_era - history_depth; current_era]`. + /// + /// The origin of this call must be _Signed_. Any account can call this function, even if + /// it is not one of the stakers. + /// + /// # + /// - Time complexity: at most O(MaxNominatorRewardedPerValidator). + /// - Contains a limited number of reads and writes. + /// ----------- + /// N is the Number of payouts for the validator (including the validator) + /// Weight: + /// - Reward Destination Staked: O(N) + /// - Reward Destination Controller (Creating): O(N) + /// + /// NOTE: weights are assuming that payouts are made to alive stash account (Staked). + /// Paying even a dead controller is cheaper weight-wise. We don't do any refunds here. + /// # + #[pallet::weight(T::WeightInfo::payout_stakers_alive_staked( + T::MaxNominatorRewardedPerValidator::get() + ))] + pub fn payout_stakers( + origin: OriginFor, + validator_stash: T::AccountId, + era: EraIndex, + ) -> DispatchResultWithPostInfo { + ensure_signed(origin)?; + Self::do_payout_stakers(validator_stash, era) + } + + /// Rebond a portion of the stash scheduled to be unlocked. + /// + /// The dispatch origin must be signed by the controller. + /// + /// # + /// - Time complexity: O(L), where L is unlocking chunks + /// - Bounded by `MAX_UNLOCKING_CHUNKS`. + /// - Storage changes: Can't increase storage, only decrease it. + /// # + #[pallet::weight(T::WeightInfo::rebond(MAX_UNLOCKING_CHUNKS as u32))] + pub fn rebond( + origin: OriginFor, + #[pallet::compact] value: BalanceOf, + ) -> DispatchResultWithPostInfo { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(!ledger.unlocking.is_empty(), Error::::NoUnlockChunk); + + let ledger = ledger.rebond(value); + // Last check: the new active amount of ledger must be more than ED. + ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); + + Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); + Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); + Ok(Some( + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), + ) + .into()) + } + + /// Set `HistoryDepth` value. This function will delete any history information + /// when `HistoryDepth` is reduced. + /// + /// Parameters: + /// - `new_history_depth`: The new history depth you would like to set. + /// - `era_items_deleted`: The number of items that will be deleted by this dispatch. + /// This should report all the storage items that will be deleted by clearing old + /// era history. Needed to report an accurate weight for the dispatch. Trusted by + /// `Root` to report an accurate number. + /// + /// Origin must be root. + /// + /// # + /// - E: Number of history depths removed, i.e. 10 -> 7 = 3 + /// - Weight: O(E) + /// - DB Weight: + /// - Reads: Current Era, History Depth + /// - Writes: History Depth + /// - Clear Prefix Each: Era Stakers, EraStakersClipped, ErasValidatorPrefs + /// - Writes Each: ErasValidatorReward, ErasRewardPoints, ErasTotalStake, ErasStartSessionIndex + /// # + #[pallet::weight(T::WeightInfo::set_history_depth(*_era_items_deleted))] + pub fn set_history_depth( + origin: OriginFor, + #[pallet::compact] new_history_depth: EraIndex, + #[pallet::compact] _era_items_deleted: u32, + ) -> DispatchResult { + ensure_root(origin)?; + if let Some(current_era) = Self::current_era() { + HistoryDepth::::mutate(|history_depth| { + let last_kept = current_era.checked_sub(*history_depth).unwrap_or(0); + let new_last_kept = current_era.checked_sub(new_history_depth).unwrap_or(0); + for era_index in last_kept..new_last_kept { + Self::clear_era_information(era_index); + } + *history_depth = new_history_depth + }) + } + Ok(()) + } + + /// Remove all data structure concerning a staker/stash once its balance is at the minimum. + /// This is essentially equivalent to `withdraw_unbonded` except it can be called by anyone + /// and the target `stash` must have no funds left beyond the ED. + /// + /// This can be called from any origin. + /// + /// - `stash`: The stash account to reap. Its balance must be zero. + /// + /// # + /// Complexity: O(S) where S is the number of slashing spans on the account. + /// DB Weight: + /// - Reads: Stash Account, Bonded, Slashing Spans, Locks + /// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Stash Account, Locks + /// - Writes Each: SpanSlash * S + /// # + #[pallet::weight(T::WeightInfo::reap_stash(*num_slashing_spans))] + pub fn reap_stash( + _origin: OriginFor, + stash: T::AccountId, + num_slashing_spans: u32, + ) -> DispatchResult { + let at_minimum = T::Currency::total_balance(&stash) == T::Currency::minimum_balance(); + ensure!(at_minimum, Error::::FundedTarget); + Self::kill_stash(&stash, num_slashing_spans)?; + T::Currency::remove_lock(STAKING_ID, &stash); + Ok(()) + } + + /// Remove the given nominations from the calling validator. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// - `who`: A list of nominator stash accounts who are nominating this validator which + /// should no longer be nominating this validator. + /// + /// Note: Making this call only makes sense if you first set the validator preferences to + /// block any further nominations. + #[pallet::weight(T::WeightInfo::kick(who.len() as u32))] + pub fn kick( + origin: OriginFor, + who: Vec<::Source>, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let stash = &ledger.stash; + + for nom_stash in who + .into_iter() + .map(T::Lookup::lookup) + .collect::, _>>()? + .into_iter() + { + Nominators::::mutate(&nom_stash, |maybe_nom| { + if let Some(ref mut nom) = maybe_nom { + if let Some(pos) = nom.targets.iter().position(|v| v == stash) { + nom.targets.swap_remove(pos); + Self::deposit_event(Event::::Kicked( + nom_stash.clone(), + stash.clone(), + )); + } + } + }); + } + + Ok(()) + } + + /// Update the various staking limits this pallet. + /// + /// * `min_nominator_bond`: The minimum active bond needed to be a nominator. + /// * `min_validator_bond`: The minimum active bond needed to be a validator. + /// * `max_nominator_count`: The max number of users who can be a nominator at once. + /// When set to `None`, no limit is enforced. + /// * `max_validator_count`: The max number of users who can be a validator at once. + /// When set to `None`, no limit is enforced. + /// + /// Origin must be Root to call this function. + /// + /// NOTE: Existing nominators and validators will not be affected by this update. + /// to kick people under the new limits, `chill_other` should be called. + #[pallet::weight(T::WeightInfo::set_staking_limits())] + pub fn set_staking_limits( + origin: OriginFor, + min_nominator_bond: BalanceOf, + min_validator_bond: BalanceOf, + max_nominator_count: Option, + max_validator_count: Option, + threshold: Option, + ) -> DispatchResult { + ensure_root(origin)?; + MinNominatorBond::::set(min_nominator_bond); + MinValidatorBond::::set(min_validator_bond); + MaxNominatorsCount::::set(max_nominator_count); + MaxValidatorsCount::::set(max_validator_count); + ChillThreshold::::set(threshold); + Ok(()) + } + + /// Declare a `controller` to stop participating as either a validator or nominator. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_, but can be called by anyone. + /// + /// If the caller is the same as the controller being targeted, then no further checks are + /// enforced, and this function behaves just like `chill`. + /// + /// If the caller is different than the controller being targeted, the following conditions + /// must be met: + /// * A `ChillThreshold` must be set and checked which defines how close to the max + /// nominators or validators we must reach before users can start chilling one-another. + /// * A `MaxNominatorCount` and `MaxValidatorCount` must be set which is used to determine + /// how close we are to the threshold. + /// * A `MinNominatorBond` and `MinValidatorBond` must be set and checked, which determines + /// if this is a person that should be chilled because they have not met the threshold + /// bond required. + /// + /// This can be helpful if bond requirements are updated, and we need to remove old users + /// who do not satisfy these requirements. + // TODO: Maybe we can deprecate `chill` in the future. + // https://github.com/paritytech/substrate/issues/9111 + #[pallet::weight(T::WeightInfo::chill_other())] + pub fn chill_other(origin: OriginFor, controller: T::AccountId) -> DispatchResult { + // Anyone can call this function. + let caller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let stash = ledger.stash; + + // In order for one user to chill another user, the following conditions must be met: + // * A `ChillThreshold` is set which defines how close to the max nominators or + // validators we must reach before users can start chilling one-another. + // * A `MaxNominatorCount` and `MaxValidatorCount` which is used to determine how close + // we are to the threshold. + // * A `MinNominatorBond` and `MinValidatorBond` which is the final condition checked to + // determine this is a person that should be chilled because they have not met the + // threshold bond required. + // + // Otherwise, if caller is the same as the controller, this is just like `chill`. + if caller != controller { + let threshold = ChillThreshold::::get().ok_or(Error::::CannotChillOther)?; + let min_active_bond = if Nominators::::contains_key(&stash) { + let max_nominator_count = + MaxNominatorsCount::::get().ok_or(Error::::CannotChillOther)?; + let current_nominator_count = CounterForNominators::::get(); + ensure!( + threshold * max_nominator_count < current_nominator_count, + Error::::CannotChillOther + ); + MinNominatorBond::::get() + } else if Validators::::contains_key(&stash) { + let max_validator_count = + MaxValidatorsCount::::get().ok_or(Error::::CannotChillOther)?; + let current_validator_count = CounterForValidators::::get(); + ensure!( + threshold * max_validator_count < current_validator_count, + Error::::CannotChillOther + ); + MinValidatorBond::::get() + } else { + Zero::zero() + }; + + ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); + } + + Self::chill_stash(&stash); + Ok(()) + } + + /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its + /// stake that it should properly fall into a different bag than its current position. + /// + /// This will adjust its position into the appropriate bag. This will affect its position + /// among the nominator/validator set once the snapshot is prepared for the election. + /// + /// Anyone can call this function about any stash. + #[pallet::weight(T::WeightInfo::rebag())] + pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { + ensure_signed(origin)?; + Pallet::::do_rebag(&stash); + Ok(()) + } + } +} + +/// Check that list is sorted and has no duplicates. +fn is_sorted_and_unique(list: &[u32]) -> bool { + list.windows(2).all(|w| w[0] < w[1]) +} diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index 227043b656eef..332c9ffc39069 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -49,7 +49,7 @@ //! //! Based on research at -use super::{ +use crate::{ BalanceOf, Config, EraIndex, Error, Exposure, NegativeImbalanceOf, Pallet, Perbill, SessionInterface, Store, UnappliedSlash, }; diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 0947a1160febf..44bd84b9a167f 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -27,6 +27,11 @@ use rand_chacha::{ }; use sp_io::hashing::blake2_256; +use crate::voter_bags::VoterList; +use frame_support::{pallet_prelude::*, traits::Currency}; +use sp_runtime::{traits::StaticLookup, Perbill}; +use sp_std::prelude::*; + const SEED: u32 = 0; /// This function removes all validators and nominators from storage. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 11432e6d68e3d..224c98490106b 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -18,10 +18,13 @@ //! Tests for the module. use super::{Event, *}; -use frame_election_provider_support::Support; +use crate::voter_bags::VoterList; +use frame_election_provider_support::{ElectionProvider, Support}; use frame_support::{ assert_noop, assert_ok, - traits::{Currency, OnInitialize, ReservableCurrency}, + dispatch::WithPostDispatchInfo, + pallet_prelude::*, + traits::{Currency, Get, OnInitialize, ReservableCurrency}, weights::{extract_actual_weight, GetDispatchInfo}, }; use mock::*; @@ -30,8 +33,13 @@ use sp_npos_elections::supports_eq_unordered; use sp_runtime::{ assert_eq_error_rate, traits::{BadOrigin, Dispatchable}, + Perbill, Percent, }; -use sp_staking::offence::OffenceDetails; +use sp_staking::{ + offence::{OffenceDetails, OnOffenceHandler}, + SessionIndex, +}; +use sp_std::prelude::*; use substrate_test_utils::assert_eq_uvec; #[test] @@ -3885,7 +3893,7 @@ mod voter_bags { // decrease stake within the range of the current bag assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 - // does not change bags + // does not change bags assert_eq!( get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] @@ -3893,7 +3901,7 @@ mod voter_bags { // reduce stake to the level of a non-existent bag assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 - // creates the bag and moves the voter into it + // creates the bag and moves the voter into it assert_eq!( get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] @@ -3901,7 +3909,7 @@ mod voter_bags { // increase stake by `rebond`-ing to the level of a pre-existing bag assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 - // moves the voter to that bag + // moves the voter to that bag assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); // TODO test rebag directly diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 04d1943164bfb..90c257fea758b 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -1464,12 +1464,7 @@ mod bags { node(voter_11, Some(101), None, bag_1000.bag_upper) ); - assert_eq!( - bag_1000, - Bag { - head: Some(11), tail: Some(11), bag_upper: 1_000 - } - ) + assert_eq!(bag_1000, Bag { head: Some(11), tail: Some(11), bag_upper: 1_000 }) }); } From 530e4d2f3d7ce39fa8074c4560911fe7d3e1377e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 28 Jul 2021 11:11:17 -0700 Subject: [PATCH 080/241] Reduce diff --- frame/staking/src/pallet/impls.rs | 2 +- frame/staking/src/pallet/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 84f7317ee9091..4c6db870d843c 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -560,7 +560,7 @@ impl Pallet { } /// Clear all era information for given era. - pub(super) fn clear_era_information(era_index: EraIndex) { + pub(crate) fn clear_era_information(era_index: EraIndex) { >::remove_prefix(era_index, None); >::remove_prefix(era_index, None); >::remove_prefix(era_index, None); diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 8150d9f7cf2a5..417005e155a81 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -127,7 +127,7 @@ pub mod pallet { type SlashCancelOrigin: EnsureOrigin; /// Interface for interacting with a session pallet. - type SessionInterface: self::SessionInterface; + type SessionInterface: SessionInterface; /// The payout for validators and the system for the current era. /// See [Era payout](./index.html#era-payout). @@ -932,7 +932,7 @@ pub mod pallet { ) -> DispatchResult { let controller = ensure_signed(origin)?; let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks); + ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks,); let mut value = value.min(ledger.active); From df990d3d544a2d6d5da4992b8bfc8cddf5336947 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Sun, 1 Aug 2021 12:54:47 -0700 Subject: [PATCH 081/241] Add comment for test to add --- frame/staking/src/tests.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index bf07858cc4057..9c96caadd7026 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3893,7 +3893,7 @@ mod voter_bags { // decrease stake within the range of the current bag assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 - // does not change bags + // does not change bags assert_eq!( get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] @@ -3901,7 +3901,7 @@ mod voter_bags { // reduce stake to the level of a non-existent bag assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 - // creates the bag and moves the voter into it + // creates the bag and moves the voter into it assert_eq!( get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] @@ -3909,12 +3909,17 @@ mod voter_bags { // increase stake by `rebond`-ing to the level of a pre-existing bag assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 - // moves the voter to that bag + // moves the voter to that bag assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); // TODO test rebag directly }); } + + // #[test] TODO + // fn rebag_head_works() { + // // rebagging the head of a bag results in the old bag having a new head and an overall correct state. + // } } mod election_data_provider { From 1632747000a8ce27e023dcc1620f9735af7230e2 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Sun, 1 Aug 2021 13:15:41 -0700 Subject: [PATCH 082/241] Add in code TODO for update_position efficiency updates --- frame/staking/src/voter_bags.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 90c257fea758b..45e94db1be83b 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -243,6 +243,10 @@ impl VoterList { node.is_misplaced(&weight_of).then(move || { let old_idx = node.bag_upper; + // TODO: there should be a way to move a non-head-tail node to another bag + // with just 1 bag read of the destination bag and zero writes + // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 + // clear the old bag head/tail pointers as necessary if let Some(mut bag) = Bag::::get(node.bag_upper) { bag.remove_node(&node); From 97ca105c8cf13ef1293a2b51b59ed0cc25b07bbb Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 2 Aug 2021 17:42:59 -0700 Subject: [PATCH 083/241] Initial impl compiles --- frame/voter-bags/Cargo.toml | 43 ++ frame/voter-bags/src/lib.rs | 158 +++++++ frame/voter-bags/src/make_bags.rs | 0 frame/voter-bags/src/tests.rs | 0 frame/voter-bags/src/voter_list.rs | 725 +++++++++++++++++++++++++++++ frame/voter-bags/src/weights.rs | 1 + 6 files changed, 927 insertions(+) create mode 100644 frame/voter-bags/Cargo.toml create mode 100644 frame/voter-bags/src/lib.rs create mode 100644 frame/voter-bags/src/make_bags.rs create mode 100644 frame/voter-bags/src/tests.rs create mode 100644 frame/voter-bags/src/voter_list.rs create mode 100644 frame/voter-bags/src/weights.rs diff --git a/frame/voter-bags/Cargo.toml b/frame/voter-bags/Cargo.toml new file mode 100644 index 0000000000000..ea620f0f00e2c --- /dev/null +++ b/frame/voter-bags/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "pallet-voter-bags" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet voter bags" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } +sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } + +log = { version = "0.4.14", default-features = false } + +# Optional imports for benchmarking +frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "frame-election-provider-support/std", + "sp-runtime/std", + "sp-std/std", + "pallet-staking/std", + "log/std", +] +runtime-benchmarks = [ + "frame-benchmarking", +] \ No newline at end of file diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs new file mode 100644 index 0000000000000..a69969388a71f --- /dev/null +++ b/frame/voter-bags/src/lib.rs @@ -0,0 +1,158 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implement a data structure for pallet-staking designed for the properties that: +//! +//! - It's efficient to insert or remove a voter +//! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of +//! voters doesn't particularly matter. + +// use codec::{Decode, Encode}; +use frame_election_provider_support::VoteWeight; +use frame_support::{ + pallet_prelude::*, + traits::{Currency, CurrencyToVote, LockableCurrency}, +}; +use pallet_staking; + +mod voter_list; +pub mod weights; + +pub use weights::WeightInfo; + +use pallet::*; + +use pallet_staking::{AccountIdOf, BalanceOf, VotingDataOf}; + +pub(crate) const LOG_TARGET: &'static str = "runtime::voter_bags"; + +// syntactic sugar for logging. +#[macro_export] +macro_rules! log { + ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { + log::$level!( + target: crate::LOG_TARGET, + concat!("[{:?}] ", $patter), >::block_number() $(, $values)* + ) + }; +} + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(crate) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + pallet_staking::Config { + /// The list of thresholds separating the various voter bags. + /// + /// Voters are separated into unsorted bags according to their vote weight. This specifies + /// the thresholds separating the bags. A voter's bag is the largest bag for which the + /// voter's weight is less than or equal to its upper threshold. + /// + /// When voters are iterated, higher bags are iterated completely before lower bags. This + /// means that iteration is _semi-sorted_: voters of higher weight tend to come before + /// voters of lower weight, but peer voters within a particular bag are sorted in insertion + /// order. + /// + /// # Expressing the constant + /// + /// This constant must be sorted in strictly increasing order. Duplicate items are not + /// permitted. + /// + /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// specified within the bag. For any two threshold lists, if one ends with + /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists + /// will behave identically. + /// + /// # Calculation + /// + /// It is recommended to generate the set of thresholds in a geometric series, such that + /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * + /// constant_ratio).max(threshold[k] + 1)` for all `k`. + /// + /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use + /// them, the `make-bags` feature must be enabled. + /// + /// # Examples + /// + /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, + /// and iteration is strictly in insertion order. + /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is equal to 2. + /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is approximately equal + /// to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will + /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. + /// + /// # Migration + /// + /// In the event that this list ever changes, a copy of the old bags list must be retained. + /// With that `VoterList::migrate` can be called, which will perform the appropriate + /// migration. + #[pallet::constant] + type BVoterBagThresholds: Get<&'static [VoteWeight]>; + + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + } + + /// How many voters are registered. + #[pallet::storage] + pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; + + /// Voter nodes store links forward and back within their respective bags, the stash id, and + /// whether the voter is a validator or nominator. + /// + /// There is nothing in this map directly identifying to which bag a particular node belongs. + /// However, the `Node` data structure has helpers which can provide that information. + #[pallet::storage] + pub(crate) type VoterNodes = + StorageMap<_, Twox64Concat, AccountIdOf, voter_list::Node>; + + /// Which bag currently contains a particular voter. + /// + /// This may not be the appropriate bag for the voter's weight if they have been rewarded or + /// slashed. + #[pallet::storage] + pub(crate) type VoterBagFor = + StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; + + /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which + /// mainly exists to store head and tail pointers to the appropriate nodes. + #[pallet::storage] + pub(crate) type VoterBags = + StorageMap<_, Twox64Concat, VoteWeight, voter_list::Bag>; +} + +/// Compute the existential weight for the specified configuration. +/// +/// Note that this value depends on the current issuance, a quantity known to change over time. +/// This makes the project of computing a static value suitable for inclusion in a static, +/// generated file _excitingly unstable_. +#[cfg(any(feature = "std", feature = "make-bags"))] +pub fn existential_weight() -> VoteWeight { + // use frame_support::traits::{Currency, CurrencyToVote}; + + let existential_deposit = >::minimum_balance(); + let issuance = >::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) +} diff --git a/frame/voter-bags/src/make_bags.rs b/frame/voter-bags/src/make_bags.rs new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/frame/voter-bags/src/tests.rs b/frame/voter-bags/src/tests.rs new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs new file mode 100644 index 0000000000000..84910426c2e9f --- /dev/null +++ b/frame/voter-bags/src/voter_list.rs @@ -0,0 +1,725 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! VoterList implementation. + +use crate::{AccountIdOf, Config}; +use codec::{Decode, Encode}; +use frame_election_provider_support::VoteWeight; +use frame_support::{ensure, traits::Get, DefaultNoBound}; +use pallet_staking as staking; +use sp_runtime::SaturatedConversion; +use sp_std::{ + boxed::Box, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + iter, + marker::PhantomData, +}; + +/// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. +pub type VoterOf = Voter>; + +/// Given a certain vote weight, which bag should contain this voter? +/// +/// Bags are identified by their upper threshold; the value returned by this function is guaranteed +/// to be a member of `T::VoterBagThresholds`. +/// +/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, +/// because in the event that bags are inserted or deleted, the number of affected voters which need +/// to be migrated is smaller. +/// +/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this +/// function behaves as if it does. +fn notional_bag_for(weight: VoteWeight) -> VoteWeight { + let thresholds = T::BVoterBagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| weight > threshold); + thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) +} + +/// Find the upper threshold of the actual bag containing the current voter. +fn current_bag_for(id: &AccountIdOf) -> Option { + crate::VoterBagFor::::try_get(id).ok() +} + +/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. +/// +/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of +/// arbitrary and unbounded length, all having a vote weight within a particular constant range. +/// This structure means that voters can be added and removed in `O(1)` time. +/// +/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. +/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall +/// stake decreases as successive bags are reached. This means that it is valid to truncate +/// iteration at any desired point; only those voters in the lowest bag (who are known to have +/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire +/// for fairness and the requirement for efficiency. +pub struct VoterList(PhantomData); + +impl VoterList { + /// Remove all data associated with the voter list from storage. + pub fn clear() { + crate::CounterForVoters::::kill(); + crate::VoterBagFor::::remove_all(None); + crate::VoterBags::::remove_all(None); + crate::VoterNodes::::remove_all(None); + } + + /// Regenerate voter data from the `Nominators` and `Validators` storage items. + /// + /// This is expensive and should only ever be performed during a migration, never during + /// consensus. + /// + /// Returns the number of voters migrated. + pub fn regenerate() -> u32 { + Self::clear(); + + let nominators_iter = staking::Nominators::::iter().map(|(id, _)| Voter::nominator(id)); + let validators_iter = staking::Validators::::iter().map(|(id, _)| Voter::validator(id)); + let weight_of = staking::Pallet::::weight_of_fn(); + + // Self::insert_many(nominators_iter.chain(validators_iter), weight_of) + 1 + } + + /// Decode the length of the voter list. + pub fn decode_len() -> Option { + let maybe_len = crate::CounterForVoters::::try_get().ok().map(|n| n.saturated_into()); + debug_assert_eq!( + maybe_len.unwrap_or_default(), + crate::VoterNodes::::iter().count(), + "stored length must match count of nodes", + ); + debug_assert_eq!( + maybe_len.unwrap_or_default() as u32, + staking::CounterForNominators::::get() + staking::CounterForValidators::::get(), + "voter count must be sum of validator and nominator count", + ); + maybe_len + } + + /// Iterate over all nodes in all bags in the voter list. + /// + /// Full iteration can be expensive; it's recommended to limit the number of items with + /// `.take(n)`. + pub fn iter() -> impl Iterator> { + // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to + // omit the final bound, we need to ensure that we explicitly include that threshold in the + // list. + // + // It's important to retain the ability to omit the final bound because it makes tests much + // easier; they can just configure `type VoterBagThresholds = ()`. + let thresholds = T::BVoterBagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter.rev()) + } else { + // otherwise, insert it here. + Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) + }; + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) + } + + /// Insert a new voter into the appropriate bag in the voter list. + /// + /// If the voter is already present in the list, their type will be updated. + /// That case is cheaper than inserting a new voter. + pub fn insert_as(account_id: &AccountIdOf, voter_type: VoterType) { + // if this is an update operation we can complete this easily and cheaply + if !Node::::update_voter_type_for(account_id, voter_type) { + // otherwise, we need to insert from scratch + let weight_of = staking::Pallet::::weight_of_fn(); + let voter = Voter { id: account_id.clone(), voter_type }; + Self::insert(voter, weight_of); + } + } + + /// Insert a new voter into the appropriate bag in the voter list. + fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { + Self::insert_many(sp_std::iter::once(voter), weight_of); + } + + /// Insert several voters into the appropriate bags in the voter list. + /// + /// This is more efficient than repeated calls to `Self::insert`. + fn insert_many( + voters: impl IntoIterator>, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + ) -> u32 { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for voter in voters.into_iter() { + let weight = weight_of(&voter.id); + let bag = notional_bag_for::(weight); + crate::log!(debug, "inserting {:?} with weight {} into bag {:?}", voter, weight, bag); + bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); + count += 1; + } + + for (_, bag) in bags { + bag.put(); + } + + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_add(count) + }); + count + } + + /// Remove a voter (by id) from the voter list. + pub fn remove(voter: &AccountIdOf) { + Self::remove_many(sp_std::iter::once(voter)); + } + + /// Remove many voters (by id) from the voter list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + pub fn remove_many<'a>(voters: impl IntoIterator>) { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for voter_id in voters.into_iter() { + let node = match Node::::from_id(voter_id) { + Some(node) => node, + None => continue, + }; + count += 1; + + // clear the bag head/tail pointers as necessary + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + bag.remove_node(&node); + + // now get rid of the node itself + crate::VoterNodes::::remove(voter_id); + crate::VoterBagFor::::remove(voter_id); + } + + for (_, bag) in bags { + bag.put(); + } + + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_sub(count) + }); + } + + /// Update a voter's position in the voter list. + /// + /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub fn update_position_for( + mut node: Node, + weight_of: impl Fn(&AccountIdOf) -> VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { + node.is_misplaced(&weight_of).then(move || { + let old_idx = node.bag_upper; + + // TODO: there should be a way to move a non-head-tail node to another bag + // with just 1 bag read of the destination bag and zero writes + // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 + + // clear the old bag head/tail pointers as necessary + if let Some(mut bag) = Bag::::get(node.bag_upper) { + bag.remove_node(&node); + bag.put(); + } else { + debug_assert!(false, "every node must have an extant bag associated with it"); + crate::log!( + error, + "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", + node.voter.id, + ); + } + + // put the voter into the appropriate new bag + let new_idx = notional_bag_for::(weight_of(&node.voter.id)); + node.bag_upper = new_idx; + let mut bag = Bag::::get_or_make(node.bag_upper); + bag.insert_node(node); + bag.put(); + + (old_idx, new_idx) + }) + } + + /// Migrate the voter list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::VoterBagThresholds` has already been updated. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - No voter is changed unless required to by the difference between the old threshold list + /// and the new. + /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// new threshold set. + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + // we can't check all preconditions, but we can check one + debug_assert!( + crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); + + let mut affected_accounts = BTreeSet::new(); + let mut affected_old_bags = BTreeSet::new(); + + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_set.difference(&old_set).copied() { + let affected_bag = notional_bag_for::(inserted_bag); + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // a removed bag means that all members of that bag must be rebagged + for removed_bag in old_set.difference(&new_set).copied() { + if !affected_old_bags.insert(removed_bag) { + continue + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // migrate the + let weight_of = pallet_staking::Pallet::::weight_of_fn(); + Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); + let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in old_set.difference(&new_set).copied() { + debug_assert!( + !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + "no voter should be present in a removed bag", + ); + crate::VoterBags::::remove(removed_bag); + } + + debug_assert!( + { + let thresholds = T::BVoterBagThresholds::get(); + crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + }, + "all `bag_upper` in storage must be members of the new thresholds", + ); + + num_affected + } + + /// Sanity check the voter list. + /// + /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) + /// is being used, after all other staking data (such as counter) has been updated. It checks + /// that: + /// + /// * Iterate all voters in list and make sure there are no duplicates. + /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. + /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. + /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are + /// checked per *any* update to `VoterList`. + pub(crate) fn sanity_check() -> Result<(), &'static str> { + let mut seen_in_list = BTreeSet::new(); + ensure!( + Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), + "duplicate identified", + ); + + let iter_count = Self::iter().collect::>().len() as u32; + let stored_count = crate::CounterForVoters::::get(); + ensure!(iter_count == stored_count, "iter_count != voter_count"); + + let validators = staking::CounterForValidators::::get(); + let nominators = staking::CounterForNominators::::get(); + ensure!(validators + nominators == stored_count, "validators + nominators != voters"); + + let _ = T::VoterBagThresholds::get() + .into_iter() + .map(|t| Bag::::get(*t).unwrap_or_default()) + .map(|b| b.sanity_check()) + .collect::>()?; + + Ok(()) + } +} + +/// A Bag is a doubly-linked list of voters. +/// +/// Note that we maintain both head and tail pointers. While it would be possible to get away +/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's +/// more desirable to ensure that there is some element of first-come, first-serve to the list's +/// iteration so that there's no incentive to churn voter positioning to improve the chances of +/// appearing within the voter set. +#[derive(DefaultNoBound, Encode, Decode)] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq))] +pub struct Bag { + head: Option>, + tail: Option>, + + #[codec(skip)] + bag_upper: VoteWeight, +} + +impl Bag { + /// Get a bag by its upper vote weight. + pub fn get(bag_upper: VoteWeight) -> Option> { + debug_assert!( + T::BVoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; + bag + }) + } + + /// Get a bag by its upper vote weight or make it, appropriately initialized. + pub fn get_or_make(bag_upper: VoteWeight) -> Bag { + debug_assert!( + T::BVoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) + } + + /// `True` if self is empty. + pub fn is_empty(&self) -> bool { + self.head.is_none() && self.tail.is_none() + } + + /// Put the bag back into storage. + pub fn put(self) { + if self.is_empty() { + crate::VoterBags::::remove(self.bag_upper); + } else { + crate::VoterBags::::insert(self.bag_upper, self); + } + } + + /// Get the head node in this bag. + pub fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + } + + /// Get the tail node in this bag. + pub fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + } + + /// Iterate over the nodes in this bag. + pub fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Insert a new voter into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + fn insert(&mut self, voter: VoterOf) { + self.insert_node(Node:: { voter, prev: None, next: None, bag_upper: self.bag_upper }); + } + + /// Insert a voter node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + fn insert_node(&mut self, mut node: Node) { + if let Some(tail) = &self.tail { + if *tail == node.voter.id { + // this should never happen, but this check prevents a worst case infinite loop + debug_assert!(false, "system logic error: inserting a node who has the id of tail"); + crate::log!(warn, "system logic error: inserting a node who has the id of tail"); + return + }; + } + + let id = node.voter.id.clone(); + + node.prev = self.tail.clone(); + node.next = None; + node.put(); + + // update the previous tail + if let Some(mut old_tail) = self.tail() { + old_tail.next = Some(id.clone()); + old_tail.put(); + } + + // update the internal bag links + if self.head.is_none() { + self.head = Some(id.clone()); + } + self.tail = Some(id.clone()); + + crate::VoterBagFor::::insert(id, self.bag_upper); + } + + /// Remove a voter node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the voter in + /// the first place. Generally, use [`VoterList::remove`] instead. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` + /// to update storage for the bag and `node`. + fn remove_node(&mut self, node: &Node) { + // Update previous node. + if let Some(mut prev) = node.prev() { + prev.next = node.next.clone(); + prev.put(); + } + // Update next node. + if let Some(mut next) = node.next() { + next.prev = node.prev.clone(); + next.put(); + } + + // clear the bag head/tail pointers as necessary + if self.head.as_ref() == Some(&node.voter.id) { + self.head = node.next.clone(); + } + if self.tail.as_ref() == Some(&node.voter.id) { + self.tail = node.prev.clone(); + } + } + + /// Sanity check this bag. + /// + /// Should be called by the call-site, after each mutating operation on a bag. The call site of + /// this struct is always `VoterList`. + /// + /// * Ensures head has no prev. + /// * Ensures tail has no next. + /// * Ensures there are no loops, traversal from head to tail is correct. + fn sanity_check(&self) -> Result<(), &'static str> { + ensure!( + self.head() + .map(|head| head.prev().is_none()) + // if there is no head, then there must not be a tail, meaning that the bag is + // empty. + .unwrap_or_else(|| self.tail.is_none()), + "head has a prev" + ); + + ensure!( + self.tail() + .map(|tail| tail.next().is_none()) + // if there is no tail, then there must not be a head, meaning that the bag is + // empty. + .unwrap_or_else(|| self.head.is_none()), + "tail has a next" + ); + + let mut seen_in_bag = BTreeSet::new(); + ensure!( + self.iter() + .map(|node| node.voter.id) + // each voter is only seen once, thus there is no cycle within a bag + .all(|voter| seen_in_bag.insert(voter)), + "Duplicate found in bag" + ); + + Ok(()) + } +} + +/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq, Clone))] +pub struct Node { + voter: Voter>, + prev: Option>, + next: Option>, + + /// The bag index is not stored in storage, but injected during all fetch operations. + #[codec(skip)] + pub(crate) bag_upper: VoteWeight, +} + +impl Node { + /// Get a node by bag idx and account id. + pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { + debug_assert!( + T::BVoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { + node.bag_upper = bag_upper; + node + }) + } + + /// Get a node by account id. + /// + /// Note that this must perform two storage lookups: one to identify which bag is appropriate, + /// and another to actually fetch the node. + pub fn from_id(account_id: &AccountIdOf) -> Option> { + let bag = current_bag_for::(account_id)?; + Self::get(bag, account_id) + } + + /// Get a node by account id, assuming it's in the same bag as this node. + pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { + Self::get(self.bag_upper, account_id) + } + + /// Put the node back into storage. + pub fn put(self) { + crate::VoterNodes::::insert(self.voter.id.clone(), self); + } + + /// Get the previous node in the bag. + pub fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| self.in_bag(id)) + } + + /// Get the next node in the bag. + pub fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| self.in_bag(id)) + } + + /// Get this voter's voting data. + pub fn voting_data( + &self, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + slashing_spans: &BTreeMap, staking::slashing::SlashingSpans>, + ) -> Option> { + let voter_weight = weight_of(&self.voter.id); + match self.voter.voter_type { + VoterType::Validator => + Some((self.voter.id.clone(), voter_weight, sp_std::vec![self.voter.id.clone()])), + VoterType::Nominator => { + let staking::Nominations { submitted_in, mut targets, .. } = + staking::Nominators::::get(&self.voter.id)?; + // Filter out nomination targets which were nominated before the most recent + // slashing span. + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + + (!targets.is_empty()).then(move || (self.voter.id.clone(), voter_weight, targets)) + }, + } + } + + /// `true` when this voter is in the wrong bag. + pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { + notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper + } + + /// Update the voter type associated with a particular node by id. + /// + /// This updates storage immediately. + /// + /// Returns whether the voter existed and was successfully updated. + pub fn update_voter_type_for(account_id: &AccountIdOf, voter_type: VoterType) -> bool { + let node = Self::from_id(account_id); + let existed = node.is_some(); + if let Some(mut node) = node { + node.voter.voter_type = voter_type; + node.put(); + } + existed + } + + /// Get the upper threshold of the bag that this node _should_ be in, given its vote weight. + /// + /// This is a helper intended only for benchmarking and should not be used in production. + #[cfg(any(test, feature = "runtime-benchmarks"))] + pub fn proper_bag_for(&self) -> VoteWeight { + let weight_of = crate::Pallet::::weight_of_fn(); + let current_weight = weight_of(&self.voter.id); + notional_bag_for::(current_weight) + } + + /// Get the underlying voter. + #[cfg(any(test, feature = "runtime-benchmarks"))] + pub fn voter(&self) -> &Voter { + &self.voter + } +} + +/// Fundamental information about a voter. +#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord, sp_runtime::RuntimeDebug)] +pub struct Voter { + /// Account Id of this voter + pub id: AccountId, + /// Whether the voter is a validator or nominator + pub voter_type: VoterType, +} + +impl Voter { + pub fn nominator(id: AccountId) -> Self { + Self { id, voter_type: VoterType::Nominator } + } + + pub fn validator(id: AccountId) -> Self { + Self { id, voter_type: VoterType::Validator } + } +} + +/// Type of voter. +/// +/// Similar to [`crate::StakerStatus`], but somewhat more limited. +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum VoterType { + Validator, + Nominator, +} diff --git a/frame/voter-bags/src/weights.rs b/frame/voter-bags/src/weights.rs new file mode 100644 index 0000000000000..5afdff89ab389 --- /dev/null +++ b/frame/voter-bags/src/weights.rs @@ -0,0 +1 @@ +pub trait WeightInfo {} From 44538b6f76a0b9108f5bc3f4293858d808af80fe Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:01:49 -0700 Subject: [PATCH 084/241] impl StakingVoterListStub --- Cargo.lock | 15 ++ Cargo.toml | 1 + frame/staking/src/lib.rs | 56 +++++- frame/voter-bags/src/lib.rs | 60 ++++-- frame/voter-bags/src/make_bags.rs | 13 ++ frame/voter-bags/src/voter_list.rs | 297 ++++++++++++++--------------- 6 files changed, 279 insertions(+), 163 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17a754a78dc0a..8ed7728089346 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5789,6 +5789,21 @@ dependencies = [ "sp-storage", ] +[[package]] +name = "pallet-voter-bags" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-election-provider-support", + "frame-support", + "frame-system", + "log", + "pallet-staking", + "parity-scale-codec", + "sp-runtime", + "sp-std", +] + [[package]] name = "parity-db" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index dd4c5bf14647b..4b51297f7cafd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,6 +130,7 @@ members = [ "frame/uniques", "frame/utility", "frame/vesting", + "frame/voter-bags", "primitives/api", "primitives/api/proc-macro", "primitives/api/test", diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 993fe349a1045..6021dcec4ca0a 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -341,8 +341,8 @@ type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; -type AccountIdOf = ::AccountId; -type VotingDataOf = (AccountIdOf, VoteWeight, Vec>); +pub type AccountIdOf = ::AccountId; +pub type VotingDataOf = (AccountIdOf, VoteWeight, Vec>); /// Information regarding the active era (era in used in session). #[derive(Encode, Decode, RuntimeDebug)] @@ -800,3 +800,55 @@ where R::is_known_offence(offenders, time_slot) } } + +/// Trait to be implemented by a voter list provider. +pub trait VoterListProvider { + /// Returns iterator over voter list, which can have `take` called on it. + fn get_voters() -> Box>>; + /// Hook for updating the list when a voter is added, their voter type is changed, + /// or their weight changes. + fn on_voter_update(voter: T::AccountId); + /// Hook for removing a voter from the list. + fn on_voter_removed(voter: T::AccountId); +} + +/// A simple voter list implementation that does not require any additional pallets. +struct StakingVoterListStub; +impl VoterListProvider for StakingVoterListStub { + /// Returns iterator over voter list, which can have `take` called on it. + fn get_voters() -> Box>> { + let weight_of = Pallet::::weight_of_fn(); + let vals = >::iter().map(move |(validator, _)| + (validator.clone(), weight_of(&validator), vec![validator.clone()]) + ); + + // Collect all slashing spans into a BTreeMap for further queries. + let slashing_spans = >::iter().collect::>(); + + let weight_of = Pallet::::weight_of_fn(); + let noms = Nominators::::iter().filter_map(move |(nominator, nominations)|{ + let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; + + // Filter out nomination targets which were nominated before the most recent + // slashing span. + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + + if !targets.is_empty() { + let vote_weight = weight_of(&nominator); + Some((nominator, vote_weight, targets)) + } else { + None + } + }); + + Box::new(vals.chain(noms)) + } + /// Hook for updating a voter in the list (unused). + fn on_voter_update(_voter: T::AccountId) {} + /// Hook for removing a voter from the list. + fn on_voter_removed(_voter: T::AccountId) {} +} \ No newline at end of file diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index a69969388a71f..afbe8983b3edd 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -27,6 +27,7 @@ use frame_support::{ pallet_prelude::*, traits::{Currency, CurrencyToVote, LockableCurrency}, }; +use frame_system::{ensure_signed, pallet_prelude::*}; use pallet_staking; mod voter_list; @@ -38,6 +39,8 @@ use pallet::*; use pallet_staking::{AccountIdOf, BalanceOf, VotingDataOf}; +use voter_list::VoterList; + pub(crate) const LOG_TARGET: &'static str = "runtime::voter_bags"; // syntactic sugar for logging. @@ -61,6 +64,9 @@ pub mod pallet { #[pallet::config] pub trait Config: frame_system::Config + pallet_staking::Config { + /// The overarching event type. + type Event: From> + IsType<::Event>; + /// The list of thresholds separating the various voter bags. /// /// Voters are separated into unsorted bags according to their vote weight. This specifies @@ -141,18 +147,48 @@ pub mod pallet { #[pallet::storage] pub(crate) type VoterBags = StorageMap<_, Twox64Concat, VoteWeight, voter_list::Bag>; + + #[pallet::event] + #[pallet::generate_deposit(pub(crate) fn deposit_event)] + #[pallet::metadata(T::AccountId = "AccountId")] + pub enum Event { + /// Moved an account from one bag to another. \[who, from, to\]. + Rebagged(T::AccountId, VoteWeight, VoteWeight), + } + + #[pallet::call] + impl Pallet { + /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its + /// stake that it should properly fall into a different bag than its current position. + /// + /// This will adjust its position into the appropriate bag. This will affect its position + /// among the nominator/validator set once the snapshot is prepared for the election. + /// + /// Anyone can call this function about any stash. + // #[pallet::weight(T::WeightInfo::rebag())] + #[pallet::weight(123456789)] // TODO + pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { + ensure_signed(origin)?; + Pallet::::do_rebag(&stash); + Ok(()) + } + } } -/// Compute the existential weight for the specified configuration. -/// -/// Note that this value depends on the current issuance, a quantity known to change over time. -/// This makes the project of computing a static value suitable for inclusion in a static, -/// generated file _excitingly unstable_. -#[cfg(any(feature = "std", feature = "make-bags"))] -pub fn existential_weight() -> VoteWeight { - // use frame_support::traits::{Currency, CurrencyToVote}; - - let existential_deposit = >::minimum_balance(); - let issuance = >::total_issuance(); - T::CurrencyToVote::to_vote(existential_deposit, issuance) +impl Pallet { + /// Move a stash account from one bag to another, depositing an event on success. + /// + /// If the stash changed bags, returns `Some((from, to))`. + pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let maybe_movement = voter_list::Node::::from_id(&stash).and_then(|node| { + let weight_of = pallet_staking::Pallet::::weight_of_fn(); + VoterList::update_position_for(node, weight_of) + }); + if let Some((from, to)) = maybe_movement { + Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); + }; + maybe_movement + } } diff --git a/frame/voter-bags/src/make_bags.rs b/frame/voter-bags/src/make_bags.rs index e69de29bb2d1d..a5e14c3f0c13b 100644 --- a/frame/voter-bags/src/make_bags.rs +++ b/frame/voter-bags/src/make_bags.rs @@ -0,0 +1,13 @@ +/// Compute the existential weight for the specified configuration. +/// +/// Note that this value depends on the current issuance, a quantity known to change over time. +/// This makes the project of computing a static value suitable for inclusion in a static, +/// generated file _excitingly unstable_. +#[cfg(any(feature = "std", feature = "make-bags"))] +pub fn existential_weight() -> VoteWeight { + // use frame_support::traits::{Currency, CurrencyToVote}; + + let existential_deposit = >::minimum_balance(); + let issuance = >::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) +} diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs index 84910426c2e9f..015ce327d4e25 100644 --- a/frame/voter-bags/src/voter_list.rs +++ b/frame/voter-bags/src/voter_list.rs @@ -91,8 +91,7 @@ impl VoterList { let validators_iter = staking::Validators::::iter().map(|(id, _)| Voter::validator(id)); let weight_of = staking::Pallet::::weight_of_fn(); - // Self::insert_many(nominators_iter.chain(validators_iter), weight_of) - 1 + Self::insert_many(nominators_iter.chain(validators_iter), weight_of) } /// Decode the length of the voter list. @@ -181,178 +180,178 @@ impl VoterList { count } - /// Remove a voter (by id) from the voter list. - pub fn remove(voter: &AccountIdOf) { - Self::remove_many(sp_std::iter::once(voter)); - } + /// Remove a voter (by id) from the voter list. + pub fn remove(voter: &AccountIdOf) { + Self::remove_many(sp_std::iter::once(voter)); + } - /// Remove many voters (by id) from the voter list. - /// - /// This is more efficient than repeated calls to `Self::remove`. - pub fn remove_many<'a>(voters: impl IntoIterator>) { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for voter_id in voters.into_iter() { - let node = match Node::::from_id(voter_id) { - Some(node) => node, - None => continue, - }; - count += 1; - - // clear the bag head/tail pointers as necessary - let bag = bags - .entry(node.bag_upper) - .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); - bag.remove_node(&node); + /// Remove many voters (by id) from the voter list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + pub fn remove_many<'a>(voters: impl IntoIterator>) { + let mut bags = BTreeMap::new(); + let mut count = 0; - // now get rid of the node itself - crate::VoterNodes::::remove(voter_id); - crate::VoterBagFor::::remove(voter_id); - } + for voter_id in voters.into_iter() { + let node = match Node::::from_id(voter_id) { + Some(node) => node, + None => continue, + }; + count += 1; - for (_, bag) in bags { - bag.put(); - } + // clear the bag head/tail pointers as necessary + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + bag.remove_node(&node); - crate::CounterForVoters::::mutate(|prev_count| { - *prev_count = prev_count.saturating_sub(count) - }); + // now get rid of the node itself + crate::VoterNodes::::remove(voter_id); + crate::VoterBagFor::::remove(voter_id); } - /// Update a voter's position in the voter list. - /// - /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they - /// are moved into the correct bag. - /// - /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. - /// - /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by - /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient - /// to call [`self.remove_many`] followed by [`self.insert_many`]. - pub fn update_position_for( - mut node: Node, - weight_of: impl Fn(&AccountIdOf) -> VoteWeight, - ) -> Option<(VoteWeight, VoteWeight)> { - node.is_misplaced(&weight_of).then(move || { - let old_idx = node.bag_upper; - - // TODO: there should be a way to move a non-head-tail node to another bag - // with just 1 bag read of the destination bag and zero writes - // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 - - // clear the old bag head/tail pointers as necessary - if let Some(mut bag) = Bag::::get(node.bag_upper) { - bag.remove_node(&node); - bag.put(); - } else { - debug_assert!(false, "every node must have an extant bag associated with it"); - crate::log!( + for (_, bag) in bags { + bag.put(); + } + + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_sub(count) + }); + } + + /// Update a voter's position in the voter list. + /// + /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub fn update_position_for( + mut node: Node, + weight_of: impl Fn(&AccountIdOf) -> VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { + node.is_misplaced(&weight_of).then(move || { + let old_idx = node.bag_upper; + + // TODO: there should be a way to move a non-head-tail node to another bag + // with just 1 bag read of the destination bag and zero writes + // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 + + // clear the old bag head/tail pointers as necessary + if let Some(mut bag) = Bag::::get(node.bag_upper) { + bag.remove_node(&node); + bag.put(); + } else { + debug_assert!(false, "every node must have an extant bag associated with it"); + crate::log!( error, "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", node.voter.id, ); - } + } - // put the voter into the appropriate new bag - let new_idx = notional_bag_for::(weight_of(&node.voter.id)); - node.bag_upper = new_idx; - let mut bag = Bag::::get_or_make(node.bag_upper); - bag.insert_node(node); - bag.put(); + // put the voter into the appropriate new bag + let new_idx = notional_bag_for::(weight_of(&node.voter.id)); + node.bag_upper = new_idx; + let mut bag = Bag::::get_or_make(node.bag_upper); + bag.insert_node(node); + bag.put(); - (old_idx, new_idx) - }) - } + (old_idx, new_idx) + }) + } - /// Migrate the voter list from one set of thresholds to another. - /// - /// This should only be called as part of an intentional migration; it's fairly expensive. - /// - /// Returns the number of accounts affected. - /// - /// Preconditions: - /// - /// - `old_thresholds` is the previous list of thresholds. - /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::VoterBagThresholds` has already been updated. - /// - /// Postconditions: - /// - /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. - /// - No voter is changed unless required to by the difference between the old threshold list - /// and the new. - /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the - /// new threshold set. - pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { - // we can't check all preconditions, but we can check one - debug_assert!( - crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), - "not all `bag_upper` currently in storage are members of `old_thresholds`", - ); + /// Migrate the voter list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::VoterBagThresholds` has already been updated. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - No voter is changed unless required to by the difference between the old threshold list + /// and the new. + /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// new threshold set. + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + // we can't check all preconditions, but we can check one + debug_assert!( + crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); - let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); - - let mut affected_accounts = BTreeSet::new(); - let mut affected_old_bags = BTreeSet::new(); - - // a new bag means that all accounts previously using the old bag's threshold must now - // be rebagged - for inserted_bag in new_set.difference(&old_set).copied() { - let affected_bag = notional_bag_for::(inserted_bag); - if !affected_old_bags.insert(affected_bag) { - // If the previous threshold list was [10, 20], and we insert [3, 5], then there's - // no point iterating through bag 10 twice. - continue - } - - if let Some(bag) = Bag::::get(affected_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); - } + let mut affected_accounts = BTreeSet::new(); + let mut affected_old_bags = BTreeSet::new(); + + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_set.difference(&old_set).copied() { + let affected_bag = notional_bag_for::(inserted_bag); + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue } - // a removed bag means that all members of that bag must be rebagged - for removed_bag in old_set.difference(&new_set).copied() { - if !affected_old_bags.insert(removed_bag) { - continue - } + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } - if let Some(bag) = Bag::::get(removed_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); - } + // a removed bag means that all members of that bag must be rebagged + for removed_bag in old_set.difference(&new_set).copied() { + if !affected_old_bags.insert(removed_bag) { + continue } - // migrate the - let weight_of = pallet_staking::Pallet::::weight_of_fn(); - Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); - let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); - - // we couldn't previously remove the old bags because both insertion and removal assume that - // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid - // of them. - // - // it's pretty cheap to iterate this again, because both sets are in-memory and require no - // lookups. - for removed_bag in old_set.difference(&new_set).copied() { - debug_assert!( - !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), - "no voter should be present in a removed bag", - ); - crate::VoterBags::::remove(removed_bag); + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); } + } + // migrate the + let weight_of = pallet_staking::Pallet::::weight_of_fn(); + Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); + let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in old_set.difference(&new_set).copied() { debug_assert!( - { - let thresholds = T::BVoterBagThresholds::get(); - crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) - }, - "all `bag_upper` in storage must be members of the new thresholds", + !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + "no voter should be present in a removed bag", ); - - num_affected + crate::VoterBags::::remove(removed_bag); } + debug_assert!( + { + let thresholds = T::BVoterBagThresholds::get(); + crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + }, + "all `bag_upper` in storage must be members of the new thresholds", + ); + + num_affected + } + /// Sanity check the voter list. /// /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) From 44dcc1df581fe610668e7f943d2e9f40d4acffdb Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:24:29 -0700 Subject: [PATCH 085/241] Sample impl VoterListProvider for VoterList --- frame/staking/src/lib.rs | 59 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 6021dcec4ca0a..d5099967eac2b 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -805,13 +805,20 @@ where pub trait VoterListProvider { /// Returns iterator over voter list, which can have `take` called on it. fn get_voters() -> Box>>; + // Hook for inserting a validator. + fn on_validator_insert(voter: &T::AccountId); + // Hook for inserting a nominator. + fn on_nominator_insert(voter: &T::AccountId); /// Hook for updating the list when a voter is added, their voter type is changed, /// or their weight changes. - fn on_voter_update(voter: T::AccountId); + fn on_voter_update(voter: &T::AccountId); /// Hook for removing a voter from the list. - fn on_voter_removed(voter: T::AccountId); + fn on_voter_removed(voter: &T::AccountId); + /// Sanity check internal state of list. Only meant for debug compilation. + fn sanity_check() -> Result<(), &'static str>; } +// TODO this should have some tests? /// A simple voter list implementation that does not require any additional pallets. struct StakingVoterListStub; impl VoterListProvider for StakingVoterListStub { @@ -847,8 +854,50 @@ impl VoterListProvider for StakingVoterListStub { Box::new(vals.chain(noms)) } + // Hook for inserting a validator. + fn on_validator_insert(_voter: &T::AccountId) {} + // Hook for inserting a nominator. + fn on_nominator_insert(_voter: &T::AccountId) {} /// Hook for updating a voter in the list (unused). - fn on_voter_update(_voter: T::AccountId) {} + fn on_voter_update(_voter: &T::AccountId) {} /// Hook for removing a voter from the list. - fn on_voter_removed(_voter: T::AccountId) {} -} \ No newline at end of file + fn on_voter_removed(_voter: &T::AccountId) {} + fn sanity_check() -> Result<(), &'static str> { Ok(()) } +} + +// below is temp + +use voter_bags::VoterList; +impl VoterListProvider for VoterList { + /// Returns iterator over voter list, which can have `take` called on it. + fn get_voters() -> Box>> { + let weight_of = Pallet::::weight_of_fn(); + // collect all slashing spans into a BTreeMap for further queries. + let slashing_spans = >::iter().collect::>(); + + Box::new(VoterList::::iter() + .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans))) + } + + fn on_validator_insert(voter: &T::AccountId) { + VoterList::::insert_as(voter, voter_bags::VoterType::Validator); + } + + fn on_nominator_insert(voter: &T::AccountId) { + VoterList::::insert_as(voter, voter_bags::VoterType::Nominator); + } + + /// Hook for updating a voter in the list (unused). + fn on_voter_update(voter: &T::AccountId) { + Pallet::::do_rebag(voter); // TODO this will use voter_list rebag + } + + /// Hook for removing a voter from the list. + fn on_voter_removed(voter: &T::AccountId) { + VoterList::::remove(voter) + } + + fn sanity_check() -> Result<(), &'static str> { + VoterList::::sanity_check() + } + } \ No newline at end of file From 79b8dca410ea9062e6b9efd263ebed501103106f Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:09:45 -0700 Subject: [PATCH 086/241] impl VoterListProvider for voter-bags --- frame/staking/src/lib.rs | 19 ++++++--------- frame/staking/src/mock.rs | 1 + frame/staking/src/pallet/impls.rs | 37 +++++++++++++++--------------- frame/staking/src/pallet/mod.rs | 2 ++ frame/voter-bags/src/lib.rs | 35 ++++++++++++++++++++++++++++ frame/voter-bags/src/voter_list.rs | 2 +- 6 files changed, 64 insertions(+), 32 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index d5099967eac2b..6d77e2e4278e6 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -804,7 +804,7 @@ where /// Trait to be implemented by a voter list provider. pub trait VoterListProvider { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters() -> Box>>; + fn get_voters(slashing_spans: BTreeMap, slashing::SlashingSpans>) -> Box>>; // Hook for inserting a validator. fn on_validator_insert(voter: &T::AccountId); // Hook for inserting a nominator. @@ -813,7 +813,7 @@ pub trait VoterListProvider { /// or their weight changes. fn on_voter_update(voter: &T::AccountId); /// Hook for removing a voter from the list. - fn on_voter_removed(voter: &T::AccountId); + fn on_voter_remove(voter: &T::AccountId); /// Sanity check internal state of list. Only meant for debug compilation. fn sanity_check() -> Result<(), &'static str>; } @@ -823,15 +823,12 @@ pub trait VoterListProvider { struct StakingVoterListStub; impl VoterListProvider for StakingVoterListStub { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters() -> Box>> { + fn get_voters(slashing_spans: BTreeMap, slashing::SlashingSpans>) -> Box>> { let weight_of = Pallet::::weight_of_fn(); let vals = >::iter().map(move |(validator, _)| (validator.clone(), weight_of(&validator), vec![validator.clone()]) ); - // Collect all slashing spans into a BTreeMap for further queries. - let slashing_spans = >::iter().collect::>(); - let weight_of = Pallet::::weight_of_fn(); let noms = Nominators::::iter().filter_map(move |(nominator, nominations)|{ let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; @@ -861,7 +858,7 @@ impl VoterListProvider for StakingVoterListStub { /// Hook for updating a voter in the list (unused). fn on_voter_update(_voter: &T::AccountId) {} /// Hook for removing a voter from the list. - fn on_voter_removed(_voter: &T::AccountId) {} + fn on_voter_remove(_voter: &T::AccountId) {} fn sanity_check() -> Result<(), &'static str> { Ok(()) } } @@ -870,10 +867,8 @@ impl VoterListProvider for StakingVoterListStub { use voter_bags::VoterList; impl VoterListProvider for VoterList { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters() -> Box>> { + fn get_voters(slashing_spans: BTreeMap, slashing::SlashingSpans>) -> Box>> { let weight_of = Pallet::::weight_of_fn(); - // collect all slashing spans into a BTreeMap for further queries. - let slashing_spans = >::iter().collect::>(); Box::new(VoterList::::iter() .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans))) @@ -893,11 +888,11 @@ impl VoterListProvider for VoterList { } /// Hook for removing a voter from the list. - fn on_voter_removed(voter: &T::AccountId) { + fn on_voter_remove(voter: &T::AccountId) { VoterList::::remove(voter) } - fn sanity_check() -> Result<(), &'static str> { + fn sanity_check() -> Result<(), &'static str> { VoterList::::sanity_check() } } \ No newline at end of file diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 970b8a0f6e981..bf7374a643c0e 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -278,6 +278,7 @@ impl Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); type VoterBagThresholds = VoterBagThresholds; + type VoterListProvider = crate::VoterList; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 474411895b4ca..ef2b097dd5979 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -36,7 +36,8 @@ use sp_staking::{ offence::{OffenceDetails, OnOffenceHandler}, SessionIndex, }; -use sp_std::{collections::btree_map::BTreeMap, prelude::*}; +use sp_std::{prelude::*, collections::btree_map::BTreeMap}; +use crate::VoterListProvider; use crate::{ log, slashing, @@ -654,13 +655,10 @@ impl Pallet { voter_count: usize, ) -> Vec> { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); - - let weight_of = Self::weight_of_fn(); - // collect all slashing spans into a BTreeMap for further queries. + // Collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - VoterList::::iter() - .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) + T::VoterListProvider::get_voters(slashing_spans) .take(wanted_voters) .collect() } @@ -683,8 +681,8 @@ impl Pallet { CounterForNominators::::mutate(|x| x.saturating_inc()) } Nominators::::insert(who, nominations); - VoterList::::insert_as(who, voter_bags::VoterType::Nominator); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + T::VoterListProvider::on_nominator_insert(who); + debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); } /// This function will remove a nominator from the `Nominators` storage map, @@ -699,8 +697,8 @@ impl Pallet { if Nominators::::contains_key(who) { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + T::VoterListProvider::on_voter_remove(who); + debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); true } else { false @@ -720,8 +718,9 @@ impl Pallet { CounterForValidators::::mutate(|x| x.saturating_inc()) } Validators::::insert(who, prefs); - VoterList::::insert_as(who, voter_bags::VoterType::Validator); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + T::VoterListProvider::on_validator_insert(who); + // VoterList::::insert_as(who, voter_bags::VoterType::Validator); + debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); } /// This function will remove a validator from the `Validators` storage map, @@ -736,8 +735,8 @@ impl Pallet { if Validators::::contains_key(who) { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + T::VoterListProvider::on_voter_remove(who); + debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); true } else { false @@ -780,11 +779,11 @@ impl // check a few counters one last time... debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); - debug_assert_eq!( - voter_count, - VoterList::::decode_len().unwrap_or_default(), - "voter_count must be accurate", - ); + // debug_assert_eq!( + // voter_count, + // T::VoterListProvider::get_voters().count(), + // "voter_count must be accurate", + // ); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index ed518b0a2894b..4998d01fe3754 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -195,6 +195,8 @@ pub mod pallet { /// migration. #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; + + type VoterListProvider: crate::VoterListProvider; } #[pallet::extra_constants] diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index afbe8983b3edd..777572ad5c0ff 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -29,6 +29,9 @@ use frame_support::{ }; use frame_system::{ensure_signed, pallet_prelude::*}; use pallet_staking; +use sp_std::{ + collections::{btree_map::BTreeMap} +}; mod voter_list; pub mod weights; @@ -192,3 +195,35 @@ impl Pallet { maybe_movement } } + +impl pallet_staking::VoterListProvider for Pallet { + /// Returns iterator over voter list, which can have `take` called on it. + fn get_voters(slashing_spans: BTreeMap, pallet_staking::slashing::SlashingSpans>) -> Box>> { + let weight_of = pallet_staking::Pallet::::weight_of_fn(); + + Box::new(VoterList::::iter() + .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans))) + } + + fn on_validator_insert(voter: &T::AccountId) { + VoterList::::insert_as(voter, voter_list::VoterType::Validator); + } + + fn on_nominator_insert(voter: &T::AccountId) { + VoterList::::insert_as(voter, voter_list::VoterType::Nominator); + } + + /// Hook for updating a voter in the list (unused). + fn on_voter_update(voter: &T::AccountId) { + Pallet::::do_rebag(voter); + } + + /// Hook for removing a voter from the list. + fn on_voter_remove(voter: &T::AccountId) { + VoterList::::remove(voter) + } + + fn sanity_check() -> Result<(), &'static str> { + VoterList::::sanity_check() + } +} diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs index 015ce327d4e25..c318beb9c89e1 100644 --- a/frame/voter-bags/src/voter_list.rs +++ b/frame/voter-bags/src/voter_list.rs @@ -682,7 +682,7 @@ impl Node { /// This is a helper intended only for benchmarking and should not be used in production. #[cfg(any(test, feature = "runtime-benchmarks"))] pub fn proper_bag_for(&self) -> VoteWeight { - let weight_of = crate::Pallet::::weight_of_fn(); + let weight_of = staking::Pallet::::weight_of_fn(); let current_weight = weight_of(&self.voter.id); notional_bag_for::(current_weight) } From b3d03304678c9d34e77a84c2965b756c76fe7cea Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:34:41 -0700 Subject: [PATCH 087/241] WIP integrate pallet-voter-bags to staking mock --- Cargo.lock | 1 + frame/staking/Cargo.toml | 1 + frame/staking/src/lib.rs | 36 ++++++++++++++++++++----------- frame/staking/src/mock.rs | 8 ++++++- frame/staking/src/pallet/impls.rs | 8 +++---- frame/staking/src/tests.rs | 8 +++---- frame/voter-bags/src/lib.rs | 25 +++++++++++---------- 7 files changed, 51 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ed7728089346..fe8323bce75e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5558,6 +5558,7 @@ dependencies = [ "pallet-session", "pallet-staking-reward-curve", "pallet-timestamp", + "pallet-voter-bags", "parity-scale-codec", "parking_lot 0.11.1", "paste 1.0.4", diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index bfe2e04475452..03630d19aa557 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -51,6 +51,7 @@ sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elect pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } +pallet-voter-bags = { version = "4.0.0-dev", path = "../voter-bags" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } frame-election-provider-support = { version = "4.0.0-dev", features = ["runtime-benchmarks"], path = "../election-provider-support" } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 6d77e2e4278e6..8051a66b27883 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -804,7 +804,9 @@ where /// Trait to be implemented by a voter list provider. pub trait VoterListProvider { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters(slashing_spans: BTreeMap, slashing::SlashingSpans>) -> Box>>; + fn get_voters( + slashing_spans: BTreeMap, slashing::SlashingSpans>, + ) -> Box>>; // Hook for inserting a validator. fn on_validator_insert(voter: &T::AccountId); // Hook for inserting a nominator. @@ -815,7 +817,7 @@ pub trait VoterListProvider { /// Hook for removing a voter from the list. fn on_voter_remove(voter: &T::AccountId); /// Sanity check internal state of list. Only meant for debug compilation. - fn sanity_check() -> Result<(), &'static str>; + fn sanity_check() -> Result<(), &'static str>; } // TODO this should have some tests? @@ -823,14 +825,16 @@ pub trait VoterListProvider { struct StakingVoterListStub; impl VoterListProvider for StakingVoterListStub { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters(slashing_spans: BTreeMap, slashing::SlashingSpans>) -> Box>> { + fn get_voters( + slashing_spans: BTreeMap, slashing::SlashingSpans>, + ) -> Box>> { let weight_of = Pallet::::weight_of_fn(); - let vals = >::iter().map(move |(validator, _)| + let vals = >::iter().map(move |(validator, _)| { (validator.clone(), weight_of(&validator), vec![validator.clone()]) - ); + }); let weight_of = Pallet::::weight_of_fn(); - let noms = Nominators::::iter().filter_map(move |(nominator, nominations)|{ + let noms = Nominators::::iter().filter_map(move |(nominator, nominations)| { let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; // Filter out nomination targets which were nominated before the most recent @@ -859,7 +863,9 @@ impl VoterListProvider for StakingVoterListStub { fn on_voter_update(_voter: &T::AccountId) {} /// Hook for removing a voter from the list. fn on_voter_remove(_voter: &T::AccountId) {} - fn sanity_check() -> Result<(), &'static str> { Ok(()) } + fn sanity_check() -> Result<(), &'static str> { + Ok(()) + } } // below is temp @@ -867,11 +873,15 @@ impl VoterListProvider for StakingVoterListStub { use voter_bags::VoterList; impl VoterListProvider for VoterList { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters(slashing_spans: BTreeMap, slashing::SlashingSpans>) -> Box>> { + fn get_voters( + slashing_spans: BTreeMap, slashing::SlashingSpans>, + ) -> Box>> { let weight_of = Pallet::::weight_of_fn(); - Box::new(VoterList::::iter() - .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans))) + Box::new( + VoterList::::iter() + .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans)), + ) } fn on_validator_insert(voter: &T::AccountId) { @@ -892,7 +902,7 @@ impl VoterListProvider for VoterList { VoterList::::remove(voter) } - fn sanity_check() -> Result<(), &'static str> { + fn sanity_check() -> Result<(), &'static str> { VoterList::::sanity_check() - } - } \ No newline at end of file + } +} diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index bf7374a643c0e..b4db46a89c679 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -106,6 +106,7 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, + VoterBags: pallet_voter_bags::{Pallet, Call, Storage, Event}, } ); @@ -257,6 +258,11 @@ parameter_types! { pub const VoterBagThresholds: &'static [VoteWeight] = &THRESHOLDS; } +impl pallet_voter_bags::Config for Test { + type Event = Event; + type BVoterBagThresholds = VoterBagThresholds; +} + impl Config for Test { const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; @@ -278,7 +284,7 @@ impl Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); type VoterBagThresholds = VoterBagThresholds; - type VoterListProvider = crate::VoterList; + type VoterListProvider = pallet_voter_bags::Pallet; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index ef2b097dd5979..6fd02eedaf683 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -17,6 +17,7 @@ //! Implementations for the Staking FRAME Pallet. +use crate::VoterListProvider; use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; use frame_support::{ pallet_prelude::*, @@ -36,8 +37,7 @@ use sp_staking::{ offence::{OffenceDetails, OnOffenceHandler}, SessionIndex, }; -use sp_std::{prelude::*, collections::btree_map::BTreeMap}; -use crate::VoterListProvider; +use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use crate::{ log, slashing, @@ -658,9 +658,7 @@ impl Pallet { // Collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - T::VoterListProvider::get_voters(slashing_spans) - .take(wanted_voters) - .collect() + T::VoterListProvider::get_voters(slashing_spans).take(wanted_voters).collect() } /// This is a very expensive function and result should be cached versus being called multiple times. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 9c96caadd7026..c295c311ea8b1 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3893,7 +3893,7 @@ mod voter_bags { // decrease stake within the range of the current bag assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 - // does not change bags + // does not change bags assert_eq!( get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] @@ -3901,7 +3901,7 @@ mod voter_bags { // reduce stake to the level of a non-existent bag assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 - // creates the bag and moves the voter into it + // creates the bag and moves the voter into it assert_eq!( get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] @@ -3909,7 +3909,7 @@ mod voter_bags { // increase stake by `rebond`-ing to the level of a pre-existing bag assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 - // moves the voter to that bag + // moves the voter to that bag assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); // TODO test rebag directly @@ -3918,7 +3918,7 @@ mod voter_bags { // #[test] TODO // fn rebag_head_works() { - // // rebagging the head of a bag results in the old bag having a new head and an overall correct state. + // // rebagging the head of a bag results in the old bag having a new head and an overall correct state. // } } diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index 777572ad5c0ff..3c342468da10e 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -28,20 +28,15 @@ use frame_support::{ traits::{Currency, CurrencyToVote, LockableCurrency}, }; use frame_system::{ensure_signed, pallet_prelude::*}; -use pallet_staking; -use sp_std::{ - collections::{btree_map::BTreeMap} -}; +use pallet_staking::{AccountIdOf, BalanceOf, VotingDataOf, GenesisConfig}; +use sp_std::collections::btree_map::BTreeMap; mod voter_list; pub mod weights; +pub use pallet::*; pub use weights::WeightInfo; -use pallet::*; - -use pallet_staking::{AccountIdOf, BalanceOf, VotingDataOf}; - use voter_list::VoterList; pub(crate) const LOG_TARGET: &'static str = "runtime::voter_bags"; @@ -198,11 +193,15 @@ impl Pallet { impl pallet_staking::VoterListProvider for Pallet { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters(slashing_spans: BTreeMap, pallet_staking::slashing::SlashingSpans>) -> Box>> { + fn get_voters( + slashing_spans: BTreeMap, pallet_staking::slashing::SlashingSpans>, + ) -> Box>> { let weight_of = pallet_staking::Pallet::::weight_of_fn(); - Box::new(VoterList::::iter() - .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans))) + Box::new( + VoterList::::iter() + .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans)), + ) } fn on_validator_insert(voter: &T::AccountId) { @@ -223,7 +222,7 @@ impl pallet_staking::VoterListProvider for Pallet { VoterList::::remove(voter) } - fn sanity_check() -> Result<(), &'static str> { + fn sanity_check() -> Result<(), &'static str> { VoterList::::sanity_check() - } + } } From 4790ad553512be18aaa87fab5b5a453416680bc0 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 2 Aug 2021 21:24:38 -0700 Subject: [PATCH 088/241] the trait `pallet_staking::pallet::pallet::Config` is not implemented for `mock::Test` --- frame/staking/src/lib.rs | 2 +- frame/staking/src/mock.rs | 2 +- frame/voter-bags/src/lib.rs | 15 +++++++-------- frame/voter-bags/src/tests.rs | 0 frame/voter-bags/src/voter_list.rs | 6 +++--- 5 files changed, 12 insertions(+), 13 deletions(-) delete mode 100644 frame/voter-bags/src/tests.rs diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 8051a66b27883..2e5ea3a928a7f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -822,7 +822,7 @@ pub trait VoterListProvider { // TODO this should have some tests? /// A simple voter list implementation that does not require any additional pallets. -struct StakingVoterListStub; +pub struct StakingVoterListStub; impl VoterListProvider for StakingVoterListStub { /// Returns iterator over voter list, which can have `take` called on it. fn get_voters( diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index b4db46a89c679..04815140ae08a 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -284,7 +284,7 @@ impl Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); type VoterBagThresholds = VoterBagThresholds; - type VoterListProvider = pallet_voter_bags::Pallet; + type VoterListProvider = pallet_voter_bags::VoterBagsVoterListProvider; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index 3c342468da10e..9f16d0dd11cf4 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -21,14 +21,10 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -// use codec::{Decode, Encode}; use frame_election_provider_support::VoteWeight; -use frame_support::{ - pallet_prelude::*, - traits::{Currency, CurrencyToVote, LockableCurrency}, -}; -use frame_system::{ensure_signed, pallet_prelude::*}; -use pallet_staking::{AccountIdOf, BalanceOf, VotingDataOf, GenesisConfig}; +use frame_support::traits::{Currency, CurrencyToVote, LockableCurrency}; +use frame_system::ensure_signed; +use pallet_staking::{AccountIdOf, BalanceOf, GenesisConfig, VotingDataOf}; use sp_std::collections::btree_map::BTreeMap; mod voter_list; @@ -55,6 +51,8 @@ macro_rules! log { #[frame_support::pallet] pub mod pallet { use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] @@ -191,7 +189,8 @@ impl Pallet { } } -impl pallet_staking::VoterListProvider for Pallet { +pub struct VoterBagsVoterListProvider; +impl pallet_staking::VoterListProvider for VoterBagsVoterListProvider { /// Returns iterator over voter list, which can have `take` called on it. fn get_voters( slashing_spans: BTreeMap, pallet_staking::slashing::SlashingSpans>, diff --git a/frame/voter-bags/src/tests.rs b/frame/voter-bags/src/tests.rs deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs index c318beb9c89e1..dfa2126ccc54b 100644 --- a/frame/voter-bags/src/voter_list.rs +++ b/frame/voter-bags/src/voter_list.rs @@ -374,9 +374,9 @@ impl VoterList { let stored_count = crate::CounterForVoters::::get(); ensure!(iter_count == stored_count, "iter_count != voter_count"); - let validators = staking::CounterForValidators::::get(); - let nominators = staking::CounterForNominators::::get(); - ensure!(validators + nominators == stored_count, "validators + nominators != voters"); + // let validators = staking::CounterForValidators::::get(); + // let nominators = staking::CounterForNominators::::get(); + // ensure!(validators + nominators == stored_count, "validators + nominators != voters"); let _ = T::VoterBagThresholds::get() .into_iter() From abfc1deb5824b313898b90291af61e8c2a2414c7 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 13:54:42 -0700 Subject: [PATCH 089/241] random --- frame/staking/src/mock.rs | 15 ++++++++------- frame/staking/src/pallet/impls.rs | 4 +++- frame/voter-bags/src/voter_list.rs | 5 +++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 04815140ae08a..1452a9ff18335 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -106,7 +106,7 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, - VoterBags: pallet_voter_bags::{Pallet, Call, Storage, Event}, + // VoterBags: pallet_voter_bags::{Pallet, Call, Storage, Event}, } ); @@ -258,12 +258,12 @@ parameter_types! { pub const VoterBagThresholds: &'static [VoteWeight] = &THRESHOLDS; } -impl pallet_voter_bags::Config for Test { - type Event = Event; - type BVoterBagThresholds = VoterBagThresholds; -} +// impl pallet_voter_bags::Config for Test { +// type Event = Event; +// type BVoterBagThresholds = VoterBagThresholds; +// } -impl Config for Test { +impl crate::pallet::pallet::Config for Test { const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; type UnixTime = Timestamp; @@ -284,7 +284,8 @@ impl Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); type VoterBagThresholds = VoterBagThresholds; - type VoterListProvider = pallet_voter_bags::VoterBagsVoterListProvider; + // type VoterListProvider = pallet_voter_bags::VoterBagsVoterListProvider; + type VoterListProvider = staking::VoterList; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 6fd02eedaf683..50ca03a75f711 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -657,12 +657,14 @@ impl Pallet { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); // Collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - + // - go through 200 validators + // - chain 10,000 nominators T::VoterListProvider::get_voters(slashing_spans).take(wanted_voters).collect() } /// This is a very expensive function and result should be cached versus being called multiple times. pub fn get_npos_targets() -> Vec { + // all current validators to be included Validators::::iter().map(|(v, _)| v).collect::>() } diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs index dfa2126ccc54b..4ae4e381702b8 100644 --- a/frame/voter-bags/src/voter_list.rs +++ b/frame/voter-bags/src/voter_list.rs @@ -587,6 +587,9 @@ pub struct Node { /// The bag index is not stored in storage, but injected during all fetch operations. #[codec(skip)] pub(crate) bag_upper: VoteWeight, + + // TODO maybe + // - store voter data here i.e targets } impl Node { @@ -631,6 +634,8 @@ impl Node { self.next.as_ref().and_then(|id| self.in_bag(id)) } + // TODO + // - voter-list pallet /// Get this voter's voting data. pub fn voting_data( &self, From 863e744ec658fd997e344f2f808eb5c3151cba87 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 4 Aug 2021 00:44:31 +0200 Subject: [PATCH 090/241] pushing my stuff --- Cargo.lock | 1 + frame/staking/Cargo.toml | 2 + frame/staking/src/lib.rs | 113 +- frame/staking/src/migrations.rs | 60 +- frame/staking/src/pallet/impls.rs | 70 +- frame/staking/src/pallet/mod.rs | 110 +- frame/staking/src/voter_bags.rs | 1728 ---------------------------- frame/support/src/traits.rs | 21 + frame/voter-bags/Cargo.toml | 3 +- frame/voter-bags/src/lib.rs | 81 +- frame/voter-bags/src/voter_list.rs | 382 +++--- 11 files changed, 310 insertions(+), 2261 deletions(-) delete mode 100644 frame/staking/src/voter_bags.rs diff --git a/Cargo.lock b/Cargo.lock index fe8323bce75e0..5559a3c649ef2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5801,6 +5801,7 @@ dependencies = [ "log", "pallet-staking", "parity-scale-codec", + "sp-io", "sp-runtime", "sp-std", ] diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 03630d19aa557..52607554f4d39 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -51,6 +51,8 @@ sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elect pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } +# TODO: all staking tests should work with this AND the staking stub. Similar to balances pallet we +# can create two different mocks with a macro helper (low priority). pallet-voter-bags = { version = "4.0.0-dev", path = "../voter-bags" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 2e5ea3a928a7f..1e82c16148b9d 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -286,7 +286,6 @@ mod tests; pub mod inflation; pub mod migrations; pub mod slashing; -pub mod voter_bags; pub mod weights; mod pallet; @@ -341,9 +340,6 @@ type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; -pub type AccountIdOf = ::AccountId; -pub type VotingDataOf = (AccountIdOf, VoteWeight, Vec>); - /// Information regarding the active era (era in used in session). #[derive(Encode, Decode, RuntimeDebug)] pub struct ActiveEraInfo { @@ -498,7 +494,7 @@ impl } if unlocking_balance >= value { - break + break; } } @@ -801,108 +797,21 @@ where } } -/// Trait to be implemented by a voter list provider. -pub trait VoterListProvider { - /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters( - slashing_spans: BTreeMap, slashing::SlashingSpans>, - ) -> Box>>; - // Hook for inserting a validator. - fn on_validator_insert(voter: &T::AccountId); - // Hook for inserting a nominator. - fn on_nominator_insert(voter: &T::AccountId); - /// Hook for updating the list when a voter is added, their voter type is changed, - /// or their weight changes. - fn on_voter_update(voter: &T::AccountId); - /// Hook for removing a voter from the list. - fn on_voter_remove(voter: &T::AccountId); - /// Sanity check internal state of list. Only meant for debug compilation. - fn sanity_check() -> Result<(), &'static str>; -} - -// TODO this should have some tests? /// A simple voter list implementation that does not require any additional pallets. -pub struct StakingVoterListStub; -impl VoterListProvider for StakingVoterListStub { +pub struct StakingVoterListStub(sp_std::marker::PhantomData); +impl frame_support::traits::VoterListProvider for StakingVoterListStub { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters( - slashing_spans: BTreeMap, slashing::SlashingSpans>, - ) -> Box>> { + fn get_voters() -> Box> { let weight_of = Pallet::::weight_of_fn(); - let vals = >::iter().map(move |(validator, _)| { - (validator.clone(), weight_of(&validator), vec![validator.clone()]) - }); - - let weight_of = Pallet::::weight_of_fn(); - let noms = Nominators::::iter().filter_map(move |(nominator, nominations)| { - let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; - - // Filter out nomination targets which were nominated before the most recent - // slashing span. - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); - - if !targets.is_empty() { - let vote_weight = weight_of(&nominator); - Some((nominator, vote_weight, targets)) - } else { - None - } - }); - - Box::new(vals.chain(noms)) + Box::new(Nominators::::iter().map(|(n, _)| n)) } - // Hook for inserting a validator. - fn on_validator_insert(_voter: &T::AccountId) {} - // Hook for inserting a nominator. - fn on_nominator_insert(_voter: &T::AccountId) {} - /// Hook for updating a voter in the list (unused). - fn on_voter_update(_voter: &T::AccountId) {} - /// Hook for removing a voter from the list. - fn on_voter_remove(_voter: &T::AccountId) {} - fn sanity_check() -> Result<(), &'static str> { - Ok(()) + fn count() -> u32 { + CounterForNominators::::get() } -} - -// below is temp - -use voter_bags::VoterList; -impl VoterListProvider for VoterList { - /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters( - slashing_spans: BTreeMap, slashing::SlashingSpans>, - ) -> Box>> { - let weight_of = Pallet::::weight_of_fn(); - - Box::new( - VoterList::::iter() - .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans)), - ) - } - - fn on_validator_insert(voter: &T::AccountId) { - VoterList::::insert_as(voter, voter_bags::VoterType::Validator); - } - - fn on_nominator_insert(voter: &T::AccountId) { - VoterList::::insert_as(voter, voter_bags::VoterType::Nominator); - } - - /// Hook for updating a voter in the list (unused). - fn on_voter_update(voter: &T::AccountId) { - Pallet::::do_rebag(voter); // TODO this will use voter_list rebag - } - - /// Hook for removing a voter from the list. - fn on_voter_remove(voter: &T::AccountId) { - VoterList::::remove(voter) - } - + fn on_insert(voter: &T::AccountId, weight: VoteWeight) {} + fn on_update(voter: &T::AccountId, weight: VoteWeight) {} + fn on_remove(voter: &T::AccountId) {} fn sanity_check() -> Result<(), &'static str> { - VoterList::::sanity_check() + Ok(()) } } diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 83f3ae9b4d83e..0c81279c3bc0a 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -18,36 +18,36 @@ use super::*; -pub mod v8 { - use super::{voter_bags::VoterList, *}; - use frame_support::ensure; - - pub fn pre_migrate() -> Result<(), &'static str> { - ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); - ensure!(VoterList::::iter().count() == 0, "voter list already exists"); - Ok(()) - } - - pub fn migrate() -> Weight { - log!(info, "Migrating staking to Releases::V8_0_0"); - - let migrated = VoterList::::regenerate(); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - - StorageVersion::::put(Releases::V8_0_0); - log!( - info, - "Completed staking migration to Releases::V8_0_0 with {} voters migrated", - migrated, - ); - - T::WeightInfo::regenerate( - CounterForValidators::::get(), - CounterForNominators::::get(), - ) - .saturating_add(T::DbWeight::get().reads(2)) - } -} +// pub mod v8 { +// use super::{voter_bags::VoterList, *}; +// use frame_support::ensure; + +// pub fn pre_migrate() -> Result<(), &'static str> { +// ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); +// ensure!(VoterList::::iter().count() == 0, "voter list already exists"); +// Ok(()) +// } + +// pub fn migrate() -> Weight { +// log!(info, "Migrating staking to Releases::V8_0_0"); + +// let migrated = VoterList::::regenerate(); +// debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + +// StorageVersion::::put(Releases::V8_0_0); +// log!( +// info, +// "Completed staking migration to Releases::V8_0_0 with {} voters migrated", +// migrated, +// ); + +// T::WeightInfo::regenerate( +// CounterForValidators::::get(), +// CounterForNominators::::get(), +// ) +// .saturating_add(T::DbWeight::get().reads(2)) +// } +// } pub mod v7 { use super::*; diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 6fd02eedaf683..76af7d270bf14 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -17,13 +17,12 @@ //! Implementations for the Staking FRAME Pallet. -use crate::VoterListProvider; use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; use frame_support::{ pallet_prelude::*, traits::{ Currency, CurrencyToVote, EstimateNextNewSession, Get, Imbalance, LockableCurrency, - OnUnbalanced, UnixTime, WithdrawReasons, + OnUnbalanced, UnixTime, VoterListProvider, WithdrawReasons, }, weights::{Weight, WithPostDispatchInfo}, }; @@ -40,12 +39,9 @@ use sp_staking::{ use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use crate::{ - log, slashing, - voter_bags::{self, VoterList}, - weights::WeightInfo, - ActiveEraInfo, BalanceOf, EraIndex, EraPayout, Exposure, ExposureOf, Forcing, - IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, SessionInterface, - StakingLedger, ValidatorPrefs, VotingDataOf, + log, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, Exposure, + ExposureOf, Forcing, IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, + SessionInterface, StakingLedger, ValidatorPrefs, }; use super::{pallet::*, STAKING_ID}; @@ -138,7 +134,7 @@ impl Pallet { // Nothing to do if they have no reward points. if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()); } // This is the fraction of the total reward that the validator and the @@ -229,8 +225,9 @@ impl Pallet { Self::update_ledger(&controller, &l); r }), - RewardDestination::Account(dest_account) => - Some(T::Currency::deposit_creating(&dest_account, amount)), + RewardDestination::Account(dest_account) => { + Some(T::Currency::deposit_creating(&dest_account, amount)) + } RewardDestination::None => None, } } @@ -258,14 +255,14 @@ impl Pallet { _ => { // Either `Forcing::ForceNone`, // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None - }, + return None; + } } // New era. let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() && - matches!(ForceEra::::get(), Forcing::ForceNew) + if maybe_new_era_validators.is_some() + && matches!(ForceEra::::get(), Forcing::ForceNew) { ForceEra::::put(Forcing::NotForcing); } @@ -446,12 +443,12 @@ impl Pallet { // TODO: this should be simplified #8911 CurrentEra::::put(0); ErasStartSessionIndex::::insert(&0, &start_session_index); - }, + } _ => (), } Self::deposit_event(Event::StakingElectionFailed); - return None + return None; } Self::deposit_event(Event::StakersElected); @@ -653,12 +650,16 @@ impl Pallet { pub fn get_npos_voters( maybe_max_len: Option, voter_count: usize, - ) -> Vec> { + ) -> Vec<(T::AccountId, VoteWeight, Vec)> { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); // Collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - T::VoterListProvider::get_voters(slashing_spans).take(wanted_voters).collect() + let unfiltered_voters: Vec = + T::VoterListProvider::get_voters().take(wanted_voters).collect(); + // TODO: filter slashing spans + // concatenate account ids to voter data. + unimplemented!() } /// This is a very expensive function and result should be cached versus being called multiple times. @@ -679,7 +680,7 @@ impl Pallet { CounterForNominators::::mutate(|x| x.saturating_inc()) } Nominators::::insert(who, nominations); - T::VoterListProvider::on_nominator_insert(who); + T::VoterListProvider::on_insert(who, Self::weight_of_fn()(who)); debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); } @@ -695,7 +696,7 @@ impl Pallet { if Nominators::::contains_key(who) { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); - T::VoterListProvider::on_voter_remove(who); + T::VoterListProvider::on_remove(who); debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); true } else { @@ -716,8 +717,7 @@ impl Pallet { CounterForValidators::::mutate(|x| x.saturating_inc()) } Validators::::insert(who, prefs); - T::VoterListProvider::on_validator_insert(who); - // VoterList::::insert_as(who, voter_bags::VoterType::Validator); + T::VoterListProvider::on_insert(who, Self::weight_of_fn()(who)); debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); } @@ -733,29 +733,13 @@ impl Pallet { if Validators::::contains_key(who) { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); - T::VoterListProvider::on_voter_remove(who); + T::VoterListProvider::on_remove(who); debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); true } else { false } } - - /// Move a stash account from one bag to another, depositing an event on success. - /// - /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { - // if no voter at that node, don't do anything. - // the caller just wasted the fee to call this. - let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { - let weight_of = Self::weight_of_fn(); - VoterList::update_position_for(node, weight_of) - }); - if let Some((from, to)) = maybe_movement { - Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); - }; - maybe_movement - } } impl @@ -797,7 +781,7 @@ impl let target_count = CounterForValidators::::get() as usize; if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big") + return Err("Target snapshot too big"); } let weight = ::DbWeight::get().reads(target_count as u64); @@ -1061,7 +1045,7 @@ where add_db_reads_writes(1, 0); if active_era.is_none() { // This offence need not be re-submitted. - return consumed_weight + return consumed_weight; } active_era.expect("value checked not to be `None`; qed").index }; @@ -1107,7 +1091,7 @@ where // Skip if the validator is invulnerable. if invulnerables.contains(stash) { - continue + continue; } let unapplied = slashing::compute_slash::(slashing::SlashParams { diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 4998d01fe3754..3beedbddb6e92 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -22,7 +22,7 @@ use frame_support::{ pallet_prelude::*, traits::{ Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, LockIdentifier, - LockableCurrency, OnUnbalanced, UnixTime, + LockableCurrency, OnUnbalanced, UnixTime, VoterListProvider, }, weights::{ constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}, @@ -42,10 +42,10 @@ mod impls; pub use impls::*; use crate::{ - log, migrations, slashing, voter_bags, weights::WeightInfo, AccountIdOf, ActiveEraInfo, - BalanceOf, EraIndex, EraPayout, EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, - Nominations, PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakerStatus, - StakingLedger, UnappliedSlash, UnlockChunk, ValidatorPrefs, + log, migrations, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, + EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, + Releases, RewardDestination, SessionInterface, StakerStatus, StakingLedger, UnappliedSlash, + UnlockChunk, ValidatorPrefs, }; pub const MAX_UNLOCKING_CHUNKS: usize = 32; @@ -196,7 +196,7 @@ pub mod pallet { #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; - type VoterListProvider: crate::VoterListProvider; + type VoterListProvider: VoterListProvider; } #[pallet::extra_constants] @@ -493,41 +493,6 @@ pub mod pallet { #[pallet::storage] pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; - // The next storage items collectively comprise the voter bags: a composite data structure - // designed to allow efficient iteration of the top N voters by stake, mostly. See - // `mod voter_bags` for details. - // - // In each of these items, voter bags are indexed by their upper weight threshold. - - /// How many voters are registered. - #[pallet::storage] - pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; - - /// Which bag currently contains a particular voter. - /// - /// This may not be the appropriate bag for the voter's weight if they have been rewarded or - /// slashed. - #[pallet::storage] - pub(crate) type VoterBagFor = - StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; - - /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which - /// mainly exists to store head and tail pointers to the appropriate nodes. - #[pallet::storage] - pub(crate) type VoterBags = - StorageMap<_, Twox64Concat, VoteWeight, voter_bags::Bag>; - - /// Voter nodes store links forward and back within their respective bags, the stash id, and - /// whether the voter is a validator or nominator. - /// - /// There is nothing in this map directly identifying to which bag a particular node belongs. - /// However, the `Node` data structure has helpers which can provide that information. - #[pallet::storage] - pub(crate) type VoterNodes = - StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::Node>; - - // End of voter bags data. - /// The threshold for when users can start calling `chill_other` for other validators / nominators. /// The threshold is compared to the actual number of validators / nominators (`CountFor*`) in /// the system compared to the configured max (`Max*Count`). @@ -603,7 +568,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue + continue; } match status { StakerStatus::Validator => { @@ -615,7 +580,7 @@ pub mod pallet { } else { num_voters += 1; } - }, + } StakerStatus::Nominator(votes) => { if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), @@ -625,14 +590,14 @@ pub mod pallet { } else { num_voters += 1; } - }, + } _ => (), }; } // all voters are inserted sanely. assert_eq!( - CounterForVoters::::get(), + T::VoterListProvider::count(), num_voters, "not all genesis stakers were inserted into bags, something is wrong." ); @@ -768,35 +733,6 @@ pub mod pallet { } // `on_finalize` weight is tracked in `on_initialize` } - - fn integrity_test() { - sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| { - assert!( - T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, - "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", - T::SlashDeferDuration::get(), - T::BondingDuration::get(), - ); - - assert!( - T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "Voter bag thresholds must strictly increase", - ); - - assert!( - { - let existential_weight = voter_bags::existential_weight::(); - T::VoterBagThresholds::get() - .first() - .map(|&lowest_threshold| lowest_threshold >= existential_weight) - .unwrap_or(true) - }, - "Smallest bag should not be smaller than existential weight", - ); - }); - } - } } #[pallet::call] @@ -903,9 +839,9 @@ pub mod pallet { Error::::InsufficientBond ); + T::VoterListProvider::on_update(&stash, Self::weight_of_fn()(&ledger.stash)); Self::deposit_event(Event::::Bonded(stash.clone(), extra)); Self::update_ledger(&controller, &ledger); - Self::do_rebag(&stash); } Ok(()) } @@ -965,7 +901,7 @@ pub mod pallet { let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); Self::update_ledger(&controller, &ledger); - Self::do_rebag(&ledger.stash); + T::VoterListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); Self::deposit_event(Event::::Unbonded(ledger.stash, value)); } Ok(()) @@ -1458,11 +1394,11 @@ pub mod pallet { Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); Self::update_ledger(&controller, &ledger); - Self::do_rebag(&ledger.stash); + T::VoterListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. Ok(Some( - 35 * WEIGHT_PER_MICROS + - 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + - T::DbWeight::get().reads_writes(3, 2), + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), ) .into()) } @@ -1679,20 +1615,6 @@ pub mod pallet { Self::chill_stash(&stash); Ok(()) } - - /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its - /// stake that it should properly fall into a different bag than its current position. - /// - /// This will adjust its position into the appropriate bag. This will affect its position - /// among the nominator/validator set once the snapshot is prepared for the election. - /// - /// Anyone can call this function about any stash. - #[pallet::weight(T::WeightInfo::rebag())] - pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { - ensure_signed(origin)?; - Pallet::::do_rebag(&stash); - Ok(()) - } } } diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs deleted file mode 100644 index 45e94db1be83b..0000000000000 --- a/frame/staking/src/voter_bags.rs +++ /dev/null @@ -1,1728 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Implement a data structure designed for the properties that: -//! -//! - It's efficient to insert or remove a voter -//! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of -//! voters doesn't particularly matter. - -use codec::{Decode, Encode}; -use frame_support::{ensure, traits::Get, DefaultNoBound}; -use sp_runtime::SaturatedConversion; -use sp_std::{ - boxed::Box, - collections::{btree_map::BTreeMap, btree_set::BTreeSet}, - iter, - marker::PhantomData, -}; - -use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, - VoteWeight, VoterBagFor, VotingDataOf, -}; - -/// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. -pub type VoterOf = Voter>; - -/// Given a certain vote weight, which bag should contain this voter? -/// -/// Bags are identified by their upper threshold; the value returned by this function is guaranteed -/// to be a member of `T::VoterBagThresholds`. -/// -/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, -/// because in the event that bags are inserted or deleted, the number of affected voters which need -/// to be migrated is smaller. -/// -/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this -/// function behaves as if it does. -fn notional_bag_for(weight: VoteWeight) -> VoteWeight { - let thresholds = T::VoterBagThresholds::get(); - let idx = thresholds.partition_point(|&threshold| weight > threshold); - thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) -} - -/// Find the upper threshold of the actual bag containing the current voter. -fn current_bag_for(id: &AccountIdOf) -> Option { - VoterBagFor::::try_get(id).ok() -} - -/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. -/// -/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of -/// arbitrary and unbounded length, all having a vote weight within a particular constant range. -/// This structure means that voters can be added and removed in `O(1)` time. -/// -/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. -/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall -/// stake decreases as successive bags are reached. This means that it is valid to truncate -/// iteration at any desired point; only those voters in the lowest bag (who are known to have -/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire -/// for fairness and the requirement for efficiency. -pub struct VoterList(PhantomData); - -impl VoterList { - /// Remove all data associated with the voter list from storage. - pub fn clear() { - crate::CounterForVoters::::kill(); - crate::VoterBagFor::::remove_all(None); - crate::VoterBags::::remove_all(None); - crate::VoterNodes::::remove_all(None); - } - - /// Regenerate voter data from the `Nominators` and `Validators` storage items. - /// - /// This is expensive and should only ever be performed during a migration, never during - /// consensus. - /// - /// Returns the number of voters migrated. - pub fn regenerate() -> u32 { - Self::clear(); - - let nominators_iter = Nominators::::iter().map(|(id, _)| Voter::nominator(id)); - let validators_iter = Validators::::iter().map(|(id, _)| Voter::validator(id)); - let weight_of = Pallet::::weight_of_fn(); - - Self::insert_many(nominators_iter.chain(validators_iter), weight_of) - } - - /// Decode the length of the voter list. - pub fn decode_len() -> Option { - let maybe_len = crate::CounterForVoters::::try_get().ok().map(|n| n.saturated_into()); - debug_assert_eq!( - maybe_len.unwrap_or_default(), - crate::VoterNodes::::iter().count(), - "stored length must match count of nodes", - ); - debug_assert_eq!( - // TODO: this case will fail in migration pre check - maybe_len.unwrap_or_default() as u32, - crate::CounterForNominators::::get() + crate::CounterForValidators::::get(), - "voter count must be sum of validator and nominator count", - ); - maybe_len - } - - /// Iterate over all nodes in all bags in the voter list. - /// - /// Full iteration can be expensive; it's recommended to limit the number of items with - /// `.take(n)`. - pub fn iter() -> impl Iterator> { - // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to - // omit the final bound, we need to ensure that we explicitly include that threshold in the - // list. - // - // It's important to retain the ability to omit the final bound because it makes tests much - // easier; they can just configure `type VoterBagThresholds = ()`. - let thresholds = T::VoterBagThresholds::get(); - let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { - // in the event that they included it, we can just pass the iterator through unchanged. - Box::new(iter.rev()) - } else { - // otherwise, insert it here. - Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) - }; - iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) - } - - /// Insert a new voter into the appropriate bag in the voter list. - /// - /// If the voter is already present in the list, their type will be updated. - /// That case is cheaper than inserting a new voter. - pub fn insert_as(account_id: &AccountIdOf, voter_type: VoterType) { - // if this is an update operation we can complete this easily and cheaply - if !Node::::update_voter_type_for(account_id, voter_type) { - // otherwise, we need to insert from scratch - let weight_of = Pallet::::weight_of_fn(); - let voter = Voter { id: account_id.clone(), voter_type }; - Self::insert(voter, weight_of); - } - } - - /// Insert a new voter into the appropriate bag in the voter list. - fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { - Self::insert_many(sp_std::iter::once(voter), weight_of); - } - - /// Insert several voters into the appropriate bags in the voter list. - /// - /// This is more efficient than repeated calls to `Self::insert`. - fn insert_many( - voters: impl IntoIterator>, - weight_of: impl Fn(&T::AccountId) -> VoteWeight, - ) -> u32 { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for voter in voters.into_iter() { - let weight = weight_of(&voter.id); - let bag = notional_bag_for::(weight); - crate::log!(debug, "inserting {:?} with weight {} into bag {:?}", voter, weight, bag); - bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); - count += 1; - } - - for (_, bag) in bags { - bag.put(); - } - - crate::CounterForVoters::::mutate(|prev_count| { - *prev_count = prev_count.saturating_add(count) - }); - count - } - - /// Remove a voter (by id) from the voter list. - pub fn remove(voter: &AccountIdOf) { - Self::remove_many(sp_std::iter::once(voter)); - } - - /// Remove many voters (by id) from the voter list. - /// - /// This is more efficient than repeated calls to `Self::remove`. - pub fn remove_many<'a>(voters: impl IntoIterator>) { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for voter_id in voters.into_iter() { - let node = match Node::::from_id(voter_id) { - Some(node) => node, - None => continue, - }; - count += 1; - - // clear the bag head/tail pointers as necessary - let bag = bags - .entry(node.bag_upper) - .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); - bag.remove_node(&node); - - // now get rid of the node itself - crate::VoterNodes::::remove(voter_id); - crate::VoterBagFor::::remove(voter_id); - } - - for (_, bag) in bags { - bag.put(); - } - - crate::CounterForVoters::::mutate(|prev_count| { - *prev_count = prev_count.saturating_sub(count) - }); - } - - /// Update a voter's position in the voter list. - /// - /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they - /// are moved into the correct bag. - /// - /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. - /// - /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by - /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient - /// to call [`self.remove_many`] followed by [`self.insert_many`]. - pub fn update_position_for( - mut node: Node, - weight_of: impl Fn(&AccountIdOf) -> VoteWeight, - ) -> Option<(VoteWeight, VoteWeight)> { - node.is_misplaced(&weight_of).then(move || { - let old_idx = node.bag_upper; - - // TODO: there should be a way to move a non-head-tail node to another bag - // with just 1 bag read of the destination bag and zero writes - // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 - - // clear the old bag head/tail pointers as necessary - if let Some(mut bag) = Bag::::get(node.bag_upper) { - bag.remove_node(&node); - bag.put(); - } else { - debug_assert!(false, "every node must have an extant bag associated with it"); - crate::log!( - error, - "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", - node.voter.id, - ); - } - - // put the voter into the appropriate new bag - let new_idx = notional_bag_for::(weight_of(&node.voter.id)); - node.bag_upper = new_idx; - let mut bag = Bag::::get_or_make(node.bag_upper); - bag.insert_node(node); - bag.put(); - - (old_idx, new_idx) - }) - } - - /// Migrate the voter list from one set of thresholds to another. - /// - /// This should only be called as part of an intentional migration; it's fairly expensive. - /// - /// Returns the number of accounts affected. - /// - /// Preconditions: - /// - /// - `old_thresholds` is the previous list of thresholds. - /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::VoterBagThresholds` has already been updated. - /// - /// Postconditions: - /// - /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. - /// - No voter is changed unless required to by the difference between the old threshold list - /// and the new. - /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the - /// new threshold set. - pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { - // we can't check all preconditions, but we can check one - debug_assert!( - crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), - "not all `bag_upper` currently in storage are members of `old_thresholds`", - ); - - let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); - - let mut affected_accounts = BTreeSet::new(); - let mut affected_old_bags = BTreeSet::new(); - - // a new bag means that all accounts previously using the old bag's threshold must now - // be rebagged - for inserted_bag in new_set.difference(&old_set).copied() { - let affected_bag = notional_bag_for::(inserted_bag); - if !affected_old_bags.insert(affected_bag) { - // If the previous threshold list was [10, 20], and we insert [3, 5], then there's - // no point iterating through bag 10 twice. - continue - } - - if let Some(bag) = Bag::::get(affected_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); - } - } - - // a removed bag means that all members of that bag must be rebagged - for removed_bag in old_set.difference(&new_set).copied() { - if !affected_old_bags.insert(removed_bag) { - continue - } - - if let Some(bag) = Bag::::get(removed_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); - } - } - - // migrate the - let weight_of = Pallet::::weight_of_fn(); - Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); - let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); - - // we couldn't previously remove the old bags because both insertion and removal assume that - // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid - // of them. - // - // it's pretty cheap to iterate this again, because both sets are in-memory and require no - // lookups. - for removed_bag in old_set.difference(&new_set).copied() { - debug_assert!( - !VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), - "no voter should be present in a removed bag", - ); - crate::VoterBags::::remove(removed_bag); - } - - debug_assert!( - { - let thresholds = T::VoterBagThresholds::get(); - crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) - }, - "all `bag_upper` in storage must be members of the new thresholds", - ); - - num_affected - } - - /// Sanity check the voter list. - /// - /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) - /// is being used, after all other staking data (such as counter) has been updated. It checks - /// that: - /// - /// * Iterate all voters in list and make sure there are no duplicates. - /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. - /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. - /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are - /// checked per *any* update to `VoterList`. - pub(super) fn sanity_check() -> Result<(), &'static str> { - let mut seen_in_list = BTreeSet::new(); - ensure!( - Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), - "duplicate identified", - ); - - let iter_count = Self::iter().collect::>().len() as u32; - let stored_count = crate::CounterForVoters::::get(); - ensure!(iter_count == stored_count, "iter_count != voter_count"); - - let validators = crate::CounterForValidators::::get(); - let nominators = crate::CounterForNominators::::get(); - ensure!(validators + nominators == stored_count, "validators + nominators != voters"); - - let _ = T::VoterBagThresholds::get() - .into_iter() - .map(|t| Bag::::get(*t).unwrap_or_default()) - .map(|b| b.sanity_check()) - .collect::>()?; - - Ok(()) - } -} - -/// A Bag is a doubly-linked list of voters. -/// -/// Note that we maintain both head and tail pointers. While it would be possible to get away -/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's -/// more desirable to ensure that there is some element of first-come, first-serve to the list's -/// iteration so that there's no incentive to churn voter positioning to improve the chances of -/// appearing within the voter set. -#[derive(DefaultNoBound, Encode, Decode)] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] -#[cfg_attr(test, derive(PartialEq))] -pub struct Bag { - head: Option>, - tail: Option>, - - #[codec(skip)] - bag_upper: VoteWeight, -} - -impl Bag { - /// Get a bag by its upper vote weight. - pub fn get(bag_upper: VoteWeight) -> Option> { - debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); - crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { - bag.bag_upper = bag_upper; - bag - }) - } - - /// Get a bag by its upper vote weight or make it, appropriately initialized. - pub fn get_or_make(bag_upper: VoteWeight) -> Bag { - debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); - Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) - } - - /// `True` if self is empty. - pub fn is_empty(&self) -> bool { - self.head.is_none() && self.tail.is_none() - } - - /// Put the bag back into storage. - pub fn put(self) { - if self.is_empty() { - crate::VoterBags::::remove(self.bag_upper); - } else { - crate::VoterBags::::insert(self.bag_upper, self); - } - } - - /// Get the head node in this bag. - pub fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) - } - - /// Get the tail node in this bag. - pub fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) - } - - /// Iterate over the nodes in this bag. - pub fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) - } - - /// Insert a new voter into this bag. - /// - /// This is private on purpose because it's naive: it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the nodes. You still need to call - /// `self.put()` after use. - fn insert(&mut self, voter: VoterOf) { - self.insert_node(Node:: { voter, prev: None, next: None, bag_upper: self.bag_upper }); - } - - /// Insert a voter node into this bag. - /// - /// This is private on purpose because it's naive; it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the node. You still need to call - /// `self.put()` after use. - fn insert_node(&mut self, mut node: Node) { - if let Some(tail) = &self.tail { - if *tail == node.voter.id { - // this should never happen, but this check prevents a worst case infinite loop - debug_assert!(false, "system logic error: inserting a node who has the id of tail"); - crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return - }; - } - - let id = node.voter.id.clone(); - - node.prev = self.tail.clone(); - node.next = None; - node.put(); - - // update the previous tail - if let Some(mut old_tail) = self.tail() { - old_tail.next = Some(id.clone()); - old_tail.put(); - } - - // update the internal bag links - if self.head.is_none() { - self.head = Some(id.clone()); - } - self.tail = Some(id.clone()); - - crate::VoterBagFor::::insert(id, self.bag_upper); - } - - /// Remove a voter node from this bag. - /// - /// This is private on purpose because it doesn't check whether this bag contains the voter in - /// the first place. Generally, use [`VoterList::remove`] instead. - /// - /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` - /// to update storage for the bag and `node`. - fn remove_node(&mut self, node: &Node) { - // Update previous node. - if let Some(mut prev) = node.prev() { - prev.next = node.next.clone(); - prev.put(); - } - // Update next node. - if let Some(mut next) = node.next() { - next.prev = node.prev.clone(); - next.put(); - } - - // clear the bag head/tail pointers as necessary - if self.head.as_ref() == Some(&node.voter.id) { - self.head = node.next.clone(); - } - if self.tail.as_ref() == Some(&node.voter.id) { - self.tail = node.prev.clone(); - } - } - - /// Sanity check this bag. - /// - /// Should be called by the call-site, after each mutating operation on a bag. The call site of - /// this struct is always `VoterList`. - /// - /// * Ensures head has no prev. - /// * Ensures tail has no next. - /// * Ensures there are no loops, traversal from head to tail is correct. - fn sanity_check(&self) -> Result<(), &'static str> { - ensure!( - self.head() - .map(|head| head.prev().is_none()) - // if there is no head, then there must not be a tail, meaning that the bag is - // empty. - .unwrap_or_else(|| self.tail.is_none()), - "head has a prev" - ); - - ensure!( - self.tail() - .map(|tail| tail.next().is_none()) - // if there is no tail, then there must not be a head, meaning that the bag is - // empty. - .unwrap_or_else(|| self.head.is_none()), - "tail has a next" - ); - - let mut seen_in_bag = BTreeSet::new(); - ensure!( - self.iter() - .map(|node| node.voter.id) - // each voter is only seen once, thus there is no cycle within a bag - .all(|voter| seen_in_bag.insert(voter)), - "Duplicate found in bag" - ); - - Ok(()) - } -} - -/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. -#[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] -#[cfg_attr(test, derive(PartialEq, Clone))] -pub struct Node { - voter: Voter>, - prev: Option>, - next: Option>, - - /// The bag index is not stored in storage, but injected during all fetch operations. - #[codec(skip)] - pub(crate) bag_upper: VoteWeight, -} - -impl Node { - /// Get a node by bag idx and account id. - pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { - debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); - crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { - node.bag_upper = bag_upper; - node - }) - } - - /// Get a node by account id. - /// - /// Note that this must perform two storage lookups: one to identify which bag is appropriate, - /// and another to actually fetch the node. - pub fn from_id(account_id: &AccountIdOf) -> Option> { - let bag = current_bag_for::(account_id)?; - Self::get(bag, account_id) - } - - /// Get a node by account id, assuming it's in the same bag as this node. - pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { - Self::get(self.bag_upper, account_id) - } - - /// Put the node back into storage. - pub fn put(self) { - crate::VoterNodes::::insert(self.voter.id.clone(), self); - } - - /// Get the previous node in the bag. - pub fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| self.in_bag(id)) - } - - /// Get the next node in the bag. - pub fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| self.in_bag(id)) - } - - /// Get this voter's voting data. - pub fn voting_data( - &self, - weight_of: impl Fn(&T::AccountId) -> VoteWeight, - slashing_spans: &BTreeMap, SlashingSpans>, - ) -> Option> { - let voter_weight = weight_of(&self.voter.id); - match self.voter.voter_type { - VoterType::Validator => - Some((self.voter.id.clone(), voter_weight, sp_std::vec![self.voter.id.clone()])), - VoterType::Nominator => { - let Nominations { submitted_in, mut targets, .. } = - Nominators::::get(&self.voter.id)?; - // Filter out nomination targets which were nominated before the most recent - // slashing span. - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); - - (!targets.is_empty()).then(move || (self.voter.id.clone(), voter_weight, targets)) - }, - } - } - - /// `true` when this voter is in the wrong bag. - pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper - } - - /// Update the voter type associated with a particular node by id. - /// - /// This updates storage immediately. - /// - /// Returns whether the voter existed and was successfully updated. - pub fn update_voter_type_for(account_id: &AccountIdOf, voter_type: VoterType) -> bool { - let node = Self::from_id(account_id); - let existed = node.is_some(); - if let Some(mut node) = node { - node.voter.voter_type = voter_type; - node.put(); - } - existed - } - - /// Get the upper threshold of the bag that this node _should_ be in, given its vote weight. - /// - /// This is a helper intended only for benchmarking and should not be used in production. - #[cfg(any(test, feature = "runtime-benchmarks"))] - pub fn proper_bag_for(&self) -> VoteWeight { - let weight_of = crate::Pallet::::weight_of_fn(); - let current_weight = weight_of(&self.voter.id); - notional_bag_for::(current_weight) - } - - /// Get the underlying voter. - #[cfg(any(test, feature = "runtime-benchmarks"))] - pub fn voter(&self) -> &Voter { - &self.voter - } -} - -/// Fundamental information about a voter. -#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord, sp_runtime::RuntimeDebug)] -pub struct Voter { - /// Account Id of this voter - pub id: AccountId, - /// Whether the voter is a validator or nominator - pub voter_type: VoterType, -} - -impl Voter { - pub fn nominator(id: AccountId) -> Self { - Self { id, voter_type: VoterType::Nominator } - } - - pub fn validator(id: AccountId) -> Self { - Self { id, voter_type: VoterType::Validator } - } -} - -/// Type of voter. -/// -/// Similar to [`crate::StakerStatus`], but somewhat more limited. -#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr(feature = "std", derive(Debug))] -pub enum VoterType { - Validator, - Nominator, -} - -/// Compute the existential weight for the specified configuration. -/// -/// Note that this value depends on the current issuance, a quantity known to change over time. -/// This makes the project of computing a static value suitable for inclusion in a static, -/// generated file _excitingly unstable_. -#[cfg(any(feature = "std", feature = "make-bags"))] -pub fn existential_weight() -> VoteWeight { - use frame_support::traits::{Currency, CurrencyToVote}; - - let existential_deposit = >>::minimum_balance(); - let issuance = >>::total_issuance(); - T::CurrencyToVote::to_vote(existential_deposit, issuance) -} - -/// Support code to ease the process of generating voter bags. -/// -/// The process of adding voter bags to a runtime requires only four steps. -/// -/// 1. Update the runtime definition. -/// -/// ```ignore -/// parameter_types!{ -/// pub const VoterBagThresholds: &'static [u64] = &[]; -/// } -/// -/// impl pallet_staking::Config for Runtime { -/// // -/// type VoterBagThresholds = VoterBagThresholds; -/// } -/// ``` -/// -/// 2. Write a little program to generate the definitions. This can be a near-identical copy of -/// `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime -/// definitions with the various calculations here. -/// -/// 3. Run that program: -/// -/// ```sh,notrust -/// $ cargo run -p node-runtime-voter-bags -- bin/node/runtime/src/voter_bags.rs -/// ``` -/// -/// 4. Update the runtime definition. -/// -/// ```diff,notrust -/// + mod voter_bags; -/// - pub const VoterBagThresholds: &'static [u64] = &[]; -/// + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; -/// ``` -#[cfg(feature = "make-bags")] -pub mod make_bags { - use crate::{voter_bags::existential_weight, Config}; - use frame_election_provider_support::VoteWeight; - use frame_support::traits::Get; - use std::{ - io::Write, - path::{Path, PathBuf}, - }; - - /// Return the path to a header file used in this repository if is exists. - /// - /// Just searches the git working directory root for files matching certain patterns; it's - /// pretty naive. - fn path_to_header_file() -> Option { - let repo = git2::Repository::open_from_env().ok()?; - let workdir = repo.workdir()?; - for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { - let path = workdir.join(file_name); - if path.exists() { - return Some(path) - } - } - None - } - - /// Create an underscore formatter: a formatter which inserts `_` every 3 digits of a number. - fn underscore_formatter() -> num_format::CustomFormat { - num_format::CustomFormat::builder() - .grouping(num_format::Grouping::Standard) - .separator("_") - .build() - .expect("format described here meets all constraints") - } - - /// Compute the constant ratio for the thresholds. - /// - /// This ratio ensures that each bag, with the possible exceptions of certain small ones and the - /// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` - /// space. - pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { - ((VoteWeight::MAX as f64 / existential_weight as f64).ln() / ((n_bags - 1) as f64)).exp() - } - - /// Compute the list of bag thresholds. - /// - /// Returns a list of exactly `n_bags` elements, except in the case of overflow. - /// The first element is always `existential_weight`. - /// The last element is always `VoteWeight::MAX`. - /// - /// All other elements are computed from the previous according to the formula - /// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1); - pub fn thresholds( - existential_weight: VoteWeight, - constant_ratio: f64, - n_bags: usize, - ) -> Vec { - const WEIGHT_LIMIT: f64 = VoteWeight::MAX as f64; - - let mut thresholds = Vec::with_capacity(n_bags); - - if n_bags > 1 { - thresholds.push(existential_weight); - } - - while n_bags > 0 && thresholds.len() < n_bags - 1 { - let last = thresholds.last().copied().unwrap_or(existential_weight); - let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); - if successor < WEIGHT_LIMIT { - thresholds.push(successor as VoteWeight); - } else { - eprintln!("unexpectedly exceeded weight limit; breaking threshold generation loop"); - break - } - } - - thresholds.push(VoteWeight::MAX); - - debug_assert_eq!(thresholds.len(), n_bags); - debug_assert!(n_bags == 0 || thresholds[0] == existential_weight); - debug_assert!(n_bags == 0 || thresholds[thresholds.len() - 1] == VoteWeight::MAX); - - thresholds - } - - /// Write a thresholds module to the path specified. - /// - /// The `output` path should terminate with a Rust module name, i.e. `foo/bar/thresholds.rs`. - /// - /// This generated module contains, in order: - /// - /// - The contents of the header file in this repository's root, if found. - /// - Module documentation noting that this is autogenerated and when. - /// - Some associated constants. - /// - The constant array of thresholds. - pub fn generate_thresholds_module( - n_bags: usize, - output: &Path, - ) -> Result<(), std::io::Error> { - // ensure the file is accessable - if let Some(parent) = output.parent() { - if !parent.exists() { - std::fs::create_dir_all(parent)?; - } - } - - // copy the header file - if let Some(header_path) = path_to_header_file() { - std::fs::copy(header_path, output)?; - } - - // open an append buffer - let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; - let mut buf = std::io::BufWriter::new(file); - - // create underscore formatter and format buffer - let mut num_buf = num_format::Buffer::new(); - let format = underscore_formatter(); - - // module docs - let now = chrono::Utc::now(); - writeln!(buf)?; - writeln!(buf, "//! Autogenerated voter bag thresholds.")?; - writeln!(buf, "//!")?; - writeln!(buf, "//! Generated on {}", now.to_rfc3339())?; - writeln!( - buf, - "//! for the {} runtime.", - ::Version::get().spec_name, - )?; - - // existential weight - let existential_weight = existential_weight::(); - num_buf.write_formatted(&existential_weight, &format); - writeln!(buf)?; - writeln!(buf, "/// Existential weight for this runtime.")?; - writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; - writeln!(buf, "#[allow(unused)]")?; - writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", num_buf.as_str())?; - - // constant ratio - let constant_ratio = constant_ratio(existential_weight, n_bags); - writeln!(buf)?; - writeln!(buf, "/// Constant ratio between bags for this runtime.")?; - writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; - writeln!(buf, "#[allow(unused)]")?; - writeln!(buf, "pub const CONSTANT_RATIO: f64 = {:.16};", constant_ratio)?; - - // thresholds - let thresholds = thresholds(existential_weight, constant_ratio, n_bags); - writeln!(buf)?; - writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; - writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; - for threshold in thresholds { - num_buf.write_formatted(&threshold, &format); - // u64::MAX, with spacers every 3 digits, is 26 characters wide - writeln!(buf, " {:>26},", num_buf.as_str())?; - } - writeln!(buf, "];")?; - - Ok(()) - } -} - -// This is the highest level of abstraction provided by this module. More generic tests are here, -// among those related to `VoterList` struct. -#[cfg(test)] -mod voter_list { - use super::*; - use crate::mock::*; - use frame_support::{assert_ok, assert_storage_noop, traits::Currency}; - - #[test] - fn basic_setup_works() { - use crate::{ - CounterForNominators, CounterForValidators, CounterForVoters, VoterBags, VoterNodes, - }; - let node = |voter, prev, next| Node:: { voter, prev, next, bag_upper: 0 }; - - // make sure ALL relevant data structures are setup correctly. - ExtBuilder::default().build_and_execute(|| { - assert_eq!(CounterForVoters::::get(), 4); - assert_eq!(VoterBagFor::::iter().count(), 4); - assert_eq!(VoterNodes::::iter().count(), 4); - assert_eq!(VoterBags::::iter().count(), 2); - assert_eq!(CounterForValidators::::get(), 3); - assert_eq!(CounterForNominators::::get(), 1); - - assert_eq!( - VoterBags::::get(10).unwrap(), - Bag:: { head: Some(31), tail: Some(31), bag_upper: 0 } - ); - assert_eq!( - VoterBags::::get(1_000).unwrap(), - Bag:: { head: Some(11), tail: Some(101), bag_upper: 0 } - ); - - let weight_of = Staking::weight_of_fn(); - - assert_eq!(weight_of(&11), 1000); - assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); - assert_eq!( - VoterNodes::::get(11).unwrap(), - node(Voter::validator(11), None, Some(21)) - ); - - assert_eq!(weight_of(&21), 1000); - assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); - assert_eq!( - VoterNodes::::get(21).unwrap(), - node(Voter::validator(21), Some(11), Some(101)) - ); - - assert_eq!(weight_of(&31), 1); - assert_eq!(VoterBagFor::::get(31).unwrap(), 10); - assert_eq!( - VoterNodes::::get(31).unwrap(), - node(Voter::validator(31), None, None) - ); - - assert_eq!(weight_of(&41), 1000); - assert_eq!(VoterBagFor::::get(41), None); // this staker is chilled! - assert_eq!(VoterNodes::::get(41), None); - - assert_eq!(weight_of(&101), 500); - assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); - assert_eq!( - VoterNodes::::get(101).unwrap(), - node(Voter::nominator(101), Some(21), None) - ); - - // iteration of the bags would yield: - assert_eq!( - VoterList::::iter().map(|n| n.voter().id).collect::>(), - vec![11, 21, 101, 31], - // ^^ note the order of insertion in genesis! - ); - - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); - }) - } - - #[test] - fn notional_bag_for_works() { - // under a threshold gives the next threshold. - assert_eq!(notional_bag_for::(0), 10); - assert_eq!(notional_bag_for::(9), 10); - assert_eq!(notional_bag_for::(11), 20); - - // at a threshold gives that threshold. - assert_eq!(notional_bag_for::(10), 10); - - let max_explicit_threshold = *::VoterBagThresholds::get().last().unwrap(); - assert_eq!(max_explicit_threshold, 10_000); - // if the max explicit threshold is less than VoteWeight::MAX, - assert!(VoteWeight::MAX > max_explicit_threshold); - // anything above it will belong to the VoteWeight::MAX bag. - assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); - } - - #[test] - fn remove_last_voter_in_bags_cleans_bag() { - ExtBuilder::default().build_and_execute(|| { - // given - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); - - // give 31 more stake to bump it to a new bag. - Balances::make_free_balance_be(&31, 10000); - assert_ok!(Staking::bond_extra(Origin::signed(31), 10000 - 10)); - - // then the bag with bound 10 is wiped from storage. - assert_eq!(get_bags(), vec![(1000, vec![11, 21, 101]), (10_000, vec![31])]); - - // and can be recreated again as needed - bond_validator(77, 777, 10); - assert_eq!( - get_bags(), - vec![(10, vec![77]), (1000, vec![11, 21, 101]), (10_000, vec![31])] - ); - }); - } - - #[test] - fn iteration_is_semi_sorted() { - ExtBuilder::default().build_and_execute(|| { - // add some new validators to the genesis state. - bond_validator(51, 50, 2000); - bond_validator(61, 60, 2000); - - // given - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], - ); - - // then - assert_eq!( - get_voter_list_as_ids(), - vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, // last bag. - ] - ); - - // when adding a voter that has a higher weight than pre-existing voters in the bag - bond_validator(71, 70, 10); - - // then - assert_eq!( - get_voter_list_as_ids(), - vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, - 71, // last bag; the new voter is last, because it is order of insertion - ] - ); - }) - } - - /// This tests that we can `take` x voters, even if that quantity ends midway through a list. - #[test] - fn take_works() { - ExtBuilder::default().build_and_execute(|| { - // add some new validators to the genesis state. - bond_validator(51, 50, 2000); - bond_validator(61, 60, 2000); - - // given - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], - ); - - // when - let iteration = - VoterList::::iter().map(|node| node.voter.id).take(4).collect::>(); - - // then - assert_eq!( - iteration, - vec![ - 51, 61, // best bag, fully iterated - 11, 21, // middle bag, partially iterated - ] - ); - }) - } - - #[test] - fn insert_works() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - // when inserting into an existing bag - bond(42, 43, 1_000); - VoterList::::insert(Voter::<_>::nominator(42), Pallet::::weight_of_fn()); - - // then - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 42, 31]); - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42])]); - - // when inserting into a non-existent bag - bond(422, 433, 1_001); - VoterList::::insert(Voter::<_>::nominator(422), Pallet::::weight_of_fn()); - - // then - assert_eq!(get_voter_list_as_ids(), vec![422, 11, 21, 101, 42, 31]); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42]), (2_000, vec![422])] - ); - }); - } - - #[test] - fn insert_as_works() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - // given - let actual = get_voter_list_as_voters(); - let mut expected: Vec> = vec![ - Voter::<_>::validator(11), - Voter::<_>::validator(21), - Voter::<_>::nominator(101), - Voter::<_>::validator(31), - ]; - assert_eq!(actual, expected); - - // when inserting a new voter - VoterList::::insert_as(&42, VoterType::Nominator); - - // then - let actual = get_voter_list_as_voters(); - expected.push(Voter::<_>::nominator(42)); - assert_eq!(actual, expected); - - // when updating the voter type of an already existing voter - VoterList::::insert_as(&42, VoterType::Validator); - - // then - let actual = get_voter_list_as_voters(); - expected[4] = Voter::<_>::validator(42); - assert_eq!(actual, expected); - }); - } - - #[test] - fn remove_works() { - use crate::{CounterForVoters, VoterBags, VoterNodes}; - - let check_storage = |id, counter, voters, bags| { - assert!(!VoterBagFor::::contains_key(id)); - assert!(!VoterNodes::::contains_key(id)); - assert_eq!(CounterForVoters::::get(), counter); - assert_eq!(VoterBagFor::::iter().count() as u32, counter); - assert_eq!(VoterNodes::::iter().count() as u32, counter); - assert_eq!(get_voter_list_as_ids(), voters); - assert_eq!(get_bags(), bags); - }; - - ExtBuilder::default().build_and_execute_without_check_count(|| { - // when removing a non-existent voter - VoterList::::remove(&42); - assert!(!VoterBagFor::::contains_key(42)); - assert!(!VoterNodes::::contains_key(42)); - - // then nothing changes - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101])]); - assert_eq!(CounterForVoters::::get(), 4); - - // when removing a node from a bag with multiple nodes - VoterList::::remove(&11); - - // then - assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); - check_storage( - 11, - 3, - vec![21, 101, 31], // voter list - vec![(10, vec![31]), (1_000, vec![21, 101])], // bags - ); - - // when removing a node from a bag with only one node: - VoterList::::remove(&31); - - // then - assert_eq!(get_voter_list_as_ids(), vec![21, 101]); - check_storage( - 31, - 2, - vec![21, 101], // voter list - vec![(1_000, vec![21, 101])], // bags - ); - assert!(!VoterBags::::contains_key(10)); // bag 10 is removed - - // remove remaining voters to make sure storage cleans up as expected - VoterList::::remove(&21); - check_storage( - 21, - 1, - vec![101], // voter list - vec![(1_000, vec![101])], // bags - ); - - VoterList::::remove(&101); - check_storage( - 101, - 0, - Vec::::new(), // voter list - vec![], // bags - ); - assert!(!VoterBags::::contains_key(1_000)); // bag 1_000 is removed - - // bags are deleted via removals - assert_eq!(VoterBags::::iter().count(), 0); - // nominator and validator counters are not updated at this level of the api - assert_eq!(crate::CounterForValidators::::get(), 3); - assert_eq!(crate::CounterForNominators::::get(), 1); - }); - } - - #[test] - fn update_position_for_works() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - let weight_of = Staking::weight_of_fn(); - - // given a correctly placed account 31 - let node_31 = Node::::from_id(&31).unwrap(); - assert!(!node_31.is_misplaced(&weight_of)); - - // when account 31 bonds extra and needs to be moved to a non-existing higher bag - // (we can't call bond_extra, because that implicitly calls update_position_for) - set_ledger_and_free_balance(&31, 11); - - assert!(node_31.is_misplaced(&weight_of)); - assert_eq!(weight_of(&31), 11); - - // then updating position moves it to the correct bag - assert_eq!(VoterList::::update_position_for(node_31, &weight_of), Some((10, 20))); - assert_eq!(get_bags(), vec![(20, vec![31]), (1_000, vec![11, 21, 101])]); - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); - - // and if you try and update the position with no change in active stake nothing changes - let node_31 = Node::::from_id(&31).unwrap(); - assert_storage_noop!(assert_eq!( - VoterList::::update_position_for(node_31, &weight_of), - None, - )); - - // when account 31 bonds extra and needs to be moved to an existing higher bag - set_ledger_and_free_balance(&31, 61); - - // then updating positions moves it to the correct bag - let node_31 = Node::::from_id(&31).unwrap(); - assert_eq!( - VoterList::::update_position_for(node_31, &weight_of), - Some((20, 1_000)) - ); - assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101, 31])]); - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); - - // when account 31 bonds extra but should not change bags - set_ledger_and_free_balance(&31, 1_000); - - // then nothing changes - let node_31 = Node::::from_id(&31).unwrap(); - assert_storage_noop!(assert_eq!( - VoterList::::update_position_for(node_31, &weight_of), - None, - )); - }); - } -} - -#[cfg(test)] -mod bags { - use super::*; - use crate::mock::*; - use frame_support::{assert_ok, assert_storage_noop}; - - #[test] - fn get_works() { - use crate::VoterBags; - ExtBuilder::default().build_and_execute_without_check_count(|| { - let check_bag = |bag_upper, head, tail, ids| { - assert_storage_noop!(Bag::::get(bag_upper)); - - let bag = Bag::::get(bag_upper).unwrap(); - let bag_ids = bag.iter().map(|n| n.voter().id).collect::>(); - - assert_eq!(bag, Bag:: { head, tail, bag_upper }); - assert_eq!(bag_ids, ids); - }; - - // given uppers of bags that exist. - let existing_bag_uppers = vec![10, 1_000]; - - // we can fetch them - check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); - // (getting the same bag twice has the same results) - check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); - check_bag(existing_bag_uppers[1], Some(11), Some(101), vec![11, 21, 101]); - - // and all other uppers don't get bags. - ::VoterBagThresholds::get() - .iter() - .chain(iter::once(&VoteWeight::MAX)) - .filter(|bag_upper| !existing_bag_uppers.contains(bag_upper)) - .for_each(|bag_upper| { - assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); - assert!(!VoterBags::::contains_key(*bag_upper)); - }); - - // when we make a pre-existing bag empty - VoterList::::remove(&31); - - // then - assert_eq!(Bag::::get(existing_bag_uppers[0]), None) - }); - } - - #[test] - #[should_panic] - fn get_panics_with_a_bad_threshold() { - // NOTE: panic is only expected with debug compilation - ExtBuilder::default().build_and_execute_without_check_count(|| { - Bag::::get(11); - }); - } - - #[test] - fn insert_node_happy_paths_works() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - let node = |voter, bag_upper| Node:: { voter, prev: None, next: None, bag_upper }; - - // when inserting into a bag with 1 node - let mut bag_10 = Bag::::get(10).unwrap(); - // (note: bags api does not care about balance or ledger) - bag_10.insert_node(node(Voter::nominator(42), bag_10.bag_upper)); - // then - assert_eq!(bag_as_ids(&bag_10), vec![31, 42]); - - // when inserting into a bag with 3 nodes - let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(Voter::nominator(52), bag_1000.bag_upper)); - // then - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 52]); - - // when inserting into a new bag - let mut bag_20 = Bag::::get_or_make(20); - bag_20.insert_node(node(Voter::nominator(71), bag_20.bag_upper)); - // then - assert_eq!(bag_as_ids(&bag_20), vec![71]); - - // when inserting a node pointing to the accounts not in the bag - let voter_61 = Voter::validator(61); - let node_61 = Node:: { - voter: voter_61.clone(), - prev: Some(21), - next: Some(101), - bag_upper: 20, - }; - bag_20.insert_node(node_61); - // then ids are in order - assert_eq!(bag_as_ids(&bag_20), vec![71, 61]); - // and when the node is re-fetched all the info is correct - assert_eq!( - Node::::get(20, &61).unwrap(), - Node:: { voter: voter_61, prev: Some(71), next: None, bag_upper: 20 } - ); - - // state of all bags is as expected - bag_20.put(); // need to put this bag so its in the storage map - assert_eq!( - get_bags(), - vec![(10, vec![31, 42]), (20, vec![71, 61]), (1_000, vec![11, 21, 101, 52])] - ); - }); - } - - // Document improper ways `insert_node` may be getting used. - #[test] - fn insert_node_bad_paths_documented() { - let node = |voter, prev, next, bag_upper| Node:: { voter, prev, next, bag_upper }; - ExtBuilder::default().build_and_execute_without_check_count(|| { - // when inserting a node with both prev & next pointing at an account in the bag - // and an incorrect bag_upper - let mut bag_1000 = Bag::::get(1_000).unwrap(); - let voter_42 = Voter::nominator(42); - bag_1000.insert_node(node(voter_42.clone(), Some(11), Some(11), 0)); - - // then the ids are in the correct order - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 42]); - // and when the node is re-fetched all the info is correct - assert_eq!( - Node::::get(1_000, &42).unwrap(), - node(voter_42, Some(101), None, bag_1000.bag_upper) - ); - - // given 21 is a validator in bag_1000 (and not a tail node) - let bag_1000_voter = - bag_1000.iter().map(|node| node.voter().clone()).collect::>(); - assert_eq!(bag_1000_voter[1], Voter::validator(21)); - - // when inserting a node with duplicate id 21 but as a nominator - let voter_21_nom = Voter::nominator(21); - bag_1000.insert_node(node(voter_21_nom.clone(), None, None, bag_1000.bag_upper)); - - // then all the nodes after the duplicate are lost (because it is set as the tail) - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21]); - // and the re-fetched node is a nominator with an **incorrect** prev pointer. - assert_eq!( - Node::::get(1_000, &21).unwrap(), - node(voter_21_nom, Some(42), None, bag_1000.bag_upper) - ); - }); - - ExtBuilder::default().build_and_execute_without_check_count(|| { - // when inserting a duplicate id of the head - let mut bag_1000 = Bag::::get(1_000).unwrap(); - let voter_11 = Voter::validator(11); - bag_1000.insert_node(node(voter_11.clone(), None, None, 0)); - // then all nodes after the head are lost - assert_eq!(bag_as_ids(&bag_1000), vec![11]); - // and the re-fetched node - assert_eq!( - Node::::get(1_000, &11).unwrap(), - node(voter_11, Some(101), None, bag_1000.bag_upper) - ); - - assert_eq!(bag_1000, Bag { head: Some(11), tail: Some(11), bag_upper: 1_000 }) - }); - } - - #[test] - #[should_panic = "system logic error: inserting a node who has the id of tail"] - fn insert_node_duplicate_tail_panics_with_debug_assert() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - let node = |voter, prev, next, bag_upper| Node:: { voter, prev, next, bag_upper }; - - // given - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); - let mut bag_1000 = Bag::::get(1_000).unwrap(); - - // when inserting a duplicate id that is already the tail - assert_eq!(bag_1000.tail, Some(101)); - let voter_101 = Voter::validator(101); - bag_1000.insert_node(node(voter_101, None, None, bag_1000.bag_upper)); // panics - }); - } - - #[test] - fn remove_node_happy_paths_works() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - // add some validators to genesis state - bond_validator(51, 50, 1_000); - bond_validator(61, 60, 1_000); - bond_validator(71, 70, 10); - bond_validator(81, 80, 10); - bond_validator(91, 90, 2_000); - bond_validator(161, 160, 2_000); - bond_validator(171, 170, 2_000); - bond_validator(181, 180, 2_000); - bond_validator(191, 190, 2_000); - - let mut bag_10 = Bag::::get(10).unwrap(); - let mut bag_1000 = Bag::::get(1_000).unwrap(); - let mut bag_2000 = Bag::::get(2_000).unwrap(); - - // given - assert_eq!(bag_as_ids(&bag_10), vec![31, 71, 81]); - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 51, 61]); - assert_eq!(bag_as_ids(&bag_2000), vec![91, 161, 171, 181, 191]); - - // remove node that is not pointing at head or tail - let node_101 = Node::::get(bag_1000.bag_upper, &101).unwrap(); - let node_101_pre_remove = node_101.clone(); - bag_1000.remove_node(&node_101); - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 51, 61]); - assert_ok!(bag_1000.sanity_check()); - // node isn't mutated when its removed - assert_eq!(node_101, node_101_pre_remove); - - // remove head when its not pointing at tail - let node_11 = Node::::get(bag_1000.bag_upper, &11).unwrap(); - bag_1000.remove_node(&node_11); - assert_eq!(bag_as_ids(&bag_1000), vec![21, 51, 61]); - assert_ok!(bag_1000.sanity_check()); - - // remove tail when its not pointing at head - let node_61 = Node::::get(bag_1000.bag_upper, &61).unwrap(); - bag_1000.remove_node(&node_61); - assert_eq!(bag_as_ids(&bag_1000), vec![21, 51]); - assert_ok!(bag_1000.sanity_check()); - - // remove tail when its pointing at head - let node_51 = Node::::get(bag_1000.bag_upper, &51).unwrap(); - bag_1000.remove_node(&node_51); - assert_eq!(bag_as_ids(&bag_1000), vec![21]); - assert_ok!(bag_1000.sanity_check()); - - // remove node that is head & tail - let node_21 = Node::::get(bag_1000.bag_upper, &21).unwrap(); - bag_1000.remove_node(&node_21); - bag_1000.put(); // put into storage so get returns the updated bag - assert_eq!(Bag::::get(1_000), None); - - // remove node that is pointing at head and tail - let node_71 = Node::::get(bag_10.bag_upper, &71).unwrap(); - bag_10.remove_node(&node_71); - assert_eq!(bag_as_ids(&bag_10), vec![31, 81]); - assert_ok!(bag_10.sanity_check()); - - // remove head when pointing at tail - let node_31 = Node::::get(bag_10.bag_upper, &31).unwrap(); - bag_10.remove_node(&node_31); - assert_eq!(bag_as_ids(&bag_10), vec![81]); - assert_ok!(bag_10.sanity_check()); - bag_10.put(); // since we updated the bag's head/tail, we need to write this storage - - // remove node that is pointing at head, but not tail - let node_161 = Node::::get(bag_2000.bag_upper, &161).unwrap(); - bag_2000.remove_node(&node_161); - assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 181, 191]); - assert_ok!(bag_2000.sanity_check()); - - // remove node that is pointing at tail, but not head - let node_181 = Node::::get(bag_2000.bag_upper, &181).unwrap(); - bag_2000.remove_node(&node_181); - assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 191]); - assert_ok!(bag_2000.sanity_check()); - - // state of all bags is as expected - assert_eq!(get_bags(), vec![(10, vec![81]), (2_000, vec![91, 171, 191])]); - }); - } - - #[test] - fn remove_node_bad_paths_documented() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - // removing a node that is in the bag but has the wrong upper works. - - let bad_upper_node_11 = Node:: { - voter: Voter::<_>::validator(11), - prev: None, - next: Some(21), - bag_upper: 10, // should be 1_000 - }; - let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.remove_node(&bad_upper_node_11); - bag_1000.put(); - - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![21, 101])]); - let bag_1000 = Bag::::get(1_000).unwrap(); - assert_ok!(bag_1000.sanity_check()); - assert_eq!(bag_1000.head, Some(21)); - assert_eq!(bag_1000.tail, Some(101)); - }); - - ExtBuilder::default().build_and_execute_without_check_count(|| { - // removing a node that is in another bag, will mess up the - // other bag. - - let node_101 = Node::::get(1_000, &101).unwrap(); - let mut bag_10 = Bag::::get(10).unwrap(); - bag_10.remove_node(&node_101); // node_101 is in bag 1_000 - bag_10.put(); - - // the node was removed from its actual bag, bag_1000. - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21])]); - - // the bag removed was called on is ok. - let bag_10 = Bag::::get(10).unwrap(); - assert_eq!(bag_10.tail, Some(31)); - assert_eq!(bag_10.head, Some(31)); - - // but the bag that the node belonged to is in an invalid state - let bag_1000 = Bag::::get(1_000).unwrap(); - // because it still has the removed node as its tail. - assert_eq!(bag_1000.tail, Some(101)); - assert_eq!(bag_1000.head, Some(11)); - assert_ok!(bag_1000.sanity_check()); - }); - } -} - -#[cfg(test)] -mod voter_node { - use super::*; - use crate::mock::*; - - #[test] - fn voting_data_works() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - let weight_of = Staking::weight_of_fn(); - - // add nominator with no targets - bond_nominator(42, 43, 1_000, vec![11]); - - // given - assert_eq!( - get_voter_list_as_voters(), - vec![ - Voter::validator(11), - Voter::validator(21), - Voter::nominator(101), - Voter::nominator(42), - Voter::validator(31), - ] - ); - assert_eq!(active_era(), 0); - - let slashing_spans = - ::SlashingSpans::iter().collect::>(); - assert_eq!(slashing_spans.keys().len(), 0); // no pre-existing slashing spans - - let node_11 = Node::::get(10, &11).unwrap(); - assert_eq!( - node_11.voting_data(&weight_of, &slashing_spans).unwrap(), - (11, 1_000, vec![11]) - ); - - // getting data for a nominators with 0 slashed targets - let node_101 = Node::::get(1_000, &101).unwrap(); - assert_eq!( - node_101.voting_data(&weight_of, &slashing_spans).unwrap(), - (101, 500, vec![11, 21]) - ); - let node_42 = Node::::get(10, &42).unwrap(); - assert_eq!( - node_42.voting_data(&weight_of, &slashing_spans).unwrap(), - (42, 1_000, vec![11]) - ); - - // roll ahead an era so any slashes will be after the previous nominations - start_active_era(1); - - // when a validator gets a slash, - add_slash(&11); - let slashing_spans = - ::SlashingSpans::iter().collect::>(); - - assert_eq!(slashing_spans.keys().cloned().collect::>(), vec![11, 42, 101]); - // then its node no longer exists - assert_eq!( - get_voter_list_as_voters(), - vec![ - Voter::validator(21), - Voter::nominator(101), - Voter::nominator(42), - Voter::validator(31), - ] - ); - // and its nominators no longer have it as a target - let node_101 = Node::::get(10, &101).unwrap(); - assert_eq!( - node_101.voting_data(&weight_of, &slashing_spans), - Some((101, 475, vec![21])), - ); - - let node_42 = Node::::get(10, &42).unwrap(); - assert_eq!( - node_42.voting_data(&weight_of, &slashing_spans), - None, // no voting data since its 1 target has been slashed since nominating - ); - }); - } - - #[test] - fn is_misplaced_works() { - ExtBuilder::default().build_and_execute_without_check_count(|| { - let weight_of = Staking::weight_of_fn(); - let node_31 = Node::::get(10, &31).unwrap(); - - // a node is properly placed if its slashable balance is in range - // of the threshold of the bag its in. - assert_eq!(Staking::slashable_balance_of(&31), 1); - assert!(!node_31.is_misplaced(&weight_of)); - - // and will become misplaced if its slashable balance does not - // correspond to the bag it is in. - set_ledger_and_free_balance(&31, 11); - - assert_eq!(Staking::slashable_balance_of(&31), 11); - assert!(node_31.is_misplaced(&weight_of)); - }); - } -} diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 024e7e6c698e2..ddf14bf054959 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -87,3 +87,24 @@ pub use dispatch::{EnsureOrigin, OriginTrait, UnfilteredDispatchable}; mod voting; pub use voting::{CurrencyToVote, SaturatingCurrencyToVote, U128CurrencyToVote}; + +/// Trait to be implemented by a voter list provider. +pub trait VoterListProvider { + /// Returns iterator over voter list, which can have `take` called on it. + fn get_voters() -> Box>; + /// get the current count of voters. + fn count() -> u32; + // Hook for inserting a validator. + fn on_insert(voter: &AccountId, weight: u64); // TODO + /// Hook for updating the list when a voter is added, their voter type is changed, + /// or their weight changes. + fn on_update(voter: &AccountId, weight: u64); + /// Hook for removing a voter from the list. + fn on_remove(voter: &AccountId); + /// Sanity check internal state of list. Only meant for debug compilation. + fn sanity_check() -> Result<(), &'static str>; +} + +pub trait StakingVoteWeight { + fn staking_vote_weight(who: &AccountId) -> u64; +} diff --git a/frame/voter-bags/Cargo.toml b/frame/voter-bags/Cargo.toml index ea620f0f00e2c..59f6d74d2c851 100644 --- a/frame/voter-bags/Cargo.toml +++ b/frame/voter-bags/Cargo.toml @@ -19,6 +19,7 @@ frame-system = { version = "4.0.0-dev", default-features = false, path = "../sys frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } log = { version = "0.4.14", default-features = false } @@ -40,4 +41,4 @@ std = [ ] runtime-benchmarks = [ "frame-benchmarking", -] \ No newline at end of file +] diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index 9f16d0dd11cf4..7fbf541111a57 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -22,10 +22,7 @@ //! voters doesn't particularly matter. use frame_election_provider_support::VoteWeight; -use frame_support::traits::{Currency, CurrencyToVote, LockableCurrency}; use frame_system::ensure_signed; -use pallet_staking::{AccountIdOf, BalanceOf, GenesisConfig, VotingDataOf}; -use sp_std::collections::btree_map::BTreeMap; mod voter_list; pub mod weights; @@ -51,7 +48,7 @@ macro_rules! log { #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::pallet_prelude::*; + use frame_support::{pallet_prelude::*, traits::StakingVoteWeight}; use frame_system::pallet_prelude::*; #[pallet::pallet] @@ -115,6 +112,8 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; + + type StakingVoteWeight: StakingVoteWeight; } /// How many voters are registered. @@ -128,15 +127,14 @@ pub mod pallet { /// However, the `Node` data structure has helpers which can provide that information. #[pallet::storage] pub(crate) type VoterNodes = - StorageMap<_, Twox64Concat, AccountIdOf, voter_list::Node>; + StorageMap<_, Twox64Concat, T::AccountId, voter_list::Node>; /// Which bag currently contains a particular voter. /// /// This may not be the appropriate bag for the voter's weight if they have been rewarded or /// slashed. #[pallet::storage] - pub(crate) type VoterBagFor = - StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; + pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, T::AccountId, VoteWeight>; /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. @@ -163,25 +161,48 @@ pub mod pallet { /// Anyone can call this function about any stash. // #[pallet::weight(T::WeightInfo::rebag())] #[pallet::weight(123456789)] // TODO - pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { + pub fn rebag(origin: OriginFor, stash: T::AccountId) -> DispatchResult { ensure_signed(origin)?; - Pallet::::do_rebag(&stash); + let weight = T::StakingVoteWeight::staking_vote_weight(&stash); + Pallet::::do_rebag(&stash, weight); Ok(()) } } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn integrity_test() { + sp_std::if_std! { + sp_io::TestExternalities::new_empty().execute_with(|| { + assert!( + T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, + "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", + T::SlashDeferDuration::get(), + T::BondingDuration::get(), + ); + + assert!( + T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "Voter bag thresholds must strictly increase", + ); + }); + } + } + } } impl Pallet { /// Move a stash account from one bag to another, depositing an event on success. /// /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { + pub fn do_rebag( + stash: &T::AccountId, + new_weight: VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = voter_list::Node::::from_id(&stash).and_then(|node| { - let weight_of = pallet_staking::Pallet::::weight_of_fn(); - VoterList::update_position_for(node, weight_of) - }); + let maybe_movement = voter_list::Node::::from_id(&stash) + .and_then(|node| VoterList::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); }; @@ -189,35 +210,31 @@ impl Pallet { } } -pub struct VoterBagsVoterListProvider; -impl pallet_staking::VoterListProvider for VoterBagsVoterListProvider { +pub struct VoterBagsVoterListProvider(sp_std::marker::PhantomData); +impl frame_support::traits::VoterListProvider + for VoterBagsVoterListProvider +{ /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters( - slashing_spans: BTreeMap, pallet_staking::slashing::SlashingSpans>, - ) -> Box>> { - let weight_of = pallet_staking::Pallet::::weight_of_fn(); - - Box::new( - VoterList::::iter() - .filter_map(move |node| node.voting_data(&weight_of, &slashing_spans)), - ) + fn get_voters() -> Box> { + Box::new(VoterList::::iter().map(|n| n.voter().clone())) } - fn on_validator_insert(voter: &T::AccountId) { - VoterList::::insert_as(voter, voter_list::VoterType::Validator); + fn count() -> u32 { + CounterForVoters::::get() } - fn on_nominator_insert(voter: &T::AccountId) { - VoterList::::insert_as(voter, voter_list::VoterType::Nominator); + fn on_insert(voter: &T::AccountId, weight: VoteWeight) { + // TODO: change the interface to not use ref. + VoterList::::insert(voter.clone(), weight); } /// Hook for updating a voter in the list (unused). - fn on_voter_update(voter: &T::AccountId) { - Pallet::::do_rebag(voter); + fn on_update(voter: &T::AccountId, new_weight: VoteWeight) { + Pallet::::do_rebag(voter, new_weight); } /// Hook for removing a voter from the list. - fn on_voter_remove(voter: &T::AccountId) { + fn on_remove(voter: &T::AccountId) { VoterList::::remove(voter) } diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs index dfa2126ccc54b..b724442039d4c 100644 --- a/frame/voter-bags/src/voter_list.rs +++ b/frame/voter-bags/src/voter_list.rs @@ -17,7 +17,7 @@ //! VoterList implementation. -use crate::{AccountIdOf, Config}; +use crate::Config; use codec::{Decode, Encode}; use frame_election_provider_support::VoteWeight; use frame_support::{ensure, traits::Get, DefaultNoBound}; @@ -30,9 +30,6 @@ use sp_std::{ marker::PhantomData, }; -/// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. -pub type VoterOf = Voter>; - /// Given a certain vote weight, which bag should contain this voter? /// /// Bags are identified by their upper threshold; the value returned by this function is guaranteed @@ -51,7 +48,7 @@ fn notional_bag_for(weight: VoteWeight) -> VoteWeight { } /// Find the upper threshold of the actual bag containing the current voter. -fn current_bag_for(id: &AccountIdOf) -> Option { +fn current_bag_for(id: &T::AccountId) -> Option { crate::VoterBagFor::::try_get(id).ok() } @@ -78,21 +75,113 @@ impl VoterList { crate::VoterNodes::::remove_all(None); } + /// Migrate the voter list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::VoterBagThresholds` has already been updated. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - No voter is changed unless required to by the difference between the old threshold list + /// and the new. + /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// new threshold set. + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + // we can't check all preconditions, but we can check one + debug_assert!( + crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); + + let mut affected_accounts = BTreeSet::new(); + let mut affected_old_bags = BTreeSet::new(); + + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_set.difference(&old_set).copied() { + let affected_bag = notional_bag_for::(inserted_bag); + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue; + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // a removed bag means that all members of that bag must be rebagged + for removed_bag in old_set.difference(&new_set).copied() { + if !affected_old_bags.insert(removed_bag) { + continue; + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // migrate the + let weight_of = pallet_staking::Pallet::::weight_of_fn(); + Self::remove_many(affected_accounts.iter().map(|voter| voter)); + let num_affected = affected_accounts.len() as u32; + Self::insert_many(affected_accounts.into_iter(), weight_of); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in old_set.difference(&new_set).copied() { + debug_assert!( + !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + "no voter should be present in a removed bag", + ); + crate::VoterBags::::remove(removed_bag); + } + + debug_assert!( + { + let thresholds = T::BVoterBagThresholds::get(); + crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + }, + "all `bag_upper` in storage must be members of the new thresholds", + ); + + num_affected + } + /// Regenerate voter data from the `Nominators` and `Validators` storage items. /// /// This is expensive and should only ever be performed during a migration, never during /// consensus. /// /// Returns the number of voters migrated. - pub fn regenerate() -> u32 { - Self::clear(); + // pub fn regenerate(weight_of: dyn FnOnce(&T::AccountId) -> VoteWeight) -> u32 { + // Self::clear(); - let nominators_iter = staking::Nominators::::iter().map(|(id, _)| Voter::nominator(id)); - let validators_iter = staking::Validators::::iter().map(|(id, _)| Voter::validator(id)); - let weight_of = staking::Pallet::::weight_of_fn(); + // let nominators_iter = staking::Nominators::::iter().map(|(id, _)| Voter::nominator(id)); + // let validators_iter = staking::Validators::::iter().map(|(id, _)| Voter::validator(id)); - Self::insert_many(nominators_iter.chain(validators_iter), weight_of) - } + // nominators_iter.chain(validators_iter).for_each(|v| { + // let weight = weight_of(v); + // Self::insert(v, weight_of); + // }); + // // TODO: use the new insert_at. it should work + // } /// Decode the length of the voter list. pub fn decode_len() -> Option { @@ -133,62 +222,49 @@ impl VoterList { iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } - /// Insert a new voter into the appropriate bag in the voter list. - /// - /// If the voter is already present in the list, their type will be updated. - /// That case is cheaper than inserting a new voter. - pub fn insert_as(account_id: &AccountIdOf, voter_type: VoterType) { - // if this is an update operation we can complete this easily and cheaply - if !Node::::update_voter_type_for(account_id, voter_type) { - // otherwise, we need to insert from scratch - let weight_of = staking::Pallet::::weight_of_fn(); - let voter = Voter { id: account_id.clone(), voter_type }; - Self::insert(voter, weight_of); - } - } - - /// Insert a new voter into the appropriate bag in the voter list. - fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { - Self::insert_many(sp_std::iter::once(voter), weight_of); - } - /// Insert several voters into the appropriate bags in the voter list. /// /// This is more efficient than repeated calls to `Self::insert`. fn insert_many( - voters: impl IntoIterator>, + voters: impl IntoIterator, weight_of: impl Fn(&T::AccountId) -> VoteWeight, - ) -> u32 { - let mut bags = BTreeMap::new(); - let mut count = 0; + ) { + voters.into_iter().for_each(|v| { + let weight = weight_of(&v); + Self::insert(v, weight); + }); + } - for voter in voters.into_iter() { - let weight = weight_of(&voter.id); - let bag = notional_bag_for::(weight); - crate::log!(debug, "inserting {:?} with weight {} into bag {:?}", voter, weight, bag); - bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); - count += 1; - } + /// Insert a new voter into the appropriate bag in the voter list. + pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { + let bag_weight = notional_bag_for::(weight); + crate::log!( + debug, + "inserting {:?} with weight {} into bag {:?}", + voter, + weight, + bag_weight + ); + let mut bag = Bag::::get_or_make(bag_weight); + bag.insert(voter); - for (_, bag) in bags { - bag.put(); - } + // TODO: why are wr writing to the bag of the voter is neither head or tail???? + bag.put(); crate::CounterForVoters::::mutate(|prev_count| { - *prev_count = prev_count.saturating_add(count) + *prev_count = prev_count.saturating_add(1) }); - count } /// Remove a voter (by id) from the voter list. - pub fn remove(voter: &AccountIdOf) { + pub fn remove(voter: &T::AccountId) { Self::remove_many(sp_std::iter::once(voter)); } /// Remove many voters (by id) from the voter list. /// /// This is more efficient than repeated calls to `Self::remove`. - pub fn remove_many<'a>(voters: impl IntoIterator>) { + pub fn remove_many<'a>(voters: impl IntoIterator) { let mut bags = BTreeMap::new(); let mut count = 0; @@ -231,9 +307,9 @@ impl VoterList { /// to call [`self.remove_many`] followed by [`self.insert_many`]. pub fn update_position_for( mut node: Node, - weight_of: impl Fn(&AccountIdOf) -> VoteWeight, + new_weight: VoteWeight, ) -> Option<(VoteWeight, VoteWeight)> { - node.is_misplaced(&weight_of).then(move || { + node.is_misplaced(new_weight).then(move || { let old_idx = node.bag_upper; // TODO: there should be a way to move a non-head-tail node to another bag @@ -249,12 +325,12 @@ impl VoterList { crate::log!( error, "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", - node.voter.id, + node.voter, ); } // put the voter into the appropriate new bag - let new_idx = notional_bag_for::(weight_of(&node.voter.id)); + let new_idx = notional_bag_for::(new_weight); node.bag_upper = new_idx; let mut bag = Bag::::get_or_make(node.bag_upper); bag.insert_node(node); @@ -264,94 +340,6 @@ impl VoterList { }) } - /// Migrate the voter list from one set of thresholds to another. - /// - /// This should only be called as part of an intentional migration; it's fairly expensive. - /// - /// Returns the number of accounts affected. - /// - /// Preconditions: - /// - /// - `old_thresholds` is the previous list of thresholds. - /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::VoterBagThresholds` has already been updated. - /// - /// Postconditions: - /// - /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. - /// - No voter is changed unless required to by the difference between the old threshold list - /// and the new. - /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the - /// new threshold set. - pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { - // we can't check all preconditions, but we can check one - debug_assert!( - crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), - "not all `bag_upper` currently in storage are members of `old_thresholds`", - ); - - let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); - - let mut affected_accounts = BTreeSet::new(); - let mut affected_old_bags = BTreeSet::new(); - - // a new bag means that all accounts previously using the old bag's threshold must now - // be rebagged - for inserted_bag in new_set.difference(&old_set).copied() { - let affected_bag = notional_bag_for::(inserted_bag); - if !affected_old_bags.insert(affected_bag) { - // If the previous threshold list was [10, 20], and we insert [3, 5], then there's - // no point iterating through bag 10 twice. - continue - } - - if let Some(bag) = Bag::::get(affected_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); - } - } - - // a removed bag means that all members of that bag must be rebagged - for removed_bag in old_set.difference(&new_set).copied() { - if !affected_old_bags.insert(removed_bag) { - continue - } - - if let Some(bag) = Bag::::get(removed_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); - } - } - - // migrate the - let weight_of = pallet_staking::Pallet::::weight_of_fn(); - Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); - let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); - - // we couldn't previously remove the old bags because both insertion and removal assume that - // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid - // of them. - // - // it's pretty cheap to iterate this again, because both sets are in-memory and require no - // lookups. - for removed_bag in old_set.difference(&new_set).copied() { - debug_assert!( - !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), - "no voter should be present in a removed bag", - ); - crate::VoterBags::::remove(removed_bag); - } - - debug_assert!( - { - let thresholds = T::BVoterBagThresholds::get(); - crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) - }, - "all `bag_upper` in storage must be members of the new thresholds", - ); - - num_affected - } - /// Sanity check the voter list. /// /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) @@ -366,7 +354,7 @@ impl VoterList { pub(crate) fn sanity_check() -> Result<(), &'static str> { let mut seen_in_list = BTreeSet::new(); ensure!( - Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), + Self::iter().map(|node| node.voter).all(|voter| seen_in_list.insert(voter)), "duplicate identified", ); @@ -399,8 +387,8 @@ impl VoterList { #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] #[cfg_attr(test, derive(PartialEq))] pub struct Bag { - head: Option>, - tail: Option>, + head: Option, + tail: Option, #[codec(skip)] bag_upper: VoteWeight, @@ -464,7 +452,7 @@ impl Bag { /// /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. - fn insert(&mut self, voter: VoterOf) { + fn insert(&mut self, voter: T::AccountId) { self.insert_node(Node:: { voter, prev: None, next: None, bag_upper: self.bag_upper }); } @@ -477,15 +465,15 @@ impl Bag { /// `self.put()` after use. fn insert_node(&mut self, mut node: Node) { if let Some(tail) = &self.tail { - if *tail == node.voter.id { + if *tail == node.voter { // this should never happen, but this check prevents a worst case infinite loop debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return + return; }; } - let id = node.voter.id.clone(); + let id = node.voter.clone(); node.prev = self.tail.clone(); node.next = None; @@ -527,10 +515,10 @@ impl Bag { } // clear the bag head/tail pointers as necessary - if self.head.as_ref() == Some(&node.voter.id) { + if self.head.as_ref() == Some(&node.voter) { self.head = node.next.clone(); } - if self.tail.as_ref() == Some(&node.voter.id) { + if self.tail.as_ref() == Some(&node.voter) { self.tail = node.prev.clone(); } } @@ -565,7 +553,7 @@ impl Bag { let mut seen_in_bag = BTreeSet::new(); ensure!( self.iter() - .map(|node| node.voter.id) + .map(|node| node.voter) // each voter is only seen once, thus there is no cycle within a bag .all(|voter| seen_in_bag.insert(voter)), "Duplicate found in bag" @@ -580,9 +568,9 @@ impl Bag { #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] #[cfg_attr(test, derive(PartialEq, Clone))] pub struct Node { - voter: Voter>, - prev: Option>, - next: Option>, + voter: T::AccountId, + prev: Option, + next: Option, /// The bag index is not stored in storage, but injected during all fetch operations. #[codec(skip)] @@ -591,7 +579,7 @@ pub struct Node { impl Node { /// Get a node by bag idx and account id. - pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { + pub fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { debug_assert!( T::BVoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" @@ -606,19 +594,19 @@ impl Node { /// /// Note that this must perform two storage lookups: one to identify which bag is appropriate, /// and another to actually fetch the node. - pub fn from_id(account_id: &AccountIdOf) -> Option> { + pub fn from_id(account_id: &T::AccountId) -> Option> { let bag = current_bag_for::(account_id)?; Self::get(bag, account_id) } /// Get a node by account id, assuming it's in the same bag as this node. - pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { + pub fn in_bag(&self, account_id: &T::AccountId) -> Option> { Self::get(self.bag_upper, account_id) } /// Put the node back into storage. pub fn put(self) { - crate::VoterNodes::::insert(self.voter.id.clone(), self); + crate::VoterNodes::::insert(self.voter.clone(), self); } /// Get the previous node in the bag. @@ -631,50 +619,9 @@ impl Node { self.next.as_ref().and_then(|id| self.in_bag(id)) } - /// Get this voter's voting data. - pub fn voting_data( - &self, - weight_of: impl Fn(&T::AccountId) -> VoteWeight, - slashing_spans: &BTreeMap, staking::slashing::SlashingSpans>, - ) -> Option> { - let voter_weight = weight_of(&self.voter.id); - match self.voter.voter_type { - VoterType::Validator => - Some((self.voter.id.clone(), voter_weight, sp_std::vec![self.voter.id.clone()])), - VoterType::Nominator => { - let staking::Nominations { submitted_in, mut targets, .. } = - staking::Nominators::::get(&self.voter.id)?; - // Filter out nomination targets which were nominated before the most recent - // slashing span. - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); - - (!targets.is_empty()).then(move || (self.voter.id.clone(), voter_weight, targets)) - }, - } - } - /// `true` when this voter is in the wrong bag. - pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper - } - - /// Update the voter type associated with a particular node by id. - /// - /// This updates storage immediately. - /// - /// Returns whether the voter existed and was successfully updated. - pub fn update_voter_type_for(account_id: &AccountIdOf, voter_type: VoterType) -> bool { - let node = Self::from_id(account_id); - let existed = node.is_some(); - if let Some(mut node) = node { - node.voter.voter_type = voter_type; - node.put(); - } - existed + pub fn is_misplaced(&self, current_weight: VoteWeight) -> bool { + notional_bag_for::(current_weight) != self.bag_upper } /// Get the upper threshold of the bag that this node _should_ be in, given its vote weight. @@ -688,37 +635,10 @@ impl Node { } /// Get the underlying voter. - #[cfg(any(test, feature = "runtime-benchmarks"))] - pub fn voter(&self) -> &Voter { + pub fn voter(&self) -> &T::AccountId { &self.voter } } -/// Fundamental information about a voter. -#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord, sp_runtime::RuntimeDebug)] -pub struct Voter { - /// Account Id of this voter - pub id: AccountId, - /// Whether the voter is a validator or nominator - pub voter_type: VoterType, -} - -impl Voter { - pub fn nominator(id: AccountId) -> Self { - Self { id, voter_type: VoterType::Nominator } - } - - pub fn validator(id: AccountId) -> Self { - Self { id, voter_type: VoterType::Validator } - } -} - -/// Type of voter. -/// -/// Similar to [`crate::StakerStatus`], but somewhat more limited. -#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr(feature = "std", derive(Debug))] -pub enum VoterType { - Validator, - Nominator, -} +#[cfg(test)] +mod test {} From e40efffad0970dfa2dce1a40e137df862f31826c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 18:51:11 -0700 Subject: [PATCH 091/241] Mock working --- Cargo.lock | 2 + frame/election-provider-support/src/lib.rs | 23 +++++++ frame/staking/src/lib.rs | 12 ++-- frame/staking/src/mock.rs | 1 - frame/staking/src/pallet/impls.rs | 35 +++++----- frame/staking/src/pallet/mod.rs | 16 ++--- frame/support/src/traits.rs | 21 ------ frame/voter-bags/Cargo.toml | 4 ++ frame/voter-bags/src/lib.rs | 48 +++++++------- frame/voter-bags/src/mock.rs | 75 ++++++++++++++++++++++ frame/voter-bags/src/tests.rs | 37 +++++++++++ frame/voter-bags/src/voter_list.rs | 69 ++++++++------------ 12 files changed, 223 insertions(+), 120 deletions(-) create mode 100644 frame/voter-bags/src/mock.rs create mode 100644 frame/voter-bags/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 5559a3c649ef2..17c1e03ced3ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5799,8 +5799,10 @@ dependencies = [ "frame-support", "frame-system", "log", + "pallet-balances", "pallet-staking", "parity-scale-codec", + "sp-core", "sp-io", "sp-runtime", "sp-std", diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 72896e5599138..c01170e0c5625 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -292,3 +292,26 @@ impl ElectionProvider for () { Err("<() as ElectionProvider> cannot do anything.") } } + +// TODO not sure where to put these, but put them here just because this is where +// export voteweight from into the staking pallet +/// Trait to be implemented by a voter list provider. +pub trait VoterListProvider { + /// Returns iterator over voter list, which can have `take` called on it. + fn get_voters() -> Box>; + /// get the current count of voters. + fn count() -> u32; + // Hook for inserting a validator. + fn on_insert(voter: AccountId, weight: VoteWeight); // TODO + /// Hook for updating the list when a voter is added, their voter type is changed, + /// or their weight changes. + fn on_update(voter: &AccountId, weight: VoteWeight); + /// Hook for removing a voter from the list. + fn on_remove(voter: &AccountId); + /// Sanity check internal state of list. Only meant for debug compilation. + fn sanity_check() -> Result<(), &'static str>; +} + +pub trait StakingVoteWeight { + fn staking_vote_weight(who: &AccountId) -> VoteWeight; +} diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 1e82c16148b9d..f8d458d9d4776 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -494,7 +494,7 @@ impl } if unlocking_balance >= value { - break; + break } } @@ -799,7 +799,9 @@ where /// A simple voter list implementation that does not require any additional pallets. pub struct StakingVoterListStub(sp_std::marker::PhantomData); -impl frame_support::traits::VoterListProvider for StakingVoterListStub { +impl frame_election_provider_support::VoterListProvider + for StakingVoterListStub +{ /// Returns iterator over voter list, which can have `take` called on it. fn get_voters() -> Box> { let weight_of = Pallet::::weight_of_fn(); @@ -808,9 +810,9 @@ impl frame_support::traits::VoterListProvider for Staki fn count() -> u32 { CounterForNominators::::get() } - fn on_insert(voter: &T::AccountId, weight: VoteWeight) {} - fn on_update(voter: &T::AccountId, weight: VoteWeight) {} - fn on_remove(voter: &T::AccountId) {} + fn on_insert(_voter: T::AccountId, _weight: VoteWeight) {} + fn on_update(_voter: &T::AccountId, _weight: VoteWeight) {} + fn on_remove(_voter: &T::AccountId) {} fn sanity_check() -> Result<(), &'static str> { Ok(()) } diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 1452a9ff18335..7486573fe9d90 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -106,7 +106,6 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, - // VoterBags: pallet_voter_bags::{Pallet, Call, Storage, Event}, } ); diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index fc546052189ba..949cacf4ac9d3 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -17,12 +17,14 @@ //! Implementations for the Staking FRAME Pallet. -use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; +use frame_election_provider_support::{ + data_provider, ElectionProvider, Supports, VoteWeight, VoterListProvider, +}; use frame_support::{ pallet_prelude::*, traits::{ Currency, CurrencyToVote, EstimateNextNewSession, Get, Imbalance, LockableCurrency, - OnUnbalanced, UnixTime, VoterListProvider, WithdrawReasons, + OnUnbalanced, UnixTime, WithdrawReasons, }, weights::{Weight, WithPostDispatchInfo}, }; @@ -134,7 +136,7 @@ impl Pallet { // Nothing to do if they have no reward points. if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()); + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) } // This is the fraction of the total reward that the validator and the @@ -225,9 +227,8 @@ impl Pallet { Self::update_ledger(&controller, &l); r }), - RewardDestination::Account(dest_account) => { - Some(T::Currency::deposit_creating(&dest_account, amount)) - } + RewardDestination::Account(dest_account) => + Some(T::Currency::deposit_creating(&dest_account, amount)), RewardDestination::None => None, } } @@ -255,14 +256,14 @@ impl Pallet { _ => { // Either `Forcing::ForceNone`, // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None; - } + return None + }, } // New era. let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() - && matches!(ForceEra::::get(), Forcing::ForceNew) + if maybe_new_era_validators.is_some() && + matches!(ForceEra::::get(), Forcing::ForceNew) { ForceEra::::put(Forcing::NotForcing); } @@ -443,12 +444,12 @@ impl Pallet { // TODO: this should be simplified #8911 CurrentEra::::put(0); ErasStartSessionIndex::::insert(&0, &start_session_index); - } + }, _ => (), } Self::deposit_event(Event::StakingElectionFailed); - return None; + return None } Self::deposit_event(Event::StakersElected); @@ -681,7 +682,7 @@ impl Pallet { CounterForNominators::::mutate(|x| x.saturating_inc()) } Nominators::::insert(who, nominations); - T::VoterListProvider::on_insert(who, Self::weight_of_fn()(who)); + T::VoterListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); } @@ -718,7 +719,7 @@ impl Pallet { CounterForValidators::::mutate(|x| x.saturating_inc()) } Validators::::insert(who, prefs); - T::VoterListProvider::on_insert(who, Self::weight_of_fn()(who)); + T::VoterListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); } @@ -782,7 +783,7 @@ impl let target_count = CounterForValidators::::get() as usize; if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big"); + return Err("Target snapshot too big") } let weight = ::DbWeight::get().reads(target_count as u64); @@ -1046,7 +1047,7 @@ where add_db_reads_writes(1, 0); if active_era.is_none() { // This offence need not be re-submitted. - return consumed_weight; + return consumed_weight } active_era.expect("value checked not to be `None`; qed").index }; @@ -1092,7 +1093,7 @@ where // Skip if the validator is invulnerable. if invulnerables.contains(stash) { - continue; + continue } let unapplied = slashing::compute_slash::(slashing::SlashParams { diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 3beedbddb6e92..0836fc98f0f10 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -17,12 +17,12 @@ //! Staking FRAME Pallet. -use frame_election_provider_support::VoteWeight; +use frame_election_provider_support::{VoteWeight, VoterListProvider}; use frame_support::{ pallet_prelude::*, traits::{ Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, LockIdentifier, - LockableCurrency, OnUnbalanced, UnixTime, VoterListProvider, + LockableCurrency, OnUnbalanced, UnixTime, }, weights::{ constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}, @@ -568,7 +568,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue; + continue } match status { StakerStatus::Validator => { @@ -580,7 +580,7 @@ pub mod pallet { } else { num_voters += 1; } - } + }, StakerStatus::Nominator(votes) => { if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), @@ -590,7 +590,7 @@ pub mod pallet { } else { num_voters += 1; } - } + }, _ => (), }; } @@ -1396,9 +1396,9 @@ pub mod pallet { Self::update_ledger(&controller, &ledger); T::VoterListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. Ok(Some( - 35 * WEIGHT_PER_MICROS - + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) - + T::DbWeight::get().reads_writes(3, 2), + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), ) .into()) } diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index ddf14bf054959..024e7e6c698e2 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -87,24 +87,3 @@ pub use dispatch::{EnsureOrigin, OriginTrait, UnfilteredDispatchable}; mod voting; pub use voting::{CurrencyToVote, SaturatingCurrencyToVote, U128CurrencyToVote}; - -/// Trait to be implemented by a voter list provider. -pub trait VoterListProvider { - /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters() -> Box>; - /// get the current count of voters. - fn count() -> u32; - // Hook for inserting a validator. - fn on_insert(voter: &AccountId, weight: u64); // TODO - /// Hook for updating the list when a voter is added, their voter type is changed, - /// or their weight changes. - fn on_update(voter: &AccountId, weight: u64); - /// Hook for removing a voter from the list. - fn on_remove(voter: &AccountId); - /// Sanity check internal state of list. Only meant for debug compilation. - fn sanity_check() -> Result<(), &'static str>; -} - -pub trait StakingVoteWeight { - fn staking_vote_weight(who: &AccountId) -> u64; -} diff --git a/frame/voter-bags/Cargo.toml b/frame/voter-bags/Cargo.toml index 59f6d74d2c851..591cfba5bc335 100644 --- a/frame/voter-bags/Cargo.toml +++ b/frame/voter-bags/Cargo.toml @@ -27,6 +27,10 @@ log = { version = "0.4.14", default-features = false } # Optional imports for benchmarking frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +[dev-dependencies] +sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} +pallet-balances = { version = "4.0.0-dev", path = "../balances" } + [features] default = ["std"] std = [ diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index 7fbf541111a57..636e728a62c19 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -24,6 +24,10 @@ use frame_election_provider_support::VoteWeight; use frame_system::ensure_signed; +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; mod voter_list; pub mod weights; @@ -48,7 +52,8 @@ macro_rules! log { #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{pallet_prelude::*, traits::StakingVoteWeight}; + use frame_election_provider_support::StakingVoteWeight; + use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; #[pallet::pallet] @@ -56,10 +61,16 @@ pub mod pallet { pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config + pallet_staking::Config { + pub trait Config: frame_system::Config { /// The overarching event type. type Event: From> + IsType<::Event>; + /// Weight information for extrinsics in this pallet. + // type WeightInfo: WeightInfo; + + // Something that implements `trait StakingVoteWeight`. + type StakingVoteWeight: StakingVoteWeight; + /// The list of thresholds separating the various voter bags. /// /// Voters are separated into unsorted bags according to their vote weight. This specifies @@ -108,12 +119,7 @@ pub mod pallet { /// With that `VoterList::migrate` can be called, which will perform the appropriate /// migration. #[pallet::constant] - type BVoterBagThresholds: Get<&'static [VoteWeight]>; - - /// Weight information for extrinsics in this pallet. - type WeightInfo: WeightInfo; - - type StakingVoteWeight: StakingVoteWeight; + type VoterBagThresholds: Get<&'static [VoteWeight]>; } /// How many voters are registered. @@ -174,13 +180,6 @@ pub mod pallet { fn integrity_test() { sp_std::if_std! { sp_io::TestExternalities::new_empty().execute_with(|| { - assert!( - T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, - "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", - T::SlashDeferDuration::get(), - T::BondingDuration::get(), - ); - assert!( T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), "Voter bag thresholds must strictly increase", @@ -192,38 +191,35 @@ pub mod pallet { } impl Pallet { - /// Move a stash account from one bag to another, depositing an event on success. + /// Move an account from one bag to another, depositing an event on success. /// - /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag( - stash: &T::AccountId, - new_weight: VoteWeight, - ) -> Option<(VoteWeight, VoteWeight)> { + /// If the account changed bags, returns `Some((from, to))`. + pub fn do_rebag(id: &T::AccountId, new_weight: VoteWeight) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = voter_list::Node::::from_id(&stash) + let maybe_movement = voter_list::Node::::from_id(&id) .and_then(|node| VoterList::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { - Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); + Self::deposit_event(Event::::Rebagged(id.clone(), from, to)); }; maybe_movement } } pub struct VoterBagsVoterListProvider(sp_std::marker::PhantomData); -impl frame_support::traits::VoterListProvider +impl frame_election_provider_support::VoterListProvider for VoterBagsVoterListProvider { /// Returns iterator over voter list, which can have `take` called on it. fn get_voters() -> Box> { - Box::new(VoterList::::iter().map(|n| n.voter().clone())) + Box::new(VoterList::::iter().map(|n| n.id().clone())) } fn count() -> u32 { CounterForVoters::::get() } - fn on_insert(voter: &T::AccountId, weight: VoteWeight) { + fn on_insert(voter: T::AccountId, weight: VoteWeight) { // TODO: change the interface to not use ref. VoterList::::insert(voter.clone(), weight); } diff --git a/frame/voter-bags/src/mock.rs b/frame/voter-bags/src/mock.rs new file mode 100644 index 0000000000000..0b80ea62fc970 --- /dev/null +++ b/frame/voter-bags/src/mock.rs @@ -0,0 +1,75 @@ +use super::*; +use frame_election_provider_support::VoteWeight; +use frame_support::parameter_types; +use std::cell::RefCell; + +type AccountId = u32; +type Balance = u32; + +thread_local! { + static VOTE_WEIGHT: RefCell = RefCell::new(Default::default()) +} + +/// Set a mock return value for `StakingVoteWeight::staking_vote_weight`. +fn set_staking_vote_weight(weight: VoteWeight) { + VOTE_WEIGHT.with(|w| *w.borrow_mut() = weight); +} + +pub struct StakingMock; +impl frame_election_provider_support::StakingVoteWeight for StakingMock { + fn staking_vote_weight(who: &AccountId) -> VoteWeight { + VOTE_WEIGHT.with(|h| h.borrow().clone()) + } +} + +impl frame_system::Config for Runtime { + type SS58Prefix = (); + type BaseCallFilter = frame_support::traits::AllowAll; + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = Call; + type Hash = sp_core::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = AccountId; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = sp_runtime::testing::Header; + type Event = Event; + type BlockHashCount = (); + type DbWeight = (); + type BlockLength = (); + type BlockWeights = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type OnSetCode = (); +} + +/// Thresholds used for bags. +const THRESHOLDS: [VoteWeight; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; + +parameter_types! { + pub const VoterBagThresholds: &'static [VoteWeight] = &THRESHOLDS; +} + +impl crate::Config for Runtime { + type Event = Event; + type VoterBagThresholds = VoterBagThresholds; + type StakingVoteWeight = StakingMock; +} + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; +frame_support::construct_runtime!( + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Storage, Event, Config}, + VoterBags: crate::{Pallet, Call, Storage, Event}, + } +); diff --git a/frame/voter-bags/src/tests.rs b/frame/voter-bags/src/tests.rs new file mode 100644 index 0000000000000..d4d6199cc7efc --- /dev/null +++ b/frame/voter-bags/src/tests.rs @@ -0,0 +1,37 @@ +use super::*; +use mock::*; + + +mod voter_bag_list_provider { + use super::*; + + #[test] + fn get_voters_works() { + todo!(); + } + + #[test] + fn count_works() { + todo!(); + } + + #[test] + fn on_insert_works() { + todo!(); + } + + #[test] + fn on_update_works() { + todo!(); + } + + #[test] + fn on_remove_works() { + todo!(); + } + + #[test] + fn sanity_check_works() { + todo!(); + } +} \ No newline at end of file diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs index d16531e526bc5..8ce041a17101d 100644 --- a/frame/voter-bags/src/voter_list.rs +++ b/frame/voter-bags/src/voter_list.rs @@ -21,7 +21,6 @@ use crate::Config; use codec::{Decode, Encode}; use frame_election_provider_support::VoteWeight; use frame_support::{ensure, traits::Get, DefaultNoBound}; -use pallet_staking as staking; use sp_runtime::SaturatedConversion; use sp_std::{ boxed::Box, @@ -29,6 +28,7 @@ use sp_std::{ iter, marker::PhantomData, }; +use frame_election_provider_support::StakingVoteWeight; /// Given a certain vote weight, which bag should contain this voter? /// @@ -42,7 +42,7 @@ use sp_std::{ /// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this /// function behaves as if it does. fn notional_bag_for(weight: VoteWeight) -> VoteWeight { - let thresholds = T::BVoterBagThresholds::get(); + let thresholds = T::VoterBagThresholds::get(); let idx = thresholds.partition_point(|&threshold| weight > threshold); thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } @@ -114,27 +114,27 @@ impl VoterList { if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's // no point iterating through bag 10 twice. - continue; + continue } if let Some(bag) = Bag::::get(affected_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); + affected_accounts.extend(bag.iter().map(|node| node.id)); } } // a removed bag means that all members of that bag must be rebagged for removed_bag in old_set.difference(&new_set).copied() { if !affected_old_bags.insert(removed_bag) { - continue; + continue } if let Some(bag) = Bag::::get(removed_bag) { - affected_accounts.extend(bag.iter().map(|node| node.voter)); + affected_accounts.extend(bag.iter().map(|node| node.id)); } } // migrate the - let weight_of = pallet_staking::Pallet::::weight_of_fn(); + let weight_of = T::StakingVoteWeight::staking_vote_weight; Self::remove_many(affected_accounts.iter().map(|voter| voter)); let num_affected = affected_accounts.len() as u32; Self::insert_many(affected_accounts.into_iter(), weight_of); @@ -155,7 +155,7 @@ impl VoterList { debug_assert!( { - let thresholds = T::BVoterBagThresholds::get(); + let thresholds = T::VoterBagThresholds::get(); crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) }, "all `bag_upper` in storage must be members of the new thresholds", @@ -191,11 +191,6 @@ impl VoterList { crate::VoterNodes::::iter().count(), "stored length must match count of nodes", ); - debug_assert_eq!( - maybe_len.unwrap_or_default() as u32, - staking::CounterForNominators::::get() + staking::CounterForValidators::::get(), - "voter count must be sum of validator and nominator count", - ); maybe_len } @@ -210,7 +205,7 @@ impl VoterList { // // It's important to retain the ability to omit the final bound because it makes tests much // easier; they can just configure `type VoterBagThresholds = ()`. - let thresholds = T::BVoterBagThresholds::get(); + let thresholds = T::VoterBagThresholds::get(); let iter = thresholds.iter().copied(); let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { // in the event that they included it, we can just pass the iterator through unchanged. @@ -248,7 +243,7 @@ impl VoterList { let mut bag = Bag::::get_or_make(bag_weight); bag.insert(voter); - // TODO: why are wr writing to the bag of the voter is neither head or tail???? + // TODO: why are we writing to the bag of the voter is neither head or tail???? bag.put(); crate::CounterForVoters::::mutate(|prev_count| { @@ -325,7 +320,7 @@ impl VoterList { crate::log!( error, "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", - node.voter, + node.id, ); } @@ -354,7 +349,7 @@ impl VoterList { pub(crate) fn sanity_check() -> Result<(), &'static str> { let mut seen_in_list = BTreeSet::new(); ensure!( - Self::iter().map(|node| node.voter).all(|voter| seen_in_list.insert(voter)), + Self::iter().map(|node| node.id).all(|voter| seen_in_list.insert(voter)), "duplicate identified", ); @@ -398,7 +393,7 @@ impl Bag { /// Get a bag by its upper vote weight. pub fn get(bag_upper: VoteWeight) -> Option> { debug_assert!( - T::BVoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" ); crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { @@ -410,7 +405,7 @@ impl Bag { /// Get a bag by its upper vote weight or make it, appropriately initialized. pub fn get_or_make(bag_upper: VoteWeight) -> Bag { debug_assert!( - T::BVoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" ); Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) @@ -452,8 +447,8 @@ impl Bag { /// /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. - fn insert(&mut self, voter: T::AccountId) { - self.insert_node(Node:: { voter, prev: None, next: None, bag_upper: self.bag_upper }); + fn insert(&mut self, id: T::AccountId) { + self.insert_node(Node:: { id, prev: None, next: None, bag_upper: self.bag_upper }); } /// Insert a voter node into this bag. @@ -465,15 +460,15 @@ impl Bag { /// `self.put()` after use. fn insert_node(&mut self, mut node: Node) { if let Some(tail) = &self.tail { - if *tail == node.voter { + if *tail == node.id { // this should never happen, but this check prevents a worst case infinite loop debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return; + return }; } - let id = node.voter.clone(); + let id = node.id.clone(); node.prev = self.tail.clone(); node.next = None; @@ -515,10 +510,10 @@ impl Bag { } // clear the bag head/tail pointers as necessary - if self.head.as_ref() == Some(&node.voter) { + if self.head.as_ref() == Some(&node.id) { self.head = node.next.clone(); } - if self.tail.as_ref() == Some(&node.voter) { + if self.tail.as_ref() == Some(&node.id) { self.tail = node.prev.clone(); } } @@ -553,7 +548,7 @@ impl Bag { let mut seen_in_bag = BTreeSet::new(); ensure!( self.iter() - .map(|node| node.voter) + .map(|node| node.id) // each voter is only seen once, thus there is no cycle within a bag .all(|voter| seen_in_bag.insert(voter)), "Duplicate found in bag" @@ -568,7 +563,7 @@ impl Bag { #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] #[cfg_attr(test, derive(PartialEq, Clone))] pub struct Node { - voter: T::AccountId, + id: T::AccountId, prev: Option, next: Option, @@ -583,7 +578,7 @@ impl Node { /// Get a node by bag idx and account id. pub fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { debug_assert!( - T::BVoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" ); crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { @@ -608,7 +603,7 @@ impl Node { /// Put the node back into storage. pub fn put(self) { - crate::VoterNodes::::insert(self.voter.clone(), self); + crate::VoterNodes::::insert(self.id.clone(), self); } /// Get the previous node in the bag. @@ -626,19 +621,9 @@ impl Node { notional_bag_for::(current_weight) != self.bag_upper } - /// Get the upper threshold of the bag that this node _should_ be in, given its vote weight. - /// - /// This is a helper intended only for benchmarking and should not be used in production. - #[cfg(any(test, feature = "runtime-benchmarks"))] - pub fn proper_bag_for(&self) -> VoteWeight { - let weight_of = staking::Pallet::::weight_of_fn(); - let current_weight = weight_of(&self.voter.id); - notional_bag_for::(current_weight) - } - /// Get the underlying voter. - pub fn voter(&self) -> &T::AccountId { - &self.voter + pub fn id(&self) -> &T::AccountId { + &self.id } } From 3d545a99f570828dd5e29688592333868b590b3e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 20:22:30 -0700 Subject: [PATCH 092/241] WIP voter list tests --- Cargo.lock | 1 + frame/voter-bags/Cargo.toml | 10 +- frame/voter-bags/src/lib.rs | 3 +- frame/voter-bags/src/mock.rs | 55 ++++++++ frame/voter-bags/src/tests.rs | 3 +- frame/voter-bags/src/voter_list.rs | 152 ++++++++++++++++++++--- frame/voter-bags/src/voter_list/mod.rs | 0 frame/voter-bags/src/voter_list/tests.rs | 0 8 files changed, 199 insertions(+), 25 deletions(-) create mode 100644 frame/voter-bags/src/voter_list/mod.rs create mode 100644 frame/voter-bags/src/voter_list/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 17c1e03ced3ae..78f1c1ef02b7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5806,6 +5806,7 @@ dependencies = [ "sp-io", "sp-runtime", "sp-std", + "sp-tracing", ] [[package]] diff --git a/frame/voter-bags/Cargo.toml b/frame/voter-bags/Cargo.toml index 591cfba5bc335..4e80cbb7dec0f 100644 --- a/frame/voter-bags/Cargo.toml +++ b/frame/voter-bags/Cargo.toml @@ -14,12 +14,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } + sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } + +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } + pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } log = { version = "0.4.14", default-features = false } @@ -29,6 +32,7 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = " [dev-dependencies] sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} +sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index 636e728a62c19..0764788d365ac 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -220,8 +220,7 @@ impl frame_election_provider_support::VoterListProvider } fn on_insert(voter: T::AccountId, weight: VoteWeight) { - // TODO: change the interface to not use ref. - VoterList::::insert(voter.clone(), weight); + VoterList::::insert(voter, weight); } /// Hook for updating a voter in the list (unused). diff --git a/frame/voter-bags/src/mock.rs b/frame/voter-bags/src/mock.rs index 0b80ea62fc970..3e1816c776c63 100644 --- a/frame/voter-bags/src/mock.rs +++ b/frame/voter-bags/src/mock.rs @@ -73,3 +73,58 @@ frame_support::construct_runtime!( VoterBags: crate::{Pallet, Call, Storage, Event}, } ); + +pub(crate) mod ext_builder { + use super::*; + + /// Default AccountIds and their weights. + const GENESIS_IDS: [(AccountId, VoteWeight); 4] = + [(31, 10), (11, 1_000), (21, 1_000), (101, 1_000)]; + #[derive(Default)] + pub(crate) struct ExtBuilder { + ids: Vec<(AccountId, VoteWeight)>, + } + + impl ExtBuilder { + /// Add some AccountIds to insert into `VoterList`. + pub(crate) fn add_ids(mut self, ids: Vec<(AccountId, VoteWeight)>) -> Self { + self.ids = ids; + self + } + + pub(crate) fn build(self) -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); + let storage = + frame_system::GenesisConfig::default().build_storage::().unwrap(); + + let mut ext = sp_io::TestExternalities::from(storage); + ext.execute_with(|| { + for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { + VoterList::::insert(*id, *weight); + } + }); + + ext + } + + pub(crate) fn build_and_execute(self, test: impl FnOnce() -> ()) { + self.build().execute_with(test); + } + } +} + +pub(crate) mod test_utils { + use super::*; + use voter_list::Bag; + + /// Returns the nodes of all non-empty bags. + pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { + VoterBagThresholds::get() + .into_iter() + .filter_map(|t| { + Bag::::get(*t) + .map(|bag| (*t, bag.iter().map(|n| n.id().clone()).collect::>())) + }) + .collect::>() + } +} diff --git a/frame/voter-bags/src/tests.rs b/frame/voter-bags/src/tests.rs index d4d6199cc7efc..d1861f3db53cd 100644 --- a/frame/voter-bags/src/tests.rs +++ b/frame/voter-bags/src/tests.rs @@ -1,7 +1,6 @@ use super::*; use mock::*; - mod voter_bag_list_provider { use super::*; @@ -34,4 +33,4 @@ mod voter_bag_list_provider { fn sanity_check_works() { todo!(); } -} \ No newline at end of file +} diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs index 8ce041a17101d..ae7199beefdd9 100644 --- a/frame/voter-bags/src/voter_list.rs +++ b/frame/voter-bags/src/voter_list.rs @@ -19,7 +19,7 @@ use crate::Config; use codec::{Decode, Encode}; -use frame_election_provider_support::VoteWeight; +use frame_election_provider_support::{StakingVoteWeight, VoteWeight}; use frame_support::{ensure, traits::Get, DefaultNoBound}; use sp_runtime::SaturatedConversion; use sp_std::{ @@ -28,7 +28,6 @@ use sp_std::{ iter, marker::PhantomData, }; -use frame_election_provider_support::StakingVoteWeight; /// Given a certain vote weight, which bag should contain this voter? /// @@ -68,6 +67,7 @@ pub struct VoterList(PhantomData); impl VoterList { /// Remove all data associated with the voter list from storage. + #[cfg(test)] pub fn clear() { crate::CounterForVoters::::kill(); crate::VoterBagFor::::remove_all(None); @@ -231,7 +231,12 @@ impl VoterList { } /// Insert a new voter into the appropriate bag in the voter list. + // WARNING: This does not check if the inserted AccountId is duplicate with + // others in any bag. pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { + // TODO: can check if this voter exists as a node by checking if `voter` exists + // in the nodes map and return early if it does. + let bag_weight = notional_bag_for::(weight); crate::log!( debug, @@ -391,7 +396,7 @@ pub struct Bag { impl Bag { /// Get a bag by its upper vote weight. - pub fn get(bag_upper: VoteWeight) -> Option> { + pub(crate) fn get(bag_upper: VoteWeight) -> Option> { debug_assert!( T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" @@ -403,7 +408,7 @@ impl Bag { } /// Get a bag by its upper vote weight or make it, appropriately initialized. - pub fn get_or_make(bag_upper: VoteWeight) -> Bag { + fn get_or_make(bag_upper: VoteWeight) -> Bag { debug_assert!( T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" @@ -412,12 +417,12 @@ impl Bag { } /// `True` if self is empty. - pub fn is_empty(&self) -> bool { + fn is_empty(&self) -> bool { self.head.is_none() && self.tail.is_none() } /// Put the bag back into storage. - pub fn put(self) { + fn put(self) { if self.is_empty() { crate::VoterBags::::remove(self.bag_upper); } else { @@ -426,17 +431,17 @@ impl Bag { } /// Get the head node in this bag. - pub fn head(&self) -> Option> { + fn head(&self) -> Option> { self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) } /// Get the tail node in this bag. - pub fn tail(&self) -> Option> { + fn tail(&self) -> Option> { self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) } /// Iterate over the nodes in this bag. - pub fn iter(&self) -> impl Iterator> { + pub(crate) fn iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } @@ -576,7 +581,7 @@ pub struct Node { impl Node { /// Get a node by bag idx and account id. - pub fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { + fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { debug_assert!( T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" @@ -591,41 +596,152 @@ impl Node { /// /// Note that this must perform two storage lookups: one to identify which bag is appropriate, /// and another to actually fetch the node. - pub fn from_id(account_id: &T::AccountId) -> Option> { + pub(crate) fn from_id(account_id: &T::AccountId) -> Option> { let bag = current_bag_for::(account_id)?; Self::get(bag, account_id) } /// Get a node by account id, assuming it's in the same bag as this node. - pub fn in_bag(&self, account_id: &T::AccountId) -> Option> { + fn in_bag(&self, account_id: &T::AccountId) -> Option> { Self::get(self.bag_upper, account_id) } /// Put the node back into storage. - pub fn put(self) { + fn put(self) { crate::VoterNodes::::insert(self.id.clone(), self); } /// Get the previous node in the bag. - pub fn prev(&self) -> Option> { + fn prev(&self) -> Option> { self.prev.as_ref().and_then(|id| self.in_bag(id)) } /// Get the next node in the bag. - pub fn next(&self) -> Option> { + fn next(&self) -> Option> { self.next.as_ref().and_then(|id| self.in_bag(id)) } /// `true` when this voter is in the wrong bag. - pub fn is_misplaced(&self, current_weight: VoteWeight) -> bool { + fn is_misplaced(&self, current_weight: VoteWeight) -> bool { notional_bag_for::(current_weight) != self.bag_upper } /// Get the underlying voter. - pub fn id(&self) -> &T::AccountId { + pub(crate) fn id(&self) -> &T::AccountId { &self.id } } #[cfg(test)] -mod test {} +mod test { + use super::*; + use crate::{ + mock::{ext_builder::*, test_utils::*, *}, + CounterForVoters, VoterBagFor, VoterBags, VoterNodes, + }; + use frame_support::{assert_ok, assert_storage_noop}; + + #[test] + fn setup_works() { + ExtBuilder::default().build_and_execute(|| { + // syntactic sugar to create a raw node + let node = |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; + + assert_eq!(CounterForVoters::::get(), 4); + assert_eq!(VoterBagFor::::iter().count(), 4); + assert_eq!(VoterNodes::::iter().count(), 4); + assert_eq!(VoterBags::::iter().count(), 2); + + // the state of the bags is as expected + assert_eq!( + VoterBags::::get(10).unwrap(), + Bag:: { head: Some(31), tail: Some(31), bag_upper: 0 } + ); + assert_eq!( + VoterBags::::get(1_000).unwrap(), + Bag:: { head: Some(11), tail: Some(101), bag_upper: 0 } + ); + + assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(11).unwrap(), + node(11, None, Some(21)) + ); + + assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(21).unwrap(), + node(21, Some(11), Some(101)) + ); + + assert_eq!(VoterBagFor::::get(31).unwrap(), 10); + assert_eq!( + VoterNodes::::get(31).unwrap(), + node(31, None, None) + ); + + assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(101).unwrap(), + node(101, Some(21), None) + ); + + // non-existent id does not have a storage footprint + assert_eq!(VoterBagFor::::get(41), None); + assert_eq!(VoterNodes::::get(41), None); + + // iteration of the bags would yield: + assert_eq!( + VoterList::::iter().map(|n| *n.id()).collect::>(), + vec![11, 21, 101, 31], + // ^^ note the order of insertion in genesis! + ); + + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + }); + } + + #[test] + fn notional_bag_for_works() { + // under a threshold gives the next threshold. + assert_eq!(notional_bag_for::(0), 10); + assert_eq!(notional_bag_for::(9), 10); + assert_eq!(notional_bag_for::(11), 20); + + // at a threshold gives that threshold. + assert_eq!(notional_bag_for::(10), 10); + + let max_explicit_threshold = *::VoterBagThresholds::get().last().unwrap(); + assert_eq!(max_explicit_threshold, 10_000); + // if the max explicit threshold is less than VoteWeight::MAX, + assert!(VoteWeight::MAX > max_explicit_threshold); + // anything above it will belong to the VoteWeight::MAX bag. + assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); + } + mod bags { + #[test] + fn get_works() {} + + #[test] + fn remove_last_voter_in_bags_cleans_bag() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // Bump 31 to a bigger bag + VoterList::::remove(31); + VoterList::::insert(31); + + // then the bag with bound 10 is wiped from storage. + assert_eq!(get_bags(), vec![(1000, vec![11, 21, 101]), (10_000, vec![31])]); + + // and can be recreated again as needed + bond_validator(77, 777, 10); + assert_eq!( + get_bags(), + vec![(10, vec![77]), (1000, vec![11, 21, 101]), (10_000, vec![31])] + ); + }); + } + } +} diff --git a/frame/voter-bags/src/voter_list/mod.rs b/frame/voter-bags/src/voter_list/mod.rs new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/frame/voter-bags/src/voter_list/tests.rs b/frame/voter-bags/src/voter_list/tests.rs new file mode 100644 index 0000000000000..e69de29bb2d1d From 61f9af9c5f867af6bdfc2c7db1efe847a171103a Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 20:38:28 -0700 Subject: [PATCH 093/241] Add bag insert, remove tests --- frame/voter-bags/src/mock.rs | 4 + frame/voter-bags/src/voter_list.rs | 747 ----------------------- frame/voter-bags/src/voter_list/mod.rs | 636 +++++++++++++++++++ frame/voter-bags/src/voter_list/tests.rs | 197 ++++++ 4 files changed, 837 insertions(+), 747 deletions(-) delete mode 100644 frame/voter-bags/src/voter_list.rs diff --git a/frame/voter-bags/src/mock.rs b/frame/voter-bags/src/mock.rs index 3e1816c776c63..862d9c5483b71 100644 --- a/frame/voter-bags/src/mock.rs +++ b/frame/voter-bags/src/mock.rs @@ -127,4 +127,8 @@ pub(crate) mod test_utils { }) .collect::>() } + + pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { + bag.iter().map(|n| *n.id()).collect::>() + } } diff --git a/frame/voter-bags/src/voter_list.rs b/frame/voter-bags/src/voter_list.rs deleted file mode 100644 index ae7199beefdd9..0000000000000 --- a/frame/voter-bags/src/voter_list.rs +++ /dev/null @@ -1,747 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! VoterList implementation. - -use crate::Config; -use codec::{Decode, Encode}; -use frame_election_provider_support::{StakingVoteWeight, VoteWeight}; -use frame_support::{ensure, traits::Get, DefaultNoBound}; -use sp_runtime::SaturatedConversion; -use sp_std::{ - boxed::Box, - collections::{btree_map::BTreeMap, btree_set::BTreeSet}, - iter, - marker::PhantomData, -}; - -/// Given a certain vote weight, which bag should contain this voter? -/// -/// Bags are identified by their upper threshold; the value returned by this function is guaranteed -/// to be a member of `T::VoterBagThresholds`. -/// -/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, -/// because in the event that bags are inserted or deleted, the number of affected voters which need -/// to be migrated is smaller. -/// -/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this -/// function behaves as if it does. -fn notional_bag_for(weight: VoteWeight) -> VoteWeight { - let thresholds = T::VoterBagThresholds::get(); - let idx = thresholds.partition_point(|&threshold| weight > threshold); - thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) -} - -/// Find the upper threshold of the actual bag containing the current voter. -fn current_bag_for(id: &T::AccountId) -> Option { - crate::VoterBagFor::::try_get(id).ok() -} - -/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. -/// -/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of -/// arbitrary and unbounded length, all having a vote weight within a particular constant range. -/// This structure means that voters can be added and removed in `O(1)` time. -/// -/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. -/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall -/// stake decreases as successive bags are reached. This means that it is valid to truncate -/// iteration at any desired point; only those voters in the lowest bag (who are known to have -/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire -/// for fairness and the requirement for efficiency. -pub struct VoterList(PhantomData); - -impl VoterList { - /// Remove all data associated with the voter list from storage. - #[cfg(test)] - pub fn clear() { - crate::CounterForVoters::::kill(); - crate::VoterBagFor::::remove_all(None); - crate::VoterBags::::remove_all(None); - crate::VoterNodes::::remove_all(None); - } - - /// Migrate the voter list from one set of thresholds to another. - /// - /// This should only be called as part of an intentional migration; it's fairly expensive. - /// - /// Returns the number of accounts affected. - /// - /// Preconditions: - /// - /// - `old_thresholds` is the previous list of thresholds. - /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::VoterBagThresholds` has already been updated. - /// - /// Postconditions: - /// - /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. - /// - No voter is changed unless required to by the difference between the old threshold list - /// and the new. - /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the - /// new threshold set. - pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { - // we can't check all preconditions, but we can check one - debug_assert!( - crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), - "not all `bag_upper` currently in storage are members of `old_thresholds`", - ); - - let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); - - let mut affected_accounts = BTreeSet::new(); - let mut affected_old_bags = BTreeSet::new(); - - // a new bag means that all accounts previously using the old bag's threshold must now - // be rebagged - for inserted_bag in new_set.difference(&old_set).copied() { - let affected_bag = notional_bag_for::(inserted_bag); - if !affected_old_bags.insert(affected_bag) { - // If the previous threshold list was [10, 20], and we insert [3, 5], then there's - // no point iterating through bag 10 twice. - continue - } - - if let Some(bag) = Bag::::get(affected_bag) { - affected_accounts.extend(bag.iter().map(|node| node.id)); - } - } - - // a removed bag means that all members of that bag must be rebagged - for removed_bag in old_set.difference(&new_set).copied() { - if !affected_old_bags.insert(removed_bag) { - continue - } - - if let Some(bag) = Bag::::get(removed_bag) { - affected_accounts.extend(bag.iter().map(|node| node.id)); - } - } - - // migrate the - let weight_of = T::StakingVoteWeight::staking_vote_weight; - Self::remove_many(affected_accounts.iter().map(|voter| voter)); - let num_affected = affected_accounts.len() as u32; - Self::insert_many(affected_accounts.into_iter(), weight_of); - - // we couldn't previously remove the old bags because both insertion and removal assume that - // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid - // of them. - // - // it's pretty cheap to iterate this again, because both sets are in-memory and require no - // lookups. - for removed_bag in old_set.difference(&new_set).copied() { - debug_assert!( - !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), - "no voter should be present in a removed bag", - ); - crate::VoterBags::::remove(removed_bag); - } - - debug_assert!( - { - let thresholds = T::VoterBagThresholds::get(); - crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) - }, - "all `bag_upper` in storage must be members of the new thresholds", - ); - - num_affected - } - - /// Regenerate voter data from the `Nominators` and `Validators` storage items. - /// - /// This is expensive and should only ever be performed during a migration, never during - /// consensus. - /// - /// Returns the number of voters migrated. - // pub fn regenerate(weight_of: dyn FnOnce(&T::AccountId) -> VoteWeight) -> u32 { - // Self::clear(); - - // let nominators_iter = staking::Nominators::::iter().map(|(id, _)| Voter::nominator(id)); - // let validators_iter = staking::Validators::::iter().map(|(id, _)| Voter::validator(id)); - - // nominators_iter.chain(validators_iter).for_each(|v| { - // let weight = weight_of(v); - // Self::insert(v, weight_of); - // }); - // // TODO: use the new insert_at. it should work - // } - - /// Decode the length of the voter list. - pub fn decode_len() -> Option { - let maybe_len = crate::CounterForVoters::::try_get().ok().map(|n| n.saturated_into()); - debug_assert_eq!( - maybe_len.unwrap_or_default(), - crate::VoterNodes::::iter().count(), - "stored length must match count of nodes", - ); - maybe_len - } - - /// Iterate over all nodes in all bags in the voter list. - /// - /// Full iteration can be expensive; it's recommended to limit the number of items with - /// `.take(n)`. - pub fn iter() -> impl Iterator> { - // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to - // omit the final bound, we need to ensure that we explicitly include that threshold in the - // list. - // - // It's important to retain the ability to omit the final bound because it makes tests much - // easier; they can just configure `type VoterBagThresholds = ()`. - let thresholds = T::VoterBagThresholds::get(); - let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { - // in the event that they included it, we can just pass the iterator through unchanged. - Box::new(iter.rev()) - } else { - // otherwise, insert it here. - Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) - }; - iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) - } - - /// Insert several voters into the appropriate bags in the voter list. - /// - /// This is more efficient than repeated calls to `Self::insert`. - fn insert_many( - voters: impl IntoIterator, - weight_of: impl Fn(&T::AccountId) -> VoteWeight, - ) { - voters.into_iter().for_each(|v| { - let weight = weight_of(&v); - Self::insert(v, weight); - }); - } - - /// Insert a new voter into the appropriate bag in the voter list. - // WARNING: This does not check if the inserted AccountId is duplicate with - // others in any bag. - pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { - // TODO: can check if this voter exists as a node by checking if `voter` exists - // in the nodes map and return early if it does. - - let bag_weight = notional_bag_for::(weight); - crate::log!( - debug, - "inserting {:?} with weight {} into bag {:?}", - voter, - weight, - bag_weight - ); - let mut bag = Bag::::get_or_make(bag_weight); - bag.insert(voter); - - // TODO: why are we writing to the bag of the voter is neither head or tail???? - bag.put(); - - crate::CounterForVoters::::mutate(|prev_count| { - *prev_count = prev_count.saturating_add(1) - }); - } - - /// Remove a voter (by id) from the voter list. - pub fn remove(voter: &T::AccountId) { - Self::remove_many(sp_std::iter::once(voter)); - } - - /// Remove many voters (by id) from the voter list. - /// - /// This is more efficient than repeated calls to `Self::remove`. - pub fn remove_many<'a>(voters: impl IntoIterator) { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for voter_id in voters.into_iter() { - let node = match Node::::from_id(voter_id) { - Some(node) => node, - None => continue, - }; - count += 1; - - // clear the bag head/tail pointers as necessary - let bag = bags - .entry(node.bag_upper) - .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); - bag.remove_node(&node); - - // now get rid of the node itself - crate::VoterNodes::::remove(voter_id); - crate::VoterBagFor::::remove(voter_id); - } - - for (_, bag) in bags { - bag.put(); - } - - crate::CounterForVoters::::mutate(|prev_count| { - *prev_count = prev_count.saturating_sub(count) - }); - } - - /// Update a voter's position in the voter list. - /// - /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they - /// are moved into the correct bag. - /// - /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. - /// - /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by - /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient - /// to call [`self.remove_many`] followed by [`self.insert_many`]. - pub fn update_position_for( - mut node: Node, - new_weight: VoteWeight, - ) -> Option<(VoteWeight, VoteWeight)> { - node.is_misplaced(new_weight).then(move || { - let old_idx = node.bag_upper; - - // TODO: there should be a way to move a non-head-tail node to another bag - // with just 1 bag read of the destination bag and zero writes - // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 - - // clear the old bag head/tail pointers as necessary - if let Some(mut bag) = Bag::::get(node.bag_upper) { - bag.remove_node(&node); - bag.put(); - } else { - debug_assert!(false, "every node must have an extant bag associated with it"); - crate::log!( - error, - "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", - node.id, - ); - } - - // put the voter into the appropriate new bag - let new_idx = notional_bag_for::(new_weight); - node.bag_upper = new_idx; - let mut bag = Bag::::get_or_make(node.bag_upper); - bag.insert_node(node); - bag.put(); - - (old_idx, new_idx) - }) - } - - /// Sanity check the voter list. - /// - /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) - /// is being used, after all other staking data (such as counter) has been updated. It checks - /// that: - /// - /// * Iterate all voters in list and make sure there are no duplicates. - /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. - /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. - /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are - /// checked per *any* update to `VoterList`. - pub(crate) fn sanity_check() -> Result<(), &'static str> { - let mut seen_in_list = BTreeSet::new(); - ensure!( - Self::iter().map(|node| node.id).all(|voter| seen_in_list.insert(voter)), - "duplicate identified", - ); - - let iter_count = Self::iter().collect::>().len() as u32; - let stored_count = crate::CounterForVoters::::get(); - ensure!(iter_count == stored_count, "iter_count != voter_count"); - - // let validators = staking::CounterForValidators::::get(); - // let nominators = staking::CounterForNominators::::get(); - // ensure!(validators + nominators == stored_count, "validators + nominators != voters"); - - let _ = T::VoterBagThresholds::get() - .into_iter() - .map(|t| Bag::::get(*t).unwrap_or_default()) - .map(|b| b.sanity_check()) - .collect::>()?; - - Ok(()) - } -} - -/// A Bag is a doubly-linked list of voters. -/// -/// Note that we maintain both head and tail pointers. While it would be possible to get away -/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's -/// more desirable to ensure that there is some element of first-come, first-serve to the list's -/// iteration so that there's no incentive to churn voter positioning to improve the chances of -/// appearing within the voter set. -#[derive(DefaultNoBound, Encode, Decode)] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] -#[cfg_attr(test, derive(PartialEq))] -pub struct Bag { - head: Option, - tail: Option, - - #[codec(skip)] - bag_upper: VoteWeight, -} - -impl Bag { - /// Get a bag by its upper vote weight. - pub(crate) fn get(bag_upper: VoteWeight) -> Option> { - debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); - crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { - bag.bag_upper = bag_upper; - bag - }) - } - - /// Get a bag by its upper vote weight or make it, appropriately initialized. - fn get_or_make(bag_upper: VoteWeight) -> Bag { - debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); - Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) - } - - /// `True` if self is empty. - fn is_empty(&self) -> bool { - self.head.is_none() && self.tail.is_none() - } - - /// Put the bag back into storage. - fn put(self) { - if self.is_empty() { - crate::VoterBags::::remove(self.bag_upper); - } else { - crate::VoterBags::::insert(self.bag_upper, self); - } - } - - /// Get the head node in this bag. - fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) - } - - /// Get the tail node in this bag. - fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) - } - - /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) - } - - /// Insert a new voter into this bag. - /// - /// This is private on purpose because it's naive: it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the nodes. You still need to call - /// `self.put()` after use. - fn insert(&mut self, id: T::AccountId) { - self.insert_node(Node:: { id, prev: None, next: None, bag_upper: self.bag_upper }); - } - - /// Insert a voter node into this bag. - /// - /// This is private on purpose because it's naive; it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the node. You still need to call - /// `self.put()` after use. - fn insert_node(&mut self, mut node: Node) { - if let Some(tail) = &self.tail { - if *tail == node.id { - // this should never happen, but this check prevents a worst case infinite loop - debug_assert!(false, "system logic error: inserting a node who has the id of tail"); - crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return - }; - } - - let id = node.id.clone(); - - node.prev = self.tail.clone(); - node.next = None; - node.put(); - - // update the previous tail - if let Some(mut old_tail) = self.tail() { - old_tail.next = Some(id.clone()); - old_tail.put(); - } - - // update the internal bag links - if self.head.is_none() { - self.head = Some(id.clone()); - } - self.tail = Some(id.clone()); - - crate::VoterBagFor::::insert(id, self.bag_upper); - } - - /// Remove a voter node from this bag. - /// - /// This is private on purpose because it doesn't check whether this bag contains the voter in - /// the first place. Generally, use [`VoterList::remove`] instead. - /// - /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` - /// to update storage for the bag and `node`. - fn remove_node(&mut self, node: &Node) { - // Update previous node. - if let Some(mut prev) = node.prev() { - prev.next = node.next.clone(); - prev.put(); - } - // Update next node. - if let Some(mut next) = node.next() { - next.prev = node.prev.clone(); - next.put(); - } - - // clear the bag head/tail pointers as necessary - if self.head.as_ref() == Some(&node.id) { - self.head = node.next.clone(); - } - if self.tail.as_ref() == Some(&node.id) { - self.tail = node.prev.clone(); - } - } - - /// Sanity check this bag. - /// - /// Should be called by the call-site, after each mutating operation on a bag. The call site of - /// this struct is always `VoterList`. - /// - /// * Ensures head has no prev. - /// * Ensures tail has no next. - /// * Ensures there are no loops, traversal from head to tail is correct. - fn sanity_check(&self) -> Result<(), &'static str> { - ensure!( - self.head() - .map(|head| head.prev().is_none()) - // if there is no head, then there must not be a tail, meaning that the bag is - // empty. - .unwrap_or_else(|| self.tail.is_none()), - "head has a prev" - ); - - ensure!( - self.tail() - .map(|tail| tail.next().is_none()) - // if there is no tail, then there must not be a head, meaning that the bag is - // empty. - .unwrap_or_else(|| self.head.is_none()), - "tail has a next" - ); - - let mut seen_in_bag = BTreeSet::new(); - ensure!( - self.iter() - .map(|node| node.id) - // each voter is only seen once, thus there is no cycle within a bag - .all(|voter| seen_in_bag.insert(voter)), - "Duplicate found in bag" - ); - - Ok(()) - } -} - -/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. -#[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] -#[cfg_attr(test, derive(PartialEq, Clone))] -pub struct Node { - id: T::AccountId, - prev: Option, - next: Option, - - /// The bag index is not stored in storage, but injected during all fetch operations. - #[codec(skip)] - pub(crate) bag_upper: VoteWeight, - // TODO maybe - // - store voter data here i.e targets -} - -impl Node { - /// Get a node by bag idx and account id. - fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { - debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); - crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { - node.bag_upper = bag_upper; - node - }) - } - - /// Get a node by account id. - /// - /// Note that this must perform two storage lookups: one to identify which bag is appropriate, - /// and another to actually fetch the node. - pub(crate) fn from_id(account_id: &T::AccountId) -> Option> { - let bag = current_bag_for::(account_id)?; - Self::get(bag, account_id) - } - - /// Get a node by account id, assuming it's in the same bag as this node. - fn in_bag(&self, account_id: &T::AccountId) -> Option> { - Self::get(self.bag_upper, account_id) - } - - /// Put the node back into storage. - fn put(self) { - crate::VoterNodes::::insert(self.id.clone(), self); - } - - /// Get the previous node in the bag. - fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| self.in_bag(id)) - } - - /// Get the next node in the bag. - fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| self.in_bag(id)) - } - - /// `true` when this voter is in the wrong bag. - fn is_misplaced(&self, current_weight: VoteWeight) -> bool { - notional_bag_for::(current_weight) != self.bag_upper - } - - /// Get the underlying voter. - pub(crate) fn id(&self) -> &T::AccountId { - &self.id - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::{ - mock::{ext_builder::*, test_utils::*, *}, - CounterForVoters, VoterBagFor, VoterBags, VoterNodes, - }; - use frame_support::{assert_ok, assert_storage_noop}; - - #[test] - fn setup_works() { - ExtBuilder::default().build_and_execute(|| { - // syntactic sugar to create a raw node - let node = |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; - - assert_eq!(CounterForVoters::::get(), 4); - assert_eq!(VoterBagFor::::iter().count(), 4); - assert_eq!(VoterNodes::::iter().count(), 4); - assert_eq!(VoterBags::::iter().count(), 2); - - // the state of the bags is as expected - assert_eq!( - VoterBags::::get(10).unwrap(), - Bag:: { head: Some(31), tail: Some(31), bag_upper: 0 } - ); - assert_eq!( - VoterBags::::get(1_000).unwrap(), - Bag:: { head: Some(11), tail: Some(101), bag_upper: 0 } - ); - - assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); - assert_eq!( - VoterNodes::::get(11).unwrap(), - node(11, None, Some(21)) - ); - - assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); - assert_eq!( - VoterNodes::::get(21).unwrap(), - node(21, Some(11), Some(101)) - ); - - assert_eq!(VoterBagFor::::get(31).unwrap(), 10); - assert_eq!( - VoterNodes::::get(31).unwrap(), - node(31, None, None) - ); - - assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); - assert_eq!( - VoterNodes::::get(101).unwrap(), - node(101, Some(21), None) - ); - - // non-existent id does not have a storage footprint - assert_eq!(VoterBagFor::::get(41), None); - assert_eq!(VoterNodes::::get(41), None); - - // iteration of the bags would yield: - assert_eq!( - VoterList::::iter().map(|n| *n.id()).collect::>(), - vec![11, 21, 101, 31], - // ^^ note the order of insertion in genesis! - ); - - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); - }); - } - - #[test] - fn notional_bag_for_works() { - // under a threshold gives the next threshold. - assert_eq!(notional_bag_for::(0), 10); - assert_eq!(notional_bag_for::(9), 10); - assert_eq!(notional_bag_for::(11), 20); - - // at a threshold gives that threshold. - assert_eq!(notional_bag_for::(10), 10); - - let max_explicit_threshold = *::VoterBagThresholds::get().last().unwrap(); - assert_eq!(max_explicit_threshold, 10_000); - // if the max explicit threshold is less than VoteWeight::MAX, - assert!(VoteWeight::MAX > max_explicit_threshold); - // anything above it will belong to the VoteWeight::MAX bag. - assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); - } - mod bags { - #[test] - fn get_works() {} - - #[test] - fn remove_last_voter_in_bags_cleans_bag() { - ExtBuilder::default().build_and_execute(|| { - // given - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); - - // Bump 31 to a bigger bag - VoterList::::remove(31); - VoterList::::insert(31); - - // then the bag with bound 10 is wiped from storage. - assert_eq!(get_bags(), vec![(1000, vec![11, 21, 101]), (10_000, vec![31])]); - - // and can be recreated again as needed - bond_validator(77, 777, 10); - assert_eq!( - get_bags(), - vec![(10, vec![77]), (1000, vec![11, 21, 101]), (10_000, vec![31])] - ); - }); - } - } -} diff --git a/frame/voter-bags/src/voter_list/mod.rs b/frame/voter-bags/src/voter_list/mod.rs index e69de29bb2d1d..49ae91dcc8c9e 100644 --- a/frame/voter-bags/src/voter_list/mod.rs +++ b/frame/voter-bags/src/voter_list/mod.rs @@ -0,0 +1,636 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! VoterList implementation. + +use crate::Config; +use codec::{Decode, Encode}; +use frame_election_provider_support::{StakingVoteWeight, VoteWeight}; +use frame_support::{ensure, traits::Get, DefaultNoBound}; +use sp_runtime::SaturatedConversion; +use sp_std::{ + boxed::Box, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + iter, + marker::PhantomData, +}; + +#[cfg(test)] +mod tests; + +/// Given a certain vote weight, which bag should contain this voter? +/// +/// Bags are identified by their upper threshold; the value returned by this function is guaranteed +/// to be a member of `T::VoterBagThresholds`. +/// +/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, +/// because in the event that bags are inserted or deleted, the number of affected voters which need +/// to be migrated is smaller. +/// +/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this +/// function behaves as if it does. +fn notional_bag_for(weight: VoteWeight) -> VoteWeight { + let thresholds = T::VoterBagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| weight > threshold); + thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) +} + +/// Find the upper threshold of the actual bag containing the current voter. +fn current_bag_for(id: &T::AccountId) -> Option { + crate::VoterBagFor::::try_get(id).ok() +} + +/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. +/// +/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of +/// arbitrary and unbounded length, all having a vote weight within a particular constant range. +/// This structure means that voters can be added and removed in `O(1)` time. +/// +/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. +/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall +/// stake decreases as successive bags are reached. This means that it is valid to truncate +/// iteration at any desired point; only those voters in the lowest bag (who are known to have +/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire +/// for fairness and the requirement for efficiency. +pub struct VoterList(PhantomData); + +impl VoterList { + /// Remove all data associated with the voter list from storage. + // #[cfg(test)] // TODO this is used for regenerate + // pub fn clear() { + // crate::CounterForVoters::::kill(); + // crate::VoterBagFor::::remove_all(None); + // crate::VoterBags::::remove_all(None); + // crate::VoterNodes::::remove_all(None); + // } + + /// Migrate the voter list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::VoterBagThresholds` has already been updated. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - No voter is changed unless required to by the difference between the old threshold list + /// and the new. + /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// new threshold set. + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + // we can't check all preconditions, but we can check one + debug_assert!( + crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); + + let mut affected_accounts = BTreeSet::new(); + let mut affected_old_bags = BTreeSet::new(); + + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_set.difference(&old_set).copied() { + let affected_bag = notional_bag_for::(inserted_bag); + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } + } + + // a removed bag means that all members of that bag must be rebagged + for removed_bag in old_set.difference(&new_set).copied() { + if !affected_old_bags.insert(removed_bag) { + continue + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } + } + + // migrate the + let weight_of = T::StakingVoteWeight::staking_vote_weight; + Self::remove_many(affected_accounts.iter().map(|voter| voter)); + let num_affected = affected_accounts.len() as u32; + Self::insert_many(affected_accounts.into_iter(), weight_of); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in old_set.difference(&new_set).copied() { + debug_assert!( + !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + "no voter should be present in a removed bag", + ); + crate::VoterBags::::remove(removed_bag); + } + + debug_assert!( + { + let thresholds = T::VoterBagThresholds::get(); + crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + }, + "all `bag_upper` in storage must be members of the new thresholds", + ); + + num_affected + } + + /// Regenerate voter data from the `Nominators` and `Validators` storage items. + /// + /// This is expensive and should only ever be performed during a migration, never during + /// consensus. + /// + /// Returns the number of voters migrated. + // pub fn regenerate(weight_of: dyn FnOnce(&T::AccountId) -> VoteWeight) -> u32 { + // Self::clear(); + + // let nominators_iter = staking::Nominators::::iter().map(|(id, _)| Voter::nominator(id)); + // let validators_iter = staking::Validators::::iter().map(|(id, _)| Voter::validator(id)); + + // nominators_iter.chain(validators_iter).for_each(|v| { + // let weight = weight_of(v); + // Self::insert(v, weight_of); + // }); + // // TODO: use the new insert_at. it should work + // } + + /// Decode the length of the voter list. + pub fn decode_len() -> Option { + let maybe_len = crate::CounterForVoters::::try_get().ok().map(|n| n.saturated_into()); + debug_assert_eq!( + maybe_len.unwrap_or_default(), + crate::VoterNodes::::iter().count(), + "stored length must match count of nodes", + ); + maybe_len + } + + /// Iterate over all nodes in all bags in the voter list. + /// + /// Full iteration can be expensive; it's recommended to limit the number of items with + /// `.take(n)`. + pub fn iter() -> impl Iterator> { + // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to + // omit the final bound, we need to ensure that we explicitly include that threshold in the + // list. + // + // It's important to retain the ability to omit the final bound because it makes tests much + // easier; they can just configure `type VoterBagThresholds = ()`. + let thresholds = T::VoterBagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter.rev()) + } else { + // otherwise, insert it here. + Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) + }; + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) + } + + /// Insert several voters into the appropriate bags in the voter list. + /// + /// This is more efficient than repeated calls to `Self::insert`. + fn insert_many( + voters: impl IntoIterator, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + ) { + voters.into_iter().for_each(|v| { + let weight = weight_of(&v); + Self::insert(v, weight); + }); + } + + /// Insert a new voter into the appropriate bag in the voter list. + // WARNING: This does not check if the inserted AccountId is duplicate with + // others in any bag. + pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { + // TODO: can check if this voter exists as a node by checking if `voter` exists + // in the nodes map and return early if it does. + + let bag_weight = notional_bag_for::(weight); + crate::log!( + debug, + "inserting {:?} with weight {} into bag {:?}", + voter, + weight, + bag_weight + ); + let mut bag = Bag::::get_or_make(bag_weight); + bag.insert(voter); + + // TODO: why are we writing to the bag of the voter is neither head or tail???? + bag.put(); + + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_add(1) + }); + } + + /// Remove a voter (by id) from the voter list. + pub fn remove(voter: &T::AccountId) { + Self::remove_many(sp_std::iter::once(voter)); + } + + /// Remove many voters (by id) from the voter list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + pub fn remove_many<'a>(voters: impl IntoIterator) { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for voter_id in voters.into_iter() { + let node = match Node::::from_id(voter_id) { + Some(node) => node, + None => continue, + }; + count += 1; + + // clear the bag head/tail pointers as necessary + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + bag.remove_node(&node); + + // now get rid of the node itself + crate::VoterNodes::::remove(voter_id); + crate::VoterBagFor::::remove(voter_id); + } + + for (_, bag) in bags { + bag.put(); + } + + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_sub(count) + }); + } + + /// Update a voter's position in the voter list. + /// + /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub fn update_position_for( + mut node: Node, + new_weight: VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { + node.is_misplaced(new_weight).then(move || { + let old_idx = node.bag_upper; + + // TODO: there should be a way to move a non-head-tail node to another bag + // with just 1 bag read of the destination bag and zero writes + // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 + + // clear the old bag head/tail pointers as necessary + if let Some(mut bag) = Bag::::get(node.bag_upper) { + bag.remove_node(&node); + bag.put(); + } else { + debug_assert!(false, "every node must have an extant bag associated with it"); + crate::log!( + error, + "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", + node.id, + ); + } + + // put the voter into the appropriate new bag + let new_idx = notional_bag_for::(new_weight); + node.bag_upper = new_idx; + let mut bag = Bag::::get_or_make(node.bag_upper); + bag.insert_node(node); + bag.put(); + + (old_idx, new_idx) + }) + } + + /// Sanity check the voter list. + /// + /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) + /// is being used, after all other staking data (such as counter) has been updated. It checks + /// that: + /// + /// * Iterate all voters in list and make sure there are no duplicates. + /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. + /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. + /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are + /// checked per *any* update to `VoterList`. + pub(crate) fn sanity_check() -> Result<(), &'static str> { + let mut seen_in_list = BTreeSet::new(); + ensure!( + Self::iter().map(|node| node.id).all(|voter| seen_in_list.insert(voter)), + "duplicate identified", + ); + + let iter_count = Self::iter().collect::>().len() as u32; + let stored_count = crate::CounterForVoters::::get(); + ensure!(iter_count == stored_count, "iter_count != voter_count"); + + // let validators = staking::CounterForValidators::::get(); + // let nominators = staking::CounterForNominators::::get(); + // ensure!(validators + nominators == stored_count, "validators + nominators != voters"); + + let _ = T::VoterBagThresholds::get() + .into_iter() + .map(|t| Bag::::get(*t).unwrap_or_default()) + .map(|b| b.sanity_check()) + .collect::>()?; + + Ok(()) + } +} + +/// A Bag is a doubly-linked list of voters. +/// +/// Note that we maintain both head and tail pointers. While it would be possible to get away +/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's +/// more desirable to ensure that there is some element of first-come, first-serve to the list's +/// iteration so that there's no incentive to churn voter positioning to improve the chances of +/// appearing within the voter set. +#[derive(DefaultNoBound, Encode, Decode)] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq))] +pub struct Bag { + head: Option, + tail: Option, + + #[codec(skip)] + bag_upper: VoteWeight, +} + +impl Bag { + /// Get a bag by its upper vote weight. + pub(crate) fn get(bag_upper: VoteWeight) -> Option> { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; + bag + }) + } + + /// Get a bag by its upper vote weight or make it, appropriately initialized. + fn get_or_make(bag_upper: VoteWeight) -> Bag { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) + } + + /// `True` if self is empty. + fn is_empty(&self) -> bool { + self.head.is_none() && self.tail.is_none() + } + + /// Put the bag back into storage. + fn put(self) { + if self.is_empty() { + crate::VoterBags::::remove(self.bag_upper); + } else { + crate::VoterBags::::insert(self.bag_upper, self); + } + } + + /// Get the head node in this bag. + fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + } + + /// Get the tail node in this bag. + fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + } + + /// Iterate over the nodes in this bag. + pub(crate) fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Insert a new voter into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + fn insert(&mut self, id: T::AccountId) { + self.insert_node(Node:: { id, prev: None, next: None, bag_upper: self.bag_upper }); + } + + /// Insert a voter node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + fn insert_node(&mut self, mut node: Node) { + if let Some(tail) = &self.tail { + if *tail == node.id { + // this should never happen, but this check prevents a worst case infinite loop + debug_assert!(false, "system logic error: inserting a node who has the id of tail"); + crate::log!(warn, "system logic error: inserting a node who has the id of tail"); + return + }; + } + + let id = node.id.clone(); + + node.prev = self.tail.clone(); + node.next = None; + node.put(); + + // update the previous tail + if let Some(mut old_tail) = self.tail() { + old_tail.next = Some(id.clone()); + old_tail.put(); + } + + // update the internal bag links + if self.head.is_none() { + self.head = Some(id.clone()); + } + self.tail = Some(id.clone()); + + crate::VoterBagFor::::insert(id, self.bag_upper); + } + + /// Remove a voter node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the voter in + /// the first place. Generally, use [`VoterList::remove`] instead. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` + /// to update storage for the bag and `node`. + fn remove_node(&mut self, node: &Node) { + // Update previous node. + if let Some(mut prev) = node.prev() { + prev.next = node.next.clone(); + prev.put(); + } + // Update next node. + if let Some(mut next) = node.next() { + next.prev = node.prev.clone(); + next.put(); + } + + // clear the bag head/tail pointers as necessary + if self.head.as_ref() == Some(&node.id) { + self.head = node.next.clone(); + } + if self.tail.as_ref() == Some(&node.id) { + self.tail = node.prev.clone(); + } + } + + /// Sanity check this bag. + /// + /// Should be called by the call-site, after each mutating operation on a bag. The call site of + /// this struct is always `VoterList`. + /// + /// * Ensures head has no prev. + /// * Ensures tail has no next. + /// * Ensures there are no loops, traversal from head to tail is correct. + fn sanity_check(&self) -> Result<(), &'static str> { + ensure!( + self.head() + .map(|head| head.prev().is_none()) + // if there is no head, then there must not be a tail, meaning that the bag is + // empty. + .unwrap_or_else(|| self.tail.is_none()), + "head has a prev" + ); + + ensure!( + self.tail() + .map(|tail| tail.next().is_none()) + // if there is no tail, then there must not be a head, meaning that the bag is + // empty. + .unwrap_or_else(|| self.head.is_none()), + "tail has a next" + ); + + let mut seen_in_bag = BTreeSet::new(); + ensure!( + self.iter() + .map(|node| node.id) + // each voter is only seen once, thus there is no cycle within a bag + .all(|voter| seen_in_bag.insert(voter)), + "Duplicate found in bag" + ); + + Ok(()) + } +} + +/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq, Clone))] +pub struct Node { + id: T::AccountId, + prev: Option, + next: Option, + + /// The bag index is not stored in storage, but injected during all fetch operations. + #[codec(skip)] + pub(crate) bag_upper: VoteWeight, + // TODO maybe + // - store voter data here i.e targets +} + +impl Node { + /// Get a node by bag idx and account id. + fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { + node.bag_upper = bag_upper; + node + }) + } + + /// Get a node by account id. + /// + /// Note that this must perform two storage lookups: one to identify which bag is appropriate, + /// and another to actually fetch the node. + pub(crate) fn from_id(account_id: &T::AccountId) -> Option> { + let bag = current_bag_for::(account_id)?; + Self::get(bag, account_id) + } + + /// Get a node by account id, assuming it's in the same bag as this node. + fn in_bag(&self, account_id: &T::AccountId) -> Option> { + Self::get(self.bag_upper, account_id) + } + + /// Put the node back into storage. + fn put(self) { + crate::VoterNodes::::insert(self.id.clone(), self); + } + + /// Get the previous node in the bag. + fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| self.in_bag(id)) + } + + /// Get the next node in the bag. + fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| self.in_bag(id)) + } + + /// `true` when this voter is in the wrong bag. + fn is_misplaced(&self, current_weight: VoteWeight) -> bool { + notional_bag_for::(current_weight) != self.bag_upper + } + + /// Get the underlying voter. + pub(crate) fn id(&self) -> &T::AccountId { + &self.id + } +} diff --git a/frame/voter-bags/src/voter_list/tests.rs b/frame/voter-bags/src/voter_list/tests.rs index e69de29bb2d1d..e28949c6a134f 100644 --- a/frame/voter-bags/src/voter_list/tests.rs +++ b/frame/voter-bags/src/voter_list/tests.rs @@ -0,0 +1,197 @@ +use super::*; +use crate::{ + mock::{ext_builder::*, test_utils::*, *}, + CounterForVoters, VoterBagFor, VoterBags, VoterNodes, +}; +use frame_support::{assert_ok, assert_storage_noop}; + +#[test] +fn setup_works() { + ExtBuilder::default().build_and_execute(|| { + // syntactic sugar to create a raw node + let node = |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; + + assert_eq!(CounterForVoters::::get(), 4); + assert_eq!(VoterBagFor::::iter().count(), 4); + assert_eq!(VoterNodes::::iter().count(), 4); + assert_eq!(VoterBags::::iter().count(), 2); + + // the state of the bags is as expected + assert_eq!( + VoterBags::::get(10).unwrap(), + Bag:: { head: Some(31), tail: Some(31), bag_upper: 0 } + ); + assert_eq!( + VoterBags::::get(1_000).unwrap(), + Bag:: { head: Some(11), tail: Some(101), bag_upper: 0 } + ); + + assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); + assert_eq!(VoterNodes::::get(11).unwrap(), node(11, None, Some(21))); + + assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); + assert_eq!(VoterNodes::::get(21).unwrap(), node(21, Some(11), Some(101))); + + assert_eq!(VoterBagFor::::get(31).unwrap(), 10); + assert_eq!(VoterNodes::::get(31).unwrap(), node(31, None, None)); + + assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); + assert_eq!(VoterNodes::::get(101).unwrap(), node(101, Some(21), None)); + + // non-existent id does not have a storage footprint + assert_eq!(VoterBagFor::::get(41), None); + assert_eq!(VoterNodes::::get(41), None); + + // iteration of the bags would yield: + assert_eq!( + VoterList::::iter().map(|n| *n.id()).collect::>(), + vec![11, 21, 101, 31], + // ^^ note the order of insertion in genesis! + ); + + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + }); +} + +#[test] +fn notional_bag_for_works() { + // under a threshold gives the next threshold. + assert_eq!(notional_bag_for::(0), 10); + assert_eq!(notional_bag_for::(9), 10); + assert_eq!(notional_bag_for::(11), 20); + + // at a threshold gives that threshold. + assert_eq!(notional_bag_for::(10), 10); + + let max_explicit_threshold = *::VoterBagThresholds::get().last().unwrap(); + assert_eq!(max_explicit_threshold, 10_000); + // if the max explicit threshold is less than VoteWeight::MAX, + assert!(VoteWeight::MAX > max_explicit_threshold); + // anything above it will belong to the VoteWeight::MAX bag. + assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); +} + +#[test] +fn remove_last_voter_in_bags_cleans_bag() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // Bump 31 to a bigger bag + VoterList::::remove(&31); + VoterList::::insert(31, 10_000); + + // then the bag with bound 10 is wiped from storage. + assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101]), (10_000, vec![31])]); + + // and can be recreated again as needed + VoterList::::insert(77, 10); + assert_eq!( + get_bags(), + vec![(10, vec![77]), (1_000, vec![11, 21, 101]), (10_000, vec![31])] + ); + }); +} +mod bags { + use super::*; + + #[test] + fn get_works() { + ExtBuilder::default().build_and_execute(|| { + let check_bag = |bag_upper, head, tail, ids| { + assert_storage_noop!(Bag::::get(bag_upper)); + + let bag = Bag::::get(bag_upper).unwrap(); + let bag_ids = bag.iter().map(|n| *n.id()).collect::>(); + + assert_eq!(bag, Bag:: { head, tail, bag_upper }); + assert_eq!(bag_ids, ids); + }; + + // given uppers of bags that exist. + let existing_bag_uppers = vec![10, 1_000]; + + // we can fetch them + check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); + // (getting the same bag twice has the same results) + check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); + check_bag(existing_bag_uppers[1], Some(11), Some(101), vec![11, 21, 101]); + + // and all other uppers don't get bags. + ::VoterBagThresholds::get() + .iter() + .chain(iter::once(&VoteWeight::MAX)) + .filter(|bag_upper| !existing_bag_uppers.contains(bag_upper)) + .for_each(|bag_upper| { + assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); + assert!(!VoterBags::::contains_key(*bag_upper)); + }); + + // when we make a pre-existing bag empty + VoterList::::remove(&31); + + // then + assert_eq!(Bag::::get(existing_bag_uppers[0]), None) + }); + } + + #[test] + #[should_panic] + fn get_panics_with_a_bad_threshold() { + // NOTE: panic is only expected with debug compilation + ExtBuilder::default().build_and_execute(|| { + Bag::::get(11); + }); + } + + #[test] + fn insert_node_happy_paths_works() { + ExtBuilder::default().build_and_execute(|| { + let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; + + // when inserting into a bag with 1 node + let mut bag_10 = Bag::::get(10).unwrap(); + // (note: bags api does not care about balance or ledger) + bag_10.insert_node(node(42, bag_10.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_10), vec![31, 42]); + + // when inserting into a bag with 3 nodes + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.insert_node(node(52, bag_1000.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 52]); + + // when inserting into a new bag + let mut bag_20 = Bag::::get_or_make(20); + bag_20.insert_node(node(71, bag_20.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_20), vec![71]); + + // when inserting a node pointing to the accounts not in the bag + let node_61 = Node:: { + id: 61, + prev: Some(21), + next: Some(101), + bag_upper: 20, + }; + bag_20.insert_node(node_61); + // then ids are in order + assert_eq!(bag_as_ids(&bag_20), vec![71, 61]); + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(20, &61).unwrap(), + Node:: { id: 61, prev: Some(71), next: None, bag_upper: 20 } + ); + + // state of all bags is as expected + bag_20.put(); // need to put this bag so its in the storage map + assert_eq!( + get_bags(), + vec![(10, vec![31, 42]), (20, vec![71, 61]), (1_000, vec![11, 21, 101, 52])] + ); + }); + } + + +} From 8b273ad934fc24a8c51029ffe475efb7d0d154f8 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 21:09:17 -0700 Subject: [PATCH 094/241] Add test for bag insert and remove --- frame/staking/src/lib.rs | 1 - frame/staking/src/pallet/impls.rs | 2 +- frame/voter-bags/src/mock.rs | 4 +- frame/voter-bags/src/voter_list/tests.rs | 212 ++++++++++++++++++++++- 4 files changed, 209 insertions(+), 10 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index f8d458d9d4776..79fee78b79716 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -804,7 +804,6 @@ impl frame_election_provider_support::VoterListProvider { /// Returns iterator over voter list, which can have `take` called on it. fn get_voters() -> Box> { - let weight_of = Pallet::::weight_of_fn(); Box::new(Nominators::::iter().map(|(n, _)| n)) } fn count() -> u32 { diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 949cacf4ac9d3..0749aba8bcd74 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -654,7 +654,7 @@ impl Pallet { ) -> Vec<(T::AccountId, VoteWeight, Vec)> { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); // Collect all slashing spans into a BTreeMap for further queries. - let slashing_spans = >::iter().collect::>(); + // let slashing_spans = >::iter().collect::>(); let unfiltered_voters: Vec = T::VoterListProvider::get_voters().take(wanted_voters).collect(); diff --git a/frame/voter-bags/src/mock.rs b/frame/voter-bags/src/mock.rs index 862d9c5483b71..0071f8fef4997 100644 --- a/frame/voter-bags/src/mock.rs +++ b/frame/voter-bags/src/mock.rs @@ -11,13 +11,13 @@ thread_local! { } /// Set a mock return value for `StakingVoteWeight::staking_vote_weight`. -fn set_staking_vote_weight(weight: VoteWeight) { +pub(crate) fn set_staking_vote_weight(weight: VoteWeight) { VOTE_WEIGHT.with(|w| *w.borrow_mut() = weight); } pub struct StakingMock; impl frame_election_provider_support::StakingVoteWeight for StakingMock { - fn staking_vote_weight(who: &AccountId) -> VoteWeight { + fn staking_vote_weight(_who: &AccountId) -> VoteWeight { VOTE_WEIGHT.with(|h| h.borrow().clone()) } } diff --git a/frame/voter-bags/src/voter_list/tests.rs b/frame/voter-bags/src/voter_list/tests.rs index e28949c6a134f..b8d646bf0438f 100644 --- a/frame/voter-bags/src/voter_list/tests.rs +++ b/frame/voter-bags/src/voter_list/tests.rs @@ -169,12 +169,8 @@ mod bags { assert_eq!(bag_as_ids(&bag_20), vec![71]); // when inserting a node pointing to the accounts not in the bag - let node_61 = Node:: { - id: 61, - prev: Some(21), - next: Some(101), - bag_upper: 20, - }; + let node_61 = + Node:: { id: 61, prev: Some(21), next: Some(101), bag_upper: 20 }; bag_20.insert_node(node_61); // then ids are in order assert_eq!(bag_as_ids(&bag_20), vec![71, 61]); @@ -193,5 +189,209 @@ mod bags { }); } + // Document improper ways `insert_node` may be getting used. + #[test] + fn insert_node_bad_paths_documented() { + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + ExtBuilder::default().build_and_execute(|| { + // when inserting a node with both prev & next pointing at an account in the bag + // and an incorrect bag_upper + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.insert_node(node(42, Some(11), Some(11), 0)); + + // then the ids are in the correct order + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 42]); + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(1_000, &42).unwrap(), + node(42, Some(101), None, bag_1000.bag_upper) + ); + }); + + ExtBuilder::default().build_and_execute(|| { + // given 21 is in in bag_1000 (and not a tail node) + let mut bag_1000 = Bag::::get(1_000).unwrap(); + assert_eq!(bag_as_ids(&bag_1000)[1], 21); + + // when inserting a node with duplicate id 21 + bag_1000.insert_node(node(21, None, None, bag_1000.bag_upper)); + // then all the nodes after the duplicate are lost (because it is set as the tail) + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21]); + // and the re-fetched node has an **incorrect** prev pointer. + assert_eq!( + Node::::get(1_000, &21).unwrap(), + node(21, Some(101), None, bag_1000.bag_upper) + ); + }); + + ExtBuilder::default().build_and_execute(|| { + // when inserting a duplicate id of the head + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.insert_node(node(11, None, None, 0)); + + // then all nodes after the head are lost + assert_eq!(bag_as_ids(&bag_1000), vec![11]); + // and the re-fetched node has bad pointers + assert_eq!( + Node::::get(1_000, &11).unwrap(), + node(11, Some(101), None, bag_1000.bag_upper) + // ^^^ despite being the bags head, it has a prev + ); + + assert_eq!(bag_1000, Bag { head: Some(11), tail: Some(11), bag_upper: 1_000 }) + }); + } + + // Panics in case of duplicate tail insert (which would result in an infinite loop). + #[test] + #[should_panic = "system logic error: inserting a node who has the id of tail"] + fn insert_node_duplicate_tail_panics_with_debug_assert() { + ExtBuilder::default().build_and_execute(|| { + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + + // when inserting a duplicate id that is already the tail + assert_eq!(bag_1000.tail, Some(101)); + bag_1000.insert_node(node(101, None, None, bag_1000.bag_upper)); // panics + }); + } + + #[test] + fn remove_node_happy_paths_works() { + ExtBuilder::default() + .add_ids(vec![ + (51, 1_000), + (61, 1_000), + (71, 10), + (81, 10), + (91, 2_000), + (161, 2_000), + (171, 2_000), + (181, 2_000), + (191, 2_000), + ]) + .build_and_execute(|| { + let mut bag_10 = Bag::::get(10).unwrap(); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let mut bag_2000 = Bag::::get(2_000).unwrap(); + + // given + assert_eq!(bag_as_ids(&bag_10), vec![31, 71, 81]); + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 51, 61]); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 161, 171, 181, 191]); + + // remove node that is not pointing at head or tail + let node_101 = Node::::get(bag_1000.bag_upper, &101).unwrap(); + let node_101_pre_remove = node_101.clone(); + bag_1000.remove_node(&node_101); + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 51, 61]); + assert_ok!(bag_1000.sanity_check()); + // node isn't mutated when its removed + assert_eq!(node_101, node_101_pre_remove); + + // remove head when its not pointing at tail + let node_11 = Node::::get(bag_1000.bag_upper, &11).unwrap(); + bag_1000.remove_node(&node_11); + assert_eq!(bag_as_ids(&bag_1000), vec![21, 51, 61]); + assert_ok!(bag_1000.sanity_check()); + + // remove tail when its not pointing at head + let node_61 = Node::::get(bag_1000.bag_upper, &61).unwrap(); + bag_1000.remove_node(&node_61); + assert_eq!(bag_as_ids(&bag_1000), vec![21, 51]); + assert_ok!(bag_1000.sanity_check()); + + // remove tail when its pointing at head + let node_51 = Node::::get(bag_1000.bag_upper, &51).unwrap(); + bag_1000.remove_node(&node_51); + assert_eq!(bag_as_ids(&bag_1000), vec![21]); + assert_ok!(bag_1000.sanity_check()); + + // remove node that is head & tail + let node_21 = Node::::get(bag_1000.bag_upper, &21).unwrap(); + bag_1000.remove_node(&node_21); + bag_1000.put(); // put into storage so get returns the updated bag + assert_eq!(Bag::::get(1_000), None); + + // remove node that is pointing at head and tail + let node_71 = Node::::get(bag_10.bag_upper, &71).unwrap(); + bag_10.remove_node(&node_71); + assert_eq!(bag_as_ids(&bag_10), vec![31, 81]); + assert_ok!(bag_10.sanity_check()); + + // remove head when pointing at tail + let node_31 = Node::::get(bag_10.bag_upper, &31).unwrap(); + bag_10.remove_node(&node_31); + assert_eq!(bag_as_ids(&bag_10), vec![81]); + assert_ok!(bag_10.sanity_check()); + bag_10.put(); // since we updated the bag's head/tail, we need to write this storage + + // remove node that is pointing at head, but not tail + let node_161 = Node::::get(bag_2000.bag_upper, &161).unwrap(); + bag_2000.remove_node(&node_161); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 181, 191]); + assert_ok!(bag_2000.sanity_check()); + + // remove node that is pointing at tail, but not head + let node_181 = Node::::get(bag_2000.bag_upper, &181).unwrap(); + bag_2000.remove_node(&node_181); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 191]); + assert_ok!(bag_2000.sanity_check()); + + // state of all bags is as expected + assert_eq!(get_bags(), vec![(10, vec![81]), (2_000, vec![91, 171, 191])]); + }); + } + + #[test] + fn remove_node_bad_paths_documented() { + ExtBuilder::default().build_and_execute(|| { + // removing a node that is in the bag but has the wrong upper works. + + let bad_upper_node_11 = Node:: { + id: 11, + prev: None, + next: Some(21), + bag_upper: 10, // should be 1_000 + }; + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.remove_node(&bad_upper_node_11); + bag_1000.put(); + + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![21, 101])]); + let bag_1000 = Bag::::get(1_000).unwrap(); + assert_ok!(bag_1000.sanity_check()); + assert_eq!(bag_1000.head, Some(21)); + assert_eq!(bag_1000.tail, Some(101)); + }); + + ExtBuilder::default().build_and_execute(|| { + // removing a node that is in another bag, will mess up the + // other bag. + + let node_101 = Node::::get(1_000, &101).unwrap(); + let mut bag_10 = Bag::::get(10).unwrap(); + bag_10.remove_node(&node_101); // node_101 is in bag 1_000 + bag_10.put(); + + // the node was removed from its actual bag, bag_1000. + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21])]); + + // the bag remove was called on is ok. + let bag_10 = Bag::::get(10).unwrap(); + assert_eq!(bag_10.tail, Some(31)); + assert_eq!(bag_10.head, Some(31)); + + // but the bag that the node belonged to is in an invalid state + let bag_1000 = Bag::::get(1_000).unwrap(); + // because it still has the removed node as its tail. + assert_eq!(bag_1000.tail, Some(101)); + assert_eq!(bag_1000.head, Some(11)); + assert_ok!(bag_1000.sanity_check()); + }); + } } From c6d41b577644710f6493e0bd6f3459275355ca61 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 21:51:21 -0700 Subject: [PATCH 095/241] Add remaining tests for VoterList --- frame/voter-bags/src/mock.rs | 4 + frame/voter-bags/src/tests.rs | 9 +- frame/voter-bags/src/voter_list/tests.rs | 216 +++++++++++++++++++++++ 3 files changed, 227 insertions(+), 2 deletions(-) diff --git a/frame/voter-bags/src/mock.rs b/frame/voter-bags/src/mock.rs index 0071f8fef4997..d4d5192e93f54 100644 --- a/frame/voter-bags/src/mock.rs +++ b/frame/voter-bags/src/mock.rs @@ -131,4 +131,8 @@ pub(crate) mod test_utils { pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { bag.iter().map(|n| *n.id()).collect::>() } + + pub(crate) fn get_voter_list_as_ids() -> Vec { + VoterList::::iter().map(|n| *n.id()).collect::>() + } } diff --git a/frame/voter-bags/src/tests.rs b/frame/voter-bags/src/tests.rs index d1861f3db53cd..ffd7540aa605a 100644 --- a/frame/voter-bags/src/tests.rs +++ b/frame/voter-bags/src/tests.rs @@ -1,5 +1,10 @@ -use super::*; -use mock::*; +// use super::*; +// use mock::*; + +#[test] +fn rebag_works() { + todo!(); +} mod voter_bag_list_provider { use super::*; diff --git a/frame/voter-bags/src/voter_list/tests.rs b/frame/voter-bags/src/voter_list/tests.rs index b8d646bf0438f..4aa9db50518f5 100644 --- a/frame/voter-bags/src/voter_list/tests.rs +++ b/frame/voter-bags/src/voter_list/tests.rs @@ -92,6 +92,218 @@ fn remove_last_voter_in_bags_cleans_bag() { ); }); } + + +mod voter_list { + use super::*; + + #[test] + fn iteration_is_semi_sorted() { + ExtBuilder::default() + .add_ids(vec![(51, 2_000), (61, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // then + assert_eq!( + get_voter_list_as_ids(), + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, // last bag. + ] + ); + + // when adding a voter that has a higher weight than pre-existing voters in the bag + VoterList::::insert(71, 10); + + // then + assert_eq!( + get_voter_list_as_ids(), + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, + 71, // last bag; the new voter is last, because it is order of insertion + ] + ); + }) + } + + /// This tests that we can `take` x voters, even if that quantity ends midway through a list. + #[test] + fn take_works() { + ExtBuilder::default() + .add_ids(vec![(51, 2_000), (61, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // when + let iteration = + VoterList::::iter().map(|node| *node.id()).take(4).collect::>(); + + // then + assert_eq!( + iteration, + vec![ + 51, 61, // best bag, fully iterated + 11, 21, // middle bag, partially iterated + ] + ); + }) + } + + #[test] + fn insert_works() { + ExtBuilder::default().build_and_execute(|| { + // when inserting into an existing bag + VoterList::::insert(71, 1_000); + + // then + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 71, 31]); + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71])]); + + // when inserting into a non-existent bag + VoterList::::insert(81, 1_001); + + // then + assert_eq!(get_voter_list_as_ids(), vec![81, 11, 21, 101, 71, 31]); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71]), (2_000, vec![81])] + ); + }); + } + + #[test] + fn remove_works() { + use crate::{CounterForVoters, VoterBags, VoterNodes}; + + let check_storage = |id, counter, voters, bags| { + assert!(!VoterBagFor::::contains_key(id)); + assert!(!VoterNodes::::contains_key(id)); + assert_eq!(CounterForVoters::::get(), counter); + assert_eq!(VoterBagFor::::iter().count() as u32, counter); + assert_eq!(VoterNodes::::iter().count() as u32, counter); + assert_eq!(get_voter_list_as_ids(), voters); + assert_eq!(get_bags(), bags); + }; + + ExtBuilder::default().build_and_execute(|| { + // when removing a non-existent voter + assert!(!VoterBagFor::::contains_key(42)); + assert!(!VoterNodes::::contains_key(42)); + VoterList::::remove(&42); + + // then nothing changes + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(CounterForVoters::::get(), 4); + + // when removing a node from a bag with multiple nodes + VoterList::::remove(&11); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); + check_storage( + 11, + 3, + vec![21, 101, 31], // voter list + vec![(10, vec![31]), (1_000, vec![21, 101])], // bags + ); + + // when removing a node from a bag with only one node: + VoterList::::remove(&31); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101]); + check_storage( + 31, + 2, + vec![21, 101], // voter list + vec![(1_000, vec![21, 101])], // bags + ); + assert!(!VoterBags::::contains_key(10)); // bag 10 is removed + + // remove remaining voters to make sure storage cleans up as expected + VoterList::::remove(&21); + check_storage( + 21, + 1, + vec![101], // voter list + vec![(1_000, vec![101])], // bags + ); + + VoterList::::remove(&101); + check_storage( + 101, + 0, + Vec::::new(), // voter list + vec![], // bags + ); + assert!(!VoterBags::::contains_key(1_000)); // bag 1_000 is removed + + // bags are deleted via removals + assert_eq!(VoterBags::::iter().count(), 0); + }); + } + + #[test] + fn update_position_for_works() { + ExtBuilder::default().build_and_execute(|| { + // given a correctly placed account 31 + let node_31 = Node::::from_id(&31).unwrap(); + assert!(!node_31.is_misplaced(10)); + + // when account 31's weight becomes 20, it is then misplaced. + let weight_20 = 20; + assert!(node_31.is_misplaced(weight_20)); + + // then updating position moves it to the correct bag + assert_eq!(VoterList::::update_position_for(node_31, weight_20), Some((10, 20))); + + assert_eq!(get_bags(), vec![(20, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + + // and if you try and update the position with no change in weight + let node_31 = Node::::from_id(&31).unwrap(); + assert_storage_noop!(assert_eq!( + VoterList::::update_position_for(node_31, weight_20), + None, + )); + + // when account 31 needs to be moved to an existing higher bag + let weight_500 = 500; + + // then updating positions moves it to the correct bag + let node_31 = Node::::from_id(&31).unwrap(); + assert_eq!( + VoterList::::update_position_for(node_31, weight_500), + Some((20, 1_000)) + ); + assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101, 31])]); + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + + // when account 31 has a higher but within its current bag + let weight_1000 = 1_000; + + // then nothing changes + let node_31 = Node::::from_id(&31).unwrap(); + assert_storage_noop!(assert_eq!( + VoterList::::update_position_for(node_31, weight_1000), + None, + )); + }); + } +} mod bags { use super::*; @@ -395,3 +607,7 @@ mod bags { }); } } + +mod node { + use super::*; +} \ No newline at end of file From bf65a34af837c9fccf15998a7e93cfbfa1e02ca6 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 21:54:57 -0700 Subject: [PATCH 096/241] Add tests for node --- frame/voter-bags/src/voter_list/tests.rs | 94 ++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/frame/voter-bags/src/voter_list/tests.rs b/frame/voter-bags/src/voter_list/tests.rs index 4aa9db50518f5..625a11c529343 100644 --- a/frame/voter-bags/src/voter_list/tests.rs +++ b/frame/voter-bags/src/voter_list/tests.rs @@ -610,4 +610,98 @@ mod bags { mod node { use super::*; + + #[test] + fn is_misplaced_works() { + ExtBuilder::default().build_and_execute(|| { + let node_31 = Node::::get(10, &31).unwrap(); + + // given + assert_eq!(node_31.bag_upper, 10); + + // then + assert!(!node_31.is_misplaced(0)); + assert!(!node_31.is_misplaced(9)); + assert!(!node_31.is_misplaced(10)); + assert!(node_31.is_misplaced(11)); + }); + } + + // TODO a test similiar to this should exist in the staking pallet + // #[test] + // fn voting_data_works() { + // ExtBuilder::default().build_and_execute_without_check_count(|| { + // let weight_of = Staking::weight_of_fn(); + + // // add nominator with no targets + // bond_nominator(42, 43, 1_000, vec![11]); + + // // given + // assert_eq!( + // get_voter_list_as_voters(), + // vec![ + // Voter::validator(11), + // Voter::validator(21), + // Voter::nominator(101), + // Voter::nominator(42), + // Voter::validator(31), + // ] + // ); + // assert_eq!(active_era(), 0); + + // let slashing_spans = + // ::SlashingSpans::iter().collect::>(); + // assert_eq!(slashing_spans.keys().len(), 0); // no pre-existing slashing spans + + // let node_11 = Node::::get(10, &11).unwrap(); + // assert_eq!( + // node_11.voting_data(&weight_of, &slashing_spans).unwrap(), + // (11, 1_000, vec![11]) + // ); + + // // getting data for a nominators with 0 slashed targets + // let node_101 = Node::::get(1_000, &101).unwrap(); + // assert_eq!( + // node_101.voting_data(&weight_of, &slashing_spans).unwrap(), + // (101, 500, vec![11, 21]) + // ); + // let node_42 = Node::::get(10, &42).unwrap(); + // assert_eq!( + // node_42.voting_data(&weight_of, &slashing_spans).unwrap(), + // (42, 1_000, vec![11]) + // ); + + // // roll ahead an era so any slashes will be after the previous nominations + // start_active_era(1); + + // // when a validator gets a slash, + // add_slash(&11); + // let slashing_spans = + // ::SlashingSpans::iter().collect::>(); + + // assert_eq!(slashing_spans.keys().cloned().collect::>(), vec![11, 42, 101]); + // // then its node no longer exists + // assert_eq!( + // get_voter_list_as_voters(), + // vec![ + // Voter::validator(21), + // Voter::nominator(101), + // Voter::nominator(42), + // Voter::validator(31), + // ] + // ); + // // and its nominators no longer have it as a target + // let node_101 = Node::::get(10, &101).unwrap(); + // assert_eq!( + // node_101.voting_data(&weight_of, &slashing_spans), + // Some((101, 475, vec![21])), + // ); + + // let node_42 = Node::::get(10, &42).unwrap(); + // assert_eq!( + // node_42.voting_data(&weight_of, &slashing_spans), + // None, // no voting data since its 1 target has been slashed since nominating + // ); + // }); + // } } \ No newline at end of file From 123776c71e07628327215657f041e0b7937c9d5a Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 22:18:14 -0700 Subject: [PATCH 097/241] Add rebag works --- frame/voter-bags/src/lib.rs | 12 ++++---- frame/voter-bags/src/tests.rs | 57 ++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index 0764788d365ac..de555687b6e37 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -167,10 +167,10 @@ pub mod pallet { /// Anyone can call this function about any stash. // #[pallet::weight(T::WeightInfo::rebag())] #[pallet::weight(123456789)] // TODO - pub fn rebag(origin: OriginFor, stash: T::AccountId) -> DispatchResult { + pub fn rebag(origin: OriginFor, account: T::AccountId) -> DispatchResult { ensure_signed(origin)?; - let weight = T::StakingVoteWeight::staking_vote_weight(&stash); - Pallet::::do_rebag(&stash, weight); + let weight = T::StakingVoteWeight::staking_vote_weight(&account); + Pallet::::do_rebag(&account, weight); Ok(()) } } @@ -194,13 +194,13 @@ impl Pallet { /// Move an account from one bag to another, depositing an event on success. /// /// If the account changed bags, returns `Some((from, to))`. - pub fn do_rebag(id: &T::AccountId, new_weight: VoteWeight) -> Option<(VoteWeight, VoteWeight)> { + pub fn do_rebag(account: &T::AccountId, new_weight: VoteWeight) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = voter_list::Node::::from_id(&id) + let maybe_movement = voter_list::Node::::from_id(&account) .and_then(|node| VoterList::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { - Self::deposit_event(Event::::Rebagged(id.clone(), from, to)); + Self::deposit_event(Event::::Rebagged(account.clone(), from, to)); }; maybe_movement } diff --git a/frame/voter-bags/src/tests.rs b/frame/voter-bags/src/tests.rs index ffd7540aa605a..c7a8b85d19ecf 100644 --- a/frame/voter-bags/src/tests.rs +++ b/frame/voter-bags/src/tests.rs @@ -1,11 +1,60 @@ -// use super::*; -// use mock::*; +use frame_support::{assert_ok, assert_storage_noop}; + +use super::*; +use mock::{*, test_utils::*, ext_builder::*, VoterBags as List}; -#[test] fn rebag_works() { - todo!(); + ExtBuilder::default() + .add_ids(vec![(42, 20)]) + .build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); + + // increase vote weight and implicitly rebag to the level of non-existent bag + set_staking_vote_weight(2_000); + assert_ok!(List::rebag(Origin::signed(0), 42)); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // decrease weight within the range of the current bag + set_staking_vote_weight(1_001); + assert_ok!(List::rebag(Origin::signed(0), 42)); + // does not change bags + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // reduce weight to the level of a non-existent bag + set_staking_vote_weight(1_001); + assert_ok!(List::rebag(Origin::signed(0), 42)); + // creates the bag and moves the voter into it + assert_eq!( + get_bags(), + vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] + ); + + // increase weight to a pre-existing bag + set_staking_vote_weight(1_001); + assert_ok!(List::rebag(Origin::signed(0), 42)); + // moves the voter to that bag + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); + }); } +#[test] +fn rebag_tail_works() { + // rebagging the head of a bag results in the old bag having a new head and an overall correct state. +} + + +// #[test] TODO +// fn rebag_head_works() { +// // rebagging the head of a bag results in the old bag having a new head and an overall correct state. +// } + mod voter_bag_list_provider { use super::*; From 2907816604b1eba4b1de50de230cd401a398441c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 22:35:40 -0700 Subject: [PATCH 098/241] Add rebag extrinsic tests --- frame/voter-bags/src/lib.rs | 5 +- frame/voter-bags/src/tests.rs | 83 +++++++++++----- frame/voter-bags/src/voter_list/mod.rs | 5 + frame/voter-bags/src/voter_list/tests.rs | 117 ++++++++++++----------- 4 files changed, 129 insertions(+), 81 deletions(-) diff --git a/frame/voter-bags/src/lib.rs b/frame/voter-bags/src/lib.rs index de555687b6e37..f14c22fe7761c 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/voter-bags/src/lib.rs @@ -194,7 +194,10 @@ impl Pallet { /// Move an account from one bag to another, depositing an event on success. /// /// If the account changed bags, returns `Some((from, to))`. - pub fn do_rebag(account: &T::AccountId, new_weight: VoteWeight) -> Option<(VoteWeight, VoteWeight)> { + pub fn do_rebag( + account: &T::AccountId, + new_weight: VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. let maybe_movement = voter_list::Node::::from_id(&account) diff --git a/frame/voter-bags/src/tests.rs b/frame/voter-bags/src/tests.rs index c7a8b85d19ecf..313f756300fbe 100644 --- a/frame/voter-bags/src/tests.rs +++ b/frame/voter-bags/src/tests.rs @@ -1,59 +1,98 @@ use frame_support::{assert_ok, assert_storage_noop}; use super::*; -use mock::{*, test_utils::*, ext_builder::*, VoterBags as List}; +use mock::{ext_builder::*, test_utils::*, VoterBags as List, *}; +use voter_list::Bag; fn rebag_works() { - ExtBuilder::default() - .add_ids(vec![(42, 20)]) - .build_and_execute(|| { + ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); // increase vote weight and implicitly rebag to the level of non-existent bag set_staking_vote_weight(2_000); assert_ok!(List::rebag(Origin::signed(0), 42)); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])]); // decrease weight within the range of the current bag set_staking_vote_weight(1_001); assert_ok!(List::rebag(Origin::signed(0), 42)); // does not change bags - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])]); // reduce weight to the level of a non-existent bag set_staking_vote_weight(1_001); assert_ok!(List::rebag(Origin::signed(0), 42)); // creates the bag and moves the voter into it - assert_eq!( - get_bags(), - vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] - ); + assert_eq!(get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101])]); // increase weight to a pre-existing bag set_staking_vote_weight(1_001); assert_ok!(List::rebag(Origin::signed(0), 42)); // moves the voter to that bag - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42])]); }); } +// Rebagging the tail of a bag results in the old bag having a new tail and an overall correct state. #[test] fn rebag_tail_works() { - // rebagging the head of a bag results in the old bag having a new head and an overall correct state. + ExtBuilder::default().build_and_execute(|| { + // when + set_staking_vote_weight(10); + assert_ok!(List::rebag(Origin::signed(0), 101)); + + // then + assert_eq!(get_bags(), vec![(10, vec![31, 101]), (1000, vec![11, 21])]); + assert_eq!( + Bag::::get(1_000).unwrap(), + Bag::new(Some(11), Some(21), 1_000) + ); + + // when + assert_ok!(List::rebag(Origin::signed(0), 21)); + + // then + assert_eq!(get_bags(), vec![(10, vec![31, 101, 21]), (1000, vec![11])]); + }); } +// Rebagging the head of a bag results in the old bag having a new head and an overall correct state. +#[test] +fn rebag_head_works() { + ExtBuilder::default().build_and_execute(|| { + // when + set_staking_vote_weight(10); + assert_ok!(List::rebag(Origin::signed(0), 11)); + + // then + assert_eq!(get_bags(), vec![(10, vec![31, 11]), (1000, vec![21, 101])]); + assert_eq!( + Bag::::get(1_000).unwrap(), + Bag::new(Some(21), Some(101), 1_000) + ); + + // when + assert_ok!(List::rebag(Origin::signed(0), 21)); + + // then + assert_eq!(get_bags(), vec![(10, vec![31, 11, 21]), (1000, vec![101])]); + assert_eq!( + Bag::::get(1_000).unwrap(), + Bag::new(Some(101), Some(101), 1_000) + ); + + // when + assert_ok!(List::rebag(Origin::signed(0), 101)); -// #[test] TODO -// fn rebag_head_works() { -// // rebagging the head of a bag results in the old bag having a new head and an overall correct state. -// } + // then + assert_eq!(get_bags(), vec![(10, vec![31, 11, 21, 101])]); + assert_eq!( + Bag::::get(1_000), + None + ); + }); +} mod voter_bag_list_provider { use super::*; diff --git a/frame/voter-bags/src/voter_list/mod.rs b/frame/voter-bags/src/voter_list/mod.rs index 49ae91dcc8c9e..7c677061ee099 100644 --- a/frame/voter-bags/src/voter_list/mod.rs +++ b/frame/voter-bags/src/voter_list/mod.rs @@ -398,6 +398,11 @@ pub struct Bag { } impl Bag { + #[cfg(test)] + pub(crate) fn new(head: Option, tail: Option, bag_upper: VoteWeight) -> Self { + Self { head, tail, bag_upper } + } + /// Get a bag by its upper vote weight. pub(crate) fn get(bag_upper: VoteWeight) -> Option> { debug_assert!( diff --git a/frame/voter-bags/src/voter_list/tests.rs b/frame/voter-bags/src/voter_list/tests.rs index 625a11c529343..3f0dc45b0132a 100644 --- a/frame/voter-bags/src/voter_list/tests.rs +++ b/frame/voter-bags/src/voter_list/tests.rs @@ -93,72 +93,71 @@ fn remove_last_voter_in_bags_cleans_bag() { }); } - mod voter_list { use super::*; #[test] fn iteration_is_semi_sorted() { ExtBuilder::default() - .add_ids(vec![(51, 2_000), (61, 2_000)]) - .build_and_execute(|| { - // given - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], - ); - - // then - assert_eq!( - get_voter_list_as_ids(), - vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, // last bag. - ] - ); - - // when adding a voter that has a higher weight than pre-existing voters in the bag - VoterList::::insert(71, 10); - - // then - assert_eq!( - get_voter_list_as_ids(), - vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, - 71, // last bag; the new voter is last, because it is order of insertion - ] - ); - }) + .add_ids(vec![(51, 2_000), (61, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // then + assert_eq!( + get_voter_list_as_ids(), + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, // last bag. + ] + ); + + // when adding a voter that has a higher weight than pre-existing voters in the bag + VoterList::::insert(71, 10); + + // then + assert_eq!( + get_voter_list_as_ids(), + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, + 71, // last bag; the new voter is last, because it is order of insertion + ] + ); + }) } /// This tests that we can `take` x voters, even if that quantity ends midway through a list. #[test] fn take_works() { ExtBuilder::default() - .add_ids(vec![(51, 2_000), (61, 2_000)]) - .build_and_execute(|| { - // given - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], - ); - - // when - let iteration = - VoterList::::iter().map(|node| *node.id()).take(4).collect::>(); - - // then - assert_eq!( - iteration, - vec![ - 51, 61, // best bag, fully iterated - 11, 21, // middle bag, partially iterated - ] - ); - }) + .add_ids(vec![(51, 2_000), (61, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // when + let iteration = + VoterList::::iter().map(|node| *node.id()).take(4).collect::>(); + + // then + assert_eq!( + iteration, + vec![ + 51, 61, // best bag, fully iterated + 11, 21, // middle bag, partially iterated + ] + ); + }) } #[test] @@ -268,7 +267,10 @@ mod voter_list { assert!(node_31.is_misplaced(weight_20)); // then updating position moves it to the correct bag - assert_eq!(VoterList::::update_position_for(node_31, weight_20), Some((10, 20))); + assert_eq!( + VoterList::::update_position_for(node_31, weight_20), + Some((10, 20)) + ); assert_eq!(get_bags(), vec![(20, vec![31]), (1_000, vec![11, 21, 101])]); assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); @@ -447,8 +449,7 @@ mod bags { // and the re-fetched node has bad pointers assert_eq!( Node::::get(1_000, &11).unwrap(), - node(11, Some(101), None, bag_1000.bag_upper) - // ^^^ despite being the bags head, it has a prev + node(11, Some(101), None, bag_1000.bag_upper) // ^^^ despite being the bags head, it has a prev ); assert_eq!(bag_1000, Bag { head: Some(11), tail: Some(11), bag_upper: 1_000 }) @@ -704,4 +705,4 @@ mod node { // ); // }); // } -} \ No newline at end of file +} From 286d07f91fe23fa950b863e9fd3700aca2ab5578 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 23:05:25 -0700 Subject: [PATCH 099/241] Rename to bags-list and name the list .. list! --- Cargo.lock | 39 +++++++++---------- Cargo.toml | 2 +- bin/node/runtime/voter-bags/src/main.rs | 32 +++++++-------- frame/{voter-bags => bags-list}/Cargo.toml | 4 +- frame/{voter-bags => bags-list}/src/lib.rs | 26 ++++++------- .../voter_list => bags-list/src/list}/mod.rs | 16 ++++---- .../src/list}/tests.rs | 36 ++++++++--------- .../src/make_bags.rs | 0 frame/{voter-bags => bags-list}/src/mock.rs | 10 ++--- frame/{voter-bags => bags-list}/src/tests.rs | 24 ++++++------ .../{voter-bags => bags-list}/src/weights.rs | 0 frame/staking/Cargo.toml | 2 +- frame/staking/src/mock.rs | 1 + 13 files changed, 96 insertions(+), 96 deletions(-) rename frame/{voter-bags => bags-list}/Cargo.toml (96%) rename frame/{voter-bags => bags-list}/src/lib.rs (92%) rename frame/{voter-bags/src/voter_list => bags-list/src/list}/mod.rs (98%) rename frame/{voter-bags/src/voter_list => bags-list/src/list}/tests.rs (96%) rename frame/{voter-bags => bags-list}/src/make_bags.rs (100%) rename frame/{voter-bags => bags-list}/src/mock.rs (93%) rename frame/{voter-bags => bags-list}/src/tests.rs (80%) rename frame/{voter-bags => bags-list}/src/weights.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 78f1c1ef02b7e..690be865739ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4877,6 +4877,25 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-bags-list" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-election-provider-support", + "frame-support", + "frame-system", + "log", + "pallet-balances", + "pallet-staking", + "parity-scale-codec", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", + "sp-tracing", +] + [[package]] name = "pallet-balances" version = "4.0.0-dev" @@ -5558,7 +5577,6 @@ dependencies = [ "pallet-session", "pallet-staking-reward-curve", "pallet-timestamp", - "pallet-voter-bags", "parity-scale-codec", "parking_lot 0.11.1", "paste 1.0.4", @@ -5790,25 +5808,6 @@ dependencies = [ "sp-storage", ] -[[package]] -name = "pallet-voter-bags" -version = "4.0.0-dev" -dependencies = [ - "frame-benchmarking", - "frame-election-provider-support", - "frame-support", - "frame-system", - "log", - "pallet-balances", - "pallet-staking", - "parity-scale-codec", - "sp-core", - "sp-io", - "sp-runtime", - "sp-std", - "sp-tracing", -] - [[package]] name = "parity-db" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index 4b51297f7cafd..5d695a0cd7cf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,7 +130,7 @@ members = [ "frame/uniques", "frame/utility", "frame/vesting", - "frame/voter-bags", + "frame/bags-list", "primitives/api", "primitives/api/proc-macro", "primitives/api/test", diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index a92af37fb5bf8..ca4454839547c 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,22 +17,22 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::voter_bags::make_bags::generate_thresholds_module; -use std::path::PathBuf; -use structopt::StructOpt; +// use pallet_staking::voter_bags::make_bags::generate_thresholds_module; +// use std::path::PathBuf; +// use structopt::StructOpt; -#[derive(Debug, StructOpt)] -struct Opt { - /// How many bags to generate. - #[structopt(long, default_value = "200")] - n_bags: usize, +// #[derive(Debug, StructOpt)] +// struct Opt { +// /// How many bags to generate. +// #[structopt(long, default_value = "200")] +// n_bags: usize, - /// Where to write the output. - output: PathBuf, -} +// /// Where to write the output. +// output: PathBuf, +// } -fn main() -> Result<(), std::io::Error> { - let Opt { n_bags, output } = Opt::from_args(); - let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) -} +// fn main() -> Result<(), std::io::Error> { +// let Opt { n_bags, output } = Opt::from_args(); +// let mut ext = sp_io::TestExternalities::new_empty(); +// ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) +// } diff --git a/frame/voter-bags/Cargo.toml b/frame/bags-list/Cargo.toml similarity index 96% rename from frame/voter-bags/Cargo.toml rename to frame/bags-list/Cargo.toml index 4e80cbb7dec0f..e637bb6c295e4 100644 --- a/frame/voter-bags/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "pallet-voter-bags" +name = "pallet-bags-list" version = "4.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -description = "FRAME pallet voter bags" +description = "FRAME pallet bags list" readme = "README.md" [package.metadata.docs.rs] diff --git a/frame/voter-bags/src/lib.rs b/frame/bags-list/src/lib.rs similarity index 92% rename from frame/voter-bags/src/lib.rs rename to frame/bags-list/src/lib.rs index f14c22fe7761c..3a6c9fbcf3ecc 100644 --- a/frame/voter-bags/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -28,13 +28,13 @@ use frame_system::ensure_signed; mod mock; #[cfg(test)] mod tests; -mod voter_list; +mod list; pub mod weights; pub use pallet::*; pub use weights::WeightInfo; -use voter_list::VoterList; +use list::List; pub(crate) const LOG_TARGET: &'static str = "runtime::voter_bags"; @@ -116,7 +116,7 @@ pub mod pallet { /// # Migration /// /// In the event that this list ever changes, a copy of the old bags list must be retained. - /// With that `VoterList::migrate` can be called, which will perform the appropriate + /// With that `List::migrate` can be called, which will perform the appropriate /// migration. #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; @@ -133,7 +133,7 @@ pub mod pallet { /// However, the `Node` data structure has helpers which can provide that information. #[pallet::storage] pub(crate) type VoterNodes = - StorageMap<_, Twox64Concat, T::AccountId, voter_list::Node>; + StorageMap<_, Twox64Concat, T::AccountId, list::Node>; /// Which bag currently contains a particular voter. /// @@ -146,7 +146,7 @@ pub mod pallet { /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] pub(crate) type VoterBags = - StorageMap<_, Twox64Concat, VoteWeight, voter_list::Bag>; + StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] @@ -200,8 +200,8 @@ impl Pallet { ) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = voter_list::Node::::from_id(&account) - .and_then(|node| VoterList::update_position_for(node, new_weight)); + let maybe_movement = list::Node::::from_id(&account) + .and_then(|node| List::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { Self::deposit_event(Event::::Rebagged(account.clone(), from, to)); }; @@ -209,13 +209,13 @@ impl Pallet { } } -pub struct VoterBagsVoterListProvider(sp_std::marker::PhantomData); +pub struct BagsListVoterListProvider(sp_std::marker::PhantomData); impl frame_election_provider_support::VoterListProvider - for VoterBagsVoterListProvider + for BagsListVoterListProvider { /// Returns iterator over voter list, which can have `take` called on it. fn get_voters() -> Box> { - Box::new(VoterList::::iter().map(|n| n.id().clone())) + Box::new(List::::iter().map(|n| n.id().clone())) } fn count() -> u32 { @@ -223,7 +223,7 @@ impl frame_election_provider_support::VoterListProvider } fn on_insert(voter: T::AccountId, weight: VoteWeight) { - VoterList::::insert(voter, weight); + List::::insert(voter, weight); } /// Hook for updating a voter in the list (unused). @@ -233,10 +233,10 @@ impl frame_election_provider_support::VoterListProvider /// Hook for removing a voter from the list. fn on_remove(voter: &T::AccountId) { - VoterList::::remove(voter) + List::::remove(voter) } fn sanity_check() -> Result<(), &'static str> { - VoterList::::sanity_check() + List::::sanity_check() } } diff --git a/frame/voter-bags/src/voter_list/mod.rs b/frame/bags-list/src/list/mod.rs similarity index 98% rename from frame/voter-bags/src/voter_list/mod.rs rename to frame/bags-list/src/list/mod.rs index 7c677061ee099..b9ba650d24d24 100644 --- a/frame/voter-bags/src/voter_list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! VoterList implementation. +//! List implementation. use crate::Config; use codec::{Decode, Encode}; @@ -66,9 +66,9 @@ fn current_bag_for(id: &T::AccountId) -> Option { /// iteration at any desired point; only those voters in the lowest bag (who are known to have /// relatively little power to affect the outcome) can be excluded. This satisfies both the desire /// for fairness and the requirement for efficiency. -pub struct VoterList(PhantomData); +pub struct List(PhantomData); -impl VoterList { +impl List { /// Remove all data associated with the voter list from storage. // #[cfg(test)] // TODO this is used for regenerate // pub fn clear() { @@ -353,7 +353,7 @@ impl VoterList { /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are - /// checked per *any* update to `VoterList`. + /// checked per *any* update to `List`. pub(crate) fn sanity_check() -> Result<(), &'static str> { let mut seen_in_list = BTreeSet::new(); ensure!( @@ -456,7 +456,7 @@ impl Bag { /// Insert a new voter into this bag. /// /// This is private on purpose because it's naive: it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// appropriate bag for this voter at all. Generally, use [`List::insert`] instead. /// /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. @@ -467,7 +467,7 @@ impl Bag { /// Insert a voter node into this bag. /// /// This is private on purpose because it's naive; it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// appropriate bag for this voter at all. Generally, use [`List::insert`] instead. /// /// Storage note: this modifies storage, but only for the node. You still need to call /// `self.put()` after use. @@ -505,7 +505,7 @@ impl Bag { /// Remove a voter node from this bag. /// /// This is private on purpose because it doesn't check whether this bag contains the voter in - /// the first place. Generally, use [`VoterList::remove`] instead. + /// the first place. Generally, use [`List::remove`] instead. /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` @@ -534,7 +534,7 @@ impl Bag { /// Sanity check this bag. /// /// Should be called by the call-site, after each mutating operation on a bag. The call site of - /// this struct is always `VoterList`. + /// this struct is always `List`. /// /// * Ensures head has no prev. /// * Ensures tail has no next. diff --git a/frame/voter-bags/src/voter_list/tests.rs b/frame/bags-list/src/list/tests.rs similarity index 96% rename from frame/voter-bags/src/voter_list/tests.rs rename to frame/bags-list/src/list/tests.rs index 3f0dc45b0132a..f09fdcc31d9da 100644 --- a/frame/voter-bags/src/voter_list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -44,7 +44,7 @@ fn setup_works() { // iteration of the bags would yield: assert_eq!( - VoterList::::iter().map(|n| *n.id()).collect::>(), + List::::iter().map(|n| *n.id()).collect::>(), vec![11, 21, 101, 31], // ^^ note the order of insertion in genesis! ); @@ -78,14 +78,14 @@ fn remove_last_voter_in_bags_cleans_bag() { assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); // Bump 31 to a bigger bag - VoterList::::remove(&31); - VoterList::::insert(31, 10_000); + List::::remove(&31); + List::::insert(31, 10_000); // then the bag with bound 10 is wiped from storage. assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101]), (10_000, vec![31])]); // and can be recreated again as needed - VoterList::::insert(77, 10); + List::::insert(77, 10); assert_eq!( get_bags(), vec![(10, vec![77]), (1_000, vec![11, 21, 101]), (10_000, vec![31])] @@ -118,7 +118,7 @@ mod voter_list { ); // when adding a voter that has a higher weight than pre-existing voters in the bag - VoterList::::insert(71, 10); + List::::insert(71, 10); // then assert_eq!( @@ -147,7 +147,7 @@ mod voter_list { // when let iteration = - VoterList::::iter().map(|node| *node.id()).take(4).collect::>(); + List::::iter().map(|node| *node.id()).take(4).collect::>(); // then assert_eq!( @@ -164,14 +164,14 @@ mod voter_list { fn insert_works() { ExtBuilder::default().build_and_execute(|| { // when inserting into an existing bag - VoterList::::insert(71, 1_000); + List::::insert(71, 1_000); // then assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 71, 31]); assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71])]); // when inserting into a non-existent bag - VoterList::::insert(81, 1_001); + List::::insert(81, 1_001); // then assert_eq!(get_voter_list_as_ids(), vec![81, 11, 21, 101, 71, 31]); @@ -200,7 +200,7 @@ mod voter_list { // when removing a non-existent voter assert!(!VoterBagFor::::contains_key(42)); assert!(!VoterNodes::::contains_key(42)); - VoterList::::remove(&42); + List::::remove(&42); // then nothing changes assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); @@ -208,7 +208,7 @@ mod voter_list { assert_eq!(CounterForVoters::::get(), 4); // when removing a node from a bag with multiple nodes - VoterList::::remove(&11); + List::::remove(&11); // then assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); @@ -220,7 +220,7 @@ mod voter_list { ); // when removing a node from a bag with only one node: - VoterList::::remove(&31); + List::::remove(&31); // then assert_eq!(get_voter_list_as_ids(), vec![21, 101]); @@ -233,7 +233,7 @@ mod voter_list { assert!(!VoterBags::::contains_key(10)); // bag 10 is removed // remove remaining voters to make sure storage cleans up as expected - VoterList::::remove(&21); + List::::remove(&21); check_storage( 21, 1, @@ -241,7 +241,7 @@ mod voter_list { vec![(1_000, vec![101])], // bags ); - VoterList::::remove(&101); + List::::remove(&101); check_storage( 101, 0, @@ -268,7 +268,7 @@ mod voter_list { // then updating position moves it to the correct bag assert_eq!( - VoterList::::update_position_for(node_31, weight_20), + List::::update_position_for(node_31, weight_20), Some((10, 20)) ); @@ -278,7 +278,7 @@ mod voter_list { // and if you try and update the position with no change in weight let node_31 = Node::::from_id(&31).unwrap(); assert_storage_noop!(assert_eq!( - VoterList::::update_position_for(node_31, weight_20), + List::::update_position_for(node_31, weight_20), None, )); @@ -288,7 +288,7 @@ mod voter_list { // then updating positions moves it to the correct bag let node_31 = Node::::from_id(&31).unwrap(); assert_eq!( - VoterList::::update_position_for(node_31, weight_500), + List::::update_position_for(node_31, weight_500), Some((20, 1_000)) ); assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101, 31])]); @@ -300,7 +300,7 @@ mod voter_list { // then nothing changes let node_31 = Node::::from_id(&31).unwrap(); assert_storage_noop!(assert_eq!( - VoterList::::update_position_for(node_31, weight_1000), + List::::update_position_for(node_31, weight_1000), None, )); }); @@ -342,7 +342,7 @@ mod bags { }); // when we make a pre-existing bag empty - VoterList::::remove(&31); + List::::remove(&31); // then assert_eq!(Bag::::get(existing_bag_uppers[0]), None) diff --git a/frame/voter-bags/src/make_bags.rs b/frame/bags-list/src/make_bags.rs similarity index 100% rename from frame/voter-bags/src/make_bags.rs rename to frame/bags-list/src/make_bags.rs diff --git a/frame/voter-bags/src/mock.rs b/frame/bags-list/src/mock.rs similarity index 93% rename from frame/voter-bags/src/mock.rs rename to frame/bags-list/src/mock.rs index d4d5192e93f54..076cc33d27f37 100644 --- a/frame/voter-bags/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -70,7 +70,7 @@ frame_support::construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic, { System: frame_system::{Pallet, Call, Storage, Event, Config}, - VoterBags: crate::{Pallet, Call, Storage, Event}, + PalletBagsList: crate::{Pallet, Call, Storage, Event}, } ); @@ -86,7 +86,7 @@ pub(crate) mod ext_builder { } impl ExtBuilder { - /// Add some AccountIds to insert into `VoterList`. + /// Add some AccountIds to insert into `List`. pub(crate) fn add_ids(mut self, ids: Vec<(AccountId, VoteWeight)>) -> Self { self.ids = ids; self @@ -100,7 +100,7 @@ pub(crate) mod ext_builder { let mut ext = sp_io::TestExternalities::from(storage); ext.execute_with(|| { for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { - VoterList::::insert(*id, *weight); + List::::insert(*id, *weight); } }); @@ -115,7 +115,7 @@ pub(crate) mod ext_builder { pub(crate) mod test_utils { use super::*; - use voter_list::Bag; + use list::Bag; /// Returns the nodes of all non-empty bags. pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { @@ -133,6 +133,6 @@ pub(crate) mod test_utils { } pub(crate) fn get_voter_list_as_ids() -> Vec { - VoterList::::iter().map(|n| *n.id()).collect::>() + List::::iter().map(|n| *n.id()).collect::>() } } diff --git a/frame/voter-bags/src/tests.rs b/frame/bags-list/src/tests.rs similarity index 80% rename from frame/voter-bags/src/tests.rs rename to frame/bags-list/src/tests.rs index 313f756300fbe..c1254bad84bc7 100644 --- a/frame/voter-bags/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,8 +1,8 @@ use frame_support::{assert_ok, assert_storage_noop}; use super::*; -use mock::{ext_builder::*, test_utils::*, VoterBags as List, *}; -use voter_list::Bag; +use mock::{ext_builder::*, test_utils::*, *}; +use list::Bag; fn rebag_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { @@ -11,24 +11,24 @@ fn rebag_works() { // increase vote weight and implicitly rebag to the level of non-existent bag set_staking_vote_weight(2_000); - assert_ok!(List::rebag(Origin::signed(0), 42)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])]); // decrease weight within the range of the current bag set_staking_vote_weight(1_001); - assert_ok!(List::rebag(Origin::signed(0), 42)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); // does not change bags assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])]); // reduce weight to the level of a non-existent bag set_staking_vote_weight(1_001); - assert_ok!(List::rebag(Origin::signed(0), 42)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); // creates the bag and moves the voter into it assert_eq!(get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101])]); // increase weight to a pre-existing bag set_staking_vote_weight(1_001); - assert_ok!(List::rebag(Origin::signed(0), 42)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); // moves the voter to that bag assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42])]); }); @@ -40,7 +40,7 @@ fn rebag_tail_works() { ExtBuilder::default().build_and_execute(|| { // when set_staking_vote_weight(10); - assert_ok!(List::rebag(Origin::signed(0), 101)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); // then assert_eq!(get_bags(), vec![(10, vec![31, 101]), (1000, vec![11, 21])]); @@ -50,7 +50,7 @@ fn rebag_tail_works() { ); // when - assert_ok!(List::rebag(Origin::signed(0), 21)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); // then assert_eq!(get_bags(), vec![(10, vec![31, 101, 21]), (1000, vec![11])]); @@ -63,7 +63,7 @@ fn rebag_head_works() { ExtBuilder::default().build_and_execute(|| { // when set_staking_vote_weight(10); - assert_ok!(List::rebag(Origin::signed(0), 11)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 11)); // then assert_eq!(get_bags(), vec![(10, vec![31, 11]), (1000, vec![21, 101])]); @@ -73,7 +73,7 @@ fn rebag_head_works() { ); // when - assert_ok!(List::rebag(Origin::signed(0), 21)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); // then assert_eq!(get_bags(), vec![(10, vec![31, 11, 21]), (1000, vec![101])]); @@ -83,7 +83,7 @@ fn rebag_head_works() { ); // when - assert_ok!(List::rebag(Origin::signed(0), 101)); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); // then assert_eq!(get_bags(), vec![(10, vec![31, 11, 21, 101])]); @@ -94,7 +94,7 @@ fn rebag_head_works() { }); } -mod voter_bag_list_provider { +mod bags_list_voter_list_provider { use super::*; #[test] diff --git a/frame/voter-bags/src/weights.rs b/frame/bags-list/src/weights.rs similarity index 100% rename from frame/voter-bags/src/weights.rs rename to frame/bags-list/src/weights.rs diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 52607554f4d39..ef569f561b68e 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -53,7 +53,7 @@ pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } # TODO: all staking tests should work with this AND the staking stub. Similar to balances pallet we # can create two different mocks with a macro helper (low priority). -pallet-voter-bags = { version = "4.0.0-dev", path = "../voter-bags" } +# pallet-bags-list = { version = "4.0.0-dev", path = "../bags-list" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } frame-election-provider-support = { version = "4.0.0-dev", features = ["runtime-benchmarks"], path = "../election-provider-support" } diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 7486573fe9d90..1452a9ff18335 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -106,6 +106,7 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, + // VoterBags: pallet_voter_bags::{Pallet, Call, Storage, Event}, } ); From 2b7dc5ee58227a44a5c27a1ef23174e4d15e224f Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 23:09:28 -0700 Subject: [PATCH 100/241] Rename VoterBagThresholds => BagThresholds --- frame/bags-list/src/lib.rs | 13 ++++++++----- frame/bags-list/src/list/mod.rs | 28 ++++++++++++++-------------- frame/bags-list/src/list/tests.rs | 4 ++-- frame/bags-list/src/mock.rs | 6 +++--- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 3a6c9fbcf3ecc..2f3160422f9c8 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -103,11 +103,11 @@ pub mod pallet { /// /// # Examples /// - /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, + /// - If `BagThresholds::get().is_empty()`, then all voters are put into the same bag, /// and iteration is strictly in insertion order. - /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined + /// - If `BagThresholds::get().len() == 64`, and the thresholds are determined /// according to the procedure given above, then the constant ratio is equal to 2. - /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined + /// - If `BagThresholds::get().len() == 200`, and the thresholds are determined /// according to the procedure given above, then the constant ratio is approximately equal /// to 1.248. /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will @@ -119,7 +119,10 @@ pub mod pallet { /// With that `List::migrate` can be called, which will perform the appropriate /// migration. #[pallet::constant] - type VoterBagThresholds: Get<&'static [VoteWeight]>; + type BagThresholds: Get<&'static [VoteWeight]>; + + // TODO VoteWeight type could be made generic? + // TODO Node.id type could be made generic? } /// How many voters are registered. @@ -181,7 +184,7 @@ pub mod pallet { sp_std::if_std! { sp_io::TestExternalities::new_empty().execute_with(|| { assert!( - T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), + T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), "Voter bag thresholds must strictly increase", ); }); diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index b9ba650d24d24..81633d419fc11 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -35,16 +35,16 @@ mod tests; /// Given a certain vote weight, which bag should contain this voter? /// /// Bags are identified by their upper threshold; the value returned by this function is guaranteed -/// to be a member of `T::VoterBagThresholds`. +/// to be a member of `T::BagThresholds`. /// -/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, +/// This is used instead of a simpler scheme, such as the index within `T::BagThresholds`, /// because in the event that bags are inserted or deleted, the number of affected voters which need /// to be migrated is smaller. /// /// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this /// function behaves as if it does. fn notional_bag_for(weight: VoteWeight) -> VoteWeight { - let thresholds = T::VoterBagThresholds::get(); + let thresholds = T::BagThresholds::get(); let idx = thresholds.partition_point(|&threshold| weight > threshold); thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } @@ -88,11 +88,11 @@ impl List { /// /// - `old_thresholds` is the previous list of thresholds. /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::VoterBagThresholds` has already been updated. + /// - `T::BagThresholds` has already been updated. /// /// Postconditions: /// - /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. /// - No voter is changed unless required to by the difference between the old threshold list /// and the new. /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the @@ -105,7 +105,7 @@ impl List { ); let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); + let new_set: BTreeSet<_> = T::BagThresholds::get().iter().copied().collect(); let mut affected_accounts = BTreeSet::new(); let mut affected_old_bags = BTreeSet::new(); @@ -158,7 +158,7 @@ impl List { debug_assert!( { - let thresholds = T::VoterBagThresholds::get(); + let thresholds = T::BagThresholds::get(); crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) }, "all `bag_upper` in storage must be members of the new thresholds", @@ -202,13 +202,13 @@ impl List { /// Full iteration can be expensive; it's recommended to limit the number of items with /// `.take(n)`. pub fn iter() -> impl Iterator> { - // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to + // We need a touch of special handling here: because we permit `T::BagThresholds` to // omit the final bound, we need to ensure that we explicitly include that threshold in the // list. // // It's important to retain the ability to omit the final bound because it makes tests much - // easier; they can just configure `type VoterBagThresholds = ()`. - let thresholds = T::VoterBagThresholds::get(); + // easier; they can just configure `type BagThresholds = ()`. + let thresholds = T::BagThresholds::get(); let iter = thresholds.iter().copied(); let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { // in the event that they included it, we can just pass the iterator through unchanged. @@ -369,7 +369,7 @@ impl List { // let nominators = staking::CounterForNominators::::get(); // ensure!(validators + nominators == stored_count, "validators + nominators != voters"); - let _ = T::VoterBagThresholds::get() + let _ = T::BagThresholds::get() .into_iter() .map(|t| Bag::::get(*t).unwrap_or_default()) .map(|b| b.sanity_check()) @@ -406,7 +406,7 @@ impl Bag { /// Get a bag by its upper vote weight. pub(crate) fn get(bag_upper: VoteWeight) -> Option> { debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" ); crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { @@ -418,7 +418,7 @@ impl Bag { /// Get a bag by its upper vote weight or make it, appropriately initialized. fn get_or_make(bag_upper: VoteWeight) -> Bag { debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" ); Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) @@ -591,7 +591,7 @@ impl Node { /// Get a node by bag idx and account id. fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { debug_assert!( - T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" ); crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index f09fdcc31d9da..3f2f06f126791 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -63,7 +63,7 @@ fn notional_bag_for_works() { // at a threshold gives that threshold. assert_eq!(notional_bag_for::(10), 10); - let max_explicit_threshold = *::VoterBagThresholds::get().last().unwrap(); + let max_explicit_threshold = *::BagThresholds::get().last().unwrap(); assert_eq!(max_explicit_threshold, 10_000); // if the max explicit threshold is less than VoteWeight::MAX, assert!(VoteWeight::MAX > max_explicit_threshold); @@ -332,7 +332,7 @@ mod bags { check_bag(existing_bag_uppers[1], Some(11), Some(101), vec![11, 21, 101]); // and all other uppers don't get bags. - ::VoterBagThresholds::get() + ::BagThresholds::get() .iter() .chain(iter::once(&VoteWeight::MAX)) .filter(|bag_upper| !existing_bag_uppers.contains(bag_upper)) diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 076cc33d27f37..453fe4283da34 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -52,12 +52,12 @@ impl frame_system::Config for Runtime { const THRESHOLDS: [VoteWeight; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; parameter_types! { - pub const VoterBagThresholds: &'static [VoteWeight] = &THRESHOLDS; + pub const BagThresholds: &'static [VoteWeight] = &THRESHOLDS; } impl crate::Config for Runtime { type Event = Event; - type VoterBagThresholds = VoterBagThresholds; + type BagThresholds = BagThresholds; type StakingVoteWeight = StakingMock; } @@ -119,7 +119,7 @@ pub(crate) mod test_utils { /// Returns the nodes of all non-empty bags. pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { - VoterBagThresholds::get() + BagThresholds::get() .into_iter() .filter_map(|t| { Bag::::get(*t) From 7d6f221c384a7e2d45805d89dd77da2fc1049385 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 23:24:54 -0700 Subject: [PATCH 101/241] Add test count_works --- frame/bags-list/src/tests.rs | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index c1254bad84bc7..3b3392b01a0b7 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,10 +1,14 @@ -use frame_support::{assert_ok, assert_storage_noop}; +use frame_support::{assert_ok}; use super::*; use mock::{ext_builder::*, test_utils::*, *}; use list::Bag; +use frame_election_provider_support::VoterListProvider; -fn rebag_works() { +mod extrinsics { + use super::*; + + fn rebag_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); @@ -93,18 +97,39 @@ fn rebag_head_works() { ); }); } +} + mod bags_list_voter_list_provider { use super::*; #[test] fn get_voters_works() { - todo!(); + ExtBuilder::default().build_and_execute(|| { + let expected = vec![11, 21, 101, 31]; + for (i, id) in BagsListVoterListProvider::::get_voters().enumerate() { + assert_eq!(id, expected[i]) + } + }); } #[test] fn count_works() { - todo!(); + ExtBuilder::default().build_and_execute(|| { + assert_eq!(BagsListVoterListProvider::::count(), 4); + + BagsListVoterListProvider::::on_insert(201, 0); + assert_eq!(BagsListVoterListProvider::::count(), 5); + + BagsListVoterListProvider::::on_remove(&201); + assert_eq!(BagsListVoterListProvider::::count(), 4); + + BagsListVoterListProvider::::on_remove(&31); + assert_eq!(BagsListVoterListProvider::::count(), 3); + + BagsListVoterListProvider::::on_remove(&21); + assert_eq!(BagsListVoterListProvider::::count(), 3); + }); } #[test] From b1f98e77065c12f69c1628b47ebb3017c5870e6d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 3 Aug 2021 23:56:49 -0700 Subject: [PATCH 102/241] Test on_update_works --- frame/bags-list/src/lib.rs | 5 +- frame/bags-list/src/list/mod.rs | 6 +- frame/bags-list/src/list/tests.rs | 5 +- frame/bags-list/src/mock.rs | 1 + frame/bags-list/src/tests.rs | 238 +++++++++++++++++++----------- 5 files changed, 163 insertions(+), 92 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 2f3160422f9c8..373cd82612cd6 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -24,11 +24,11 @@ use frame_election_provider_support::VoteWeight; use frame_system::ensure_signed; +mod list; #[cfg(test)] mod mock; #[cfg(test)] mod tests; -mod list; pub mod weights; pub use pallet::*; @@ -148,8 +148,7 @@ pub mod pallet { /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] - pub(crate) type VoterBags = - StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; + pub(crate) type VoterBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 81633d419fc11..d2dcf0a08bee0 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -399,7 +399,11 @@ pub struct Bag { impl Bag { #[cfg(test)] - pub(crate) fn new(head: Option, tail: Option, bag_upper: VoteWeight) -> Self { + pub(crate) fn new( + head: Option, + tail: Option, + bag_upper: VoteWeight, + ) -> Self { Self { head, tail, bag_upper } } diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 3f2f06f126791..fc7c8dd1e73e9 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -267,10 +267,7 @@ mod voter_list { assert!(node_31.is_misplaced(weight_20)); // then updating position moves it to the correct bag - assert_eq!( - List::::update_position_for(node_31, weight_20), - Some((10, 20)) - ); + assert_eq!(List::::update_position_for(node_31, weight_20), Some((10, 20))); assert_eq!(get_bags(), vec![(20, vec![31]), (1_000, vec![11, 21, 101])]); assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 453fe4283da34..dc5dc6cd29368 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -121,6 +121,7 @@ pub(crate) mod test_utils { pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { BagThresholds::get() .into_iter() + .chain(std::iter::once(&VoteWeight::MAX)) .filter_map(|t| { Bag::::get(*t) .map(|bag| (*t, bag.iter().map(|n| n.id().clone()).collect::>())) diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 3b3392b01a0b7..42e8f911e962f 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,105 +1,98 @@ -use frame_support::{assert_ok}; +use frame_support::assert_ok; use super::*; -use mock::{ext_builder::*, test_utils::*, *}; -use list::Bag; use frame_election_provider_support::VoterListProvider; +use list::Bag; +use mock::{ext_builder::*, test_utils::*, *}; mod extrinsics { use super::*; fn rebag_works() { - ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { - // given - assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); - - // increase vote weight and implicitly rebag to the level of non-existent bag - set_staking_vote_weight(2_000); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])]); - - // decrease weight within the range of the current bag - set_staking_vote_weight(1_001); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); - // does not change bags - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])]); - - // reduce weight to the level of a non-existent bag - set_staking_vote_weight(1_001); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); - // creates the bag and moves the voter into it - assert_eq!(get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101])]); - - // increase weight to a pre-existing bag - set_staking_vote_weight(1_001); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); - // moves the voter to that bag - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42])]); - }); -} + ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); + + // increase vote weight and implicitly rebag to the level of non-existent bag + set_staking_vote_weight(2_000); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // decrease weight within the range of the current bag + set_staking_vote_weight(1_001); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); + // does not change bags + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // reduce weight to the level of a non-existent bag + set_staking_vote_weight(1_001); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); + // creates the bag and moves the voter into it + assert_eq!(get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101])]); + + // increase weight to a pre-existing bag + set_staking_vote_weight(1_001); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); + // moves the voter to that bag + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42])]); + }); + } -// Rebagging the tail of a bag results in the old bag having a new tail and an overall correct state. -#[test] -fn rebag_tail_works() { - ExtBuilder::default().build_and_execute(|| { - // when - set_staking_vote_weight(10); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); + // Rebagging the tail of a bag results in the old bag having a new tail and an overall correct state. + #[test] + fn rebag_tail_works() { + ExtBuilder::default().build_and_execute(|| { + // when + set_staking_vote_weight(10); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); - // then - assert_eq!(get_bags(), vec![(10, vec![31, 101]), (1000, vec![11, 21])]); - assert_eq!( - Bag::::get(1_000).unwrap(), - Bag::new(Some(11), Some(21), 1_000) - ); + // then + assert_eq!(get_bags(), vec![(10, vec![31, 101]), (1000, vec![11, 21])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(11), Some(21), 1_000)); - // when - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); + // when + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); - // then - assert_eq!(get_bags(), vec![(10, vec![31, 101, 21]), (1000, vec![11])]); - }); -} + // then + assert_eq!(get_bags(), vec![(10, vec![31, 101, 21]), (1000, vec![11])]); + }); + } -// Rebagging the head of a bag results in the old bag having a new head and an overall correct state. -#[test] -fn rebag_head_works() { - ExtBuilder::default().build_and_execute(|| { - // when - set_staking_vote_weight(10); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 11)); + // Rebagging the head of a bag results in the old bag having a new head and an overall correct state. + #[test] + fn rebag_head_works() { + ExtBuilder::default().build_and_execute(|| { + // when + set_staking_vote_weight(10); + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 11)); - // then - assert_eq!(get_bags(), vec![(10, vec![31, 11]), (1000, vec![21, 101])]); - assert_eq!( - Bag::::get(1_000).unwrap(), - Bag::new(Some(21), Some(101), 1_000) - ); + // then + assert_eq!(get_bags(), vec![(10, vec![31, 11]), (1000, vec![21, 101])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(21), Some(101), 1_000)); - // when - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); + // when + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); - // then - assert_eq!(get_bags(), vec![(10, vec![31, 11, 21]), (1000, vec![101])]); - assert_eq!( - Bag::::get(1_000).unwrap(), - Bag::new(Some(101), Some(101), 1_000) - ); + // then + assert_eq!(get_bags(), vec![(10, vec![31, 11, 21]), (1000, vec![101])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(101), Some(101), 1_000)); - // when - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); + // when + assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); - // then - assert_eq!(get_bags(), vec![(10, vec![31, 11, 21, 101])]); - assert_eq!( - Bag::::get(1_000), - None - ); - }); -} + // then + assert_eq!(get_bags(), vec![(10, vec![31, 11, 21, 101])]); + assert_eq!(Bag::::get(1_000), None); + }); + } } - mod bags_list_voter_list_provider { use super::*; @@ -134,12 +127,89 @@ mod bags_list_voter_list_provider { #[test] fn on_insert_works() { - todo!(); + // when + BagsListVoterListProvider::::on_insert(71, 1_000); + + // then + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71])]); + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + vec![11, 21, 101, 71, 31] + ); + assert_eq!(BagsListVoterListProvider::::count(), 5); + assert_ok!(BagsListVoterListProvider::::sanity_check()); + + // when + List::::insert(81, 1_001); + + // then + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + vec![81, 11, 21, 101, 71, 31] + ); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71]), (2_000, vec![81])] + ); + assert_eq!(BagsListVoterListProvider::::count(), 6); + assert_ok!(BagsListVoterListProvider::::sanity_check()); } #[test] fn on_update_works() { - todo!(); + ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1_000, vec![11, 21, 101])]); + + // update weight to the level of non-existent bag + BagsListVoterListProvider::::on_update(&42, 2_000); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (2_000, vec![42])] + ); + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + vec![42, 11, 21, 101, 31] + ); + assert_eq!(BagsListVoterListProvider::::count(), 5); + assert_ok!(BagsListVoterListProvider::::sanity_check()); + + // decrease weight within the range of the current bag + BagsListVoterListProvider::::on_update(&42, 1_001); + // does not change bags + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (2_000, vec![42])] + ); + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + vec![42, 11, 21, 101, 31] + ); + assert_eq!(BagsListVoterListProvider::::count(), 5); + assert_ok!(BagsListVoterListProvider::::sanity_check()); + + // reduce weight to the level of a non-existent bag + BagsListVoterListProvider::::on_update(&42, VoteWeight::MAX); + // creates the bag and moves the voter into it + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (VoteWeight::MAX, vec![42])]); + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + vec![42, 11, 21, 101, 31] + ); + assert_eq!(BagsListVoterListProvider::::count(), 5); + assert_ok!(BagsListVoterListProvider::::sanity_check()); + + // increase weight to a pre-existing bag + BagsListVoterListProvider::::on_update(&42, 999); + // moves the voter to that bag + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42])]); + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + vec![11, 21, 101, 42, 31] + ); + assert_eq!(BagsListVoterListProvider::::count(), 5); + assert_ok!(BagsListVoterListProvider::::sanity_check()); + }); } #[test] From 6d1d77ea6b701bd65fdec545ae5a5bd2c90fd15c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 00:42:23 -0700 Subject: [PATCH 103/241] test sanity check --- frame/bags-list/src/list/mod.rs | 5 +- frame/bags-list/src/mock.rs | 2 +- frame/bags-list/src/tests.rs | 112 ++++++++++++++++++++++++++++++-- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index d2dcf0a08bee0..c9e47e191a329 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -351,7 +351,6 @@ impl List { /// /// * Iterate all voters in list and make sure there are no duplicates. /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. - /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `List`. pub(crate) fn sanity_check() -> Result<(), &'static str> { @@ -365,7 +364,7 @@ impl List { let stored_count = crate::CounterForVoters::::get(); ensure!(iter_count == stored_count, "iter_count != voter_count"); - // let validators = staking::CounterForValidators::::get(); + // let validators = staking::CounterForValidators::::get(); TOOD can we just remove? // let nominators = staking::CounterForNominators::::get(); // ensure!(validators + nominators == stored_count, "validators + nominators != voters"); @@ -568,7 +567,7 @@ impl Bag { .map(|node| node.id) // each voter is only seen once, thus there is no cycle within a bag .all(|voter| seen_in_bag.insert(voter)), - "Duplicate found in bag" + "duplicate found in bag" ); Ok(()) diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index dc5dc6cd29368..4d13ca4b25c02 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -121,7 +121,7 @@ pub(crate) mod test_utils { pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { BagThresholds::get() .into_iter() - .chain(std::iter::once(&VoteWeight::MAX)) + .chain(std::iter::once(&VoteWeight::MAX)) // assumes this is not an explicit threshold .filter_map(|t| { Bag::::get(*t) .map(|bag| (*t, bag.iter().map(|n| n.id().clone()).collect::>())) diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 42e8f911e962f..3720cc813e83f 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -157,9 +157,12 @@ mod bags_list_voter_list_provider { #[test] fn on_update_works() { - ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { + ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1_000, vec![11, 21, 101])]); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (20, vec![42]), (1_000, vec![11, 21, 101])] + ); // update weight to the level of non-existent bag BagsListVoterListProvider::::on_update(&42, 2_000); @@ -191,7 +194,10 @@ mod bags_list_voter_list_provider { // reduce weight to the level of a non-existent bag BagsListVoterListProvider::::on_update(&42, VoteWeight::MAX); // creates the bag and moves the voter into it - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (VoteWeight::MAX, vec![42])]); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (VoteWeight::MAX, vec![42])] + ); assert_eq!( BagsListVoterListProvider::::get_voters().collect::>(), vec![42, 11, 21, 101, 31] @@ -214,11 +220,107 @@ mod bags_list_voter_list_provider { #[test] fn on_remove_works() { - todo!(); + let check_storage = |id, counter, accounts, bags| { + assert!(!VoterBagFor::::contains_key(id)); + assert!(!VoterNodes::::contains_key(id)); + assert_eq!(BagsListVoterListProvider::::count(), counter); + assert_eq!(CounterForVoters::::get(), counter); + assert_eq!(VoterBagFor::::iter().count() as u32, counter); + assert_eq!(VoterNodes::::iter().count() as u32, counter); + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + accounts + ); + assert_eq!(get_bags(), bags); + }; + + ExtBuilder::default().build_and_execute(|| { + // when removing a non-existent voter + assert!(!BagsListVoterListProvider::::get_voters() + .collect::>() + .contains(&42)); + assert!(!VoterNodes::::contains_key(42)); + BagsListVoterListProvider::::on_remove(&42); + + // then nothing changes + assert_eq!( + BagsListVoterListProvider::::get_voters().collect::>(), + vec![11, 21, 101, 31] + ); + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101])]); + + // when removing a node from a bag with multiple nodes + BagsListVoterListProvider::::on_remove(&11); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); + check_storage( + 11, + 3, + vec![21, 101, 31], // list + vec![(10, vec![31]), (1_000, vec![21, 101])], // bags + ); + + // when removing a node from a bag with only one node: + BagsListVoterListProvider::::on_remove(&31); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101]); + check_storage( + 31, + 2, + vec![21, 101], // list + vec![(1_000, vec![21, 101])], // bags + ); + assert!(!VoterBags::::contains_key(10)); // bag 10 is removed + + // remove remaining voters to make sure storage cleans up as expected + BagsListVoterListProvider::::on_remove(&21); + check_storage( + 21, + 1, + vec![101], // list + vec![(1_000, vec![101])], // bags + ); + + BagsListVoterListProvider::::on_remove(&101); + check_storage( + 101, + 0, + Vec::::new(), // list + vec![], // bags + ); + assert!(!VoterBags::::contains_key(1_000)); // bag 1_000 is removed + + // bags are deleted via removals + assert_eq!(VoterBags::::iter().count(), 0); + }); } #[test] fn sanity_check_works() { - todo!(); + ExtBuilder::default().build_and_execute(|| { + assert_ok!(BagsListVoterListProvider::::sanity_check()); + }); + + // make sure there are no duplicates. + ExtBuilder::default().build_and_execute(|| { + BagsListVoterListProvider::::on_insert(11, 10); + assert_eq!( + BagsListVoterListProvider::::sanity_check(), + Err("duplicate identified") + ); + + }); + + // ensure count is in sync with `CounterForVoters`. + ExtBuilder::default().build_and_execute(|| { + crate::CounterForVoters::::mutate(|counter| *counter += 1); + assert_eq!(crate::CounterForVoters::::get(), 5); + assert_eq!( + BagsListVoterListProvider::::sanity_check(), + Err("iter_count != voter_count") + ); + }); } } From 333052ab5c96e7ed195abd9c8721c8c62e28591f Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 4 Aug 2021 13:05:16 +0200 Subject: [PATCH 104/241] a round of test fixes --- Cargo.lock | 2 +- frame/bags-list/Cargo.toml | 12 +- frame/bags-list/src/lib.rs | 26 +- frame/bags-list/src/list/mod.rs | 51 ++-- frame/bags-list/src/mock.rs | 35 ++- frame/bags-list/src/tests.rs | 313 +++++++++------------ frame/election-provider-support/src/lib.rs | 34 ++- frame/staking/src/lib.rs | 4 +- frame/staking/src/mock.rs | 10 +- frame/staking/src/pallet/impls.rs | 47 ++-- frame/staking/src/pallet/mod.rs | 27 +- 11 files changed, 259 insertions(+), 302 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 690be865739ac..9d99723dce57d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4887,8 +4887,8 @@ dependencies = [ "frame-system", "log", "pallet-balances", - "pallet-staking", "parity-scale-codec", + "sp-arithmetic", "sp-core", "sp-io", "sp-runtime", diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index e637bb6c295e4..afff9980aa531 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -18,13 +18,12 @@ codec = { package = "parity-scale-codec", version = "2.0.0", default-features = sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../primitives/arithmetic" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } - log = { version = "0.4.14", default-features = false } # Optional imports for benchmarking @@ -39,13 +38,12 @@ pallet-balances = { version = "4.0.0-dev", path = "../balances" } default = ["std"] std = [ "codec/std", + "sp-runtime/std", + "sp-std/std", "frame-support/std", "frame-system/std", - "frame-election-provider-support/std", - "sp-runtime/std", - "sp-std/std", - "pallet-staking/std", - "log/std", + "frame-election-provider-support/std", + "log/std", ] runtime-benchmarks = [ "frame-benchmarking", diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 373cd82612cd6..96c8fd20945c7 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -21,7 +21,7 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -use frame_election_provider_support::VoteWeight; +use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; use frame_system::ensure_signed; mod list; @@ -36,7 +36,7 @@ pub use weights::WeightInfo; use list::List; -pub(crate) const LOG_TARGET: &'static str = "runtime::voter_bags"; +pub(crate) const LOG_TARGET: &'static str = "runtime::bags_list"; // syntactic sugar for logging. #[macro_export] @@ -52,7 +52,6 @@ macro_rules! log { #[frame_support::pallet] pub mod pallet { use super::*; - use frame_election_provider_support::StakingVoteWeight; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; @@ -69,7 +68,7 @@ pub mod pallet { // type WeightInfo: WeightInfo; // Something that implements `trait StakingVoteWeight`. - type StakingVoteWeight: StakingVoteWeight; + type VoteWeightProvider: VoteWeightProvider; /// The list of thresholds separating the various voter bags. /// @@ -121,7 +120,6 @@ pub mod pallet { #[pallet::constant] type BagThresholds: Get<&'static [VoteWeight]>; - // TODO VoteWeight type could be made generic? // TODO Node.id type could be made generic? } @@ -143,7 +141,7 @@ pub mod pallet { /// This may not be the appropriate bag for the voter's weight if they have been rewarded or /// slashed. #[pallet::storage] - pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, T::AccountId, VoteWeight>; + pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, T::AccountId, VoteWeight>; // TODO: retire this storage item. /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. @@ -171,7 +169,7 @@ pub mod pallet { #[pallet::weight(123456789)] // TODO pub fn rebag(origin: OriginFor, account: T::AccountId) -> DispatchResult { ensure_signed(origin)?; - let weight = T::StakingVoteWeight::staking_vote_weight(&account); + let weight = T::VoteWeightProvider::vote_weight(&account); Pallet::::do_rebag(&account, weight); Ok(()) } @@ -211,12 +209,8 @@ impl Pallet { } } -pub struct BagsListVoterListProvider(sp_std::marker::PhantomData); -impl frame_election_provider_support::VoterListProvider - for BagsListVoterListProvider -{ - /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters() -> Box> { +impl SortedListProvider for Pallet { + fn iter() -> Box> { Box::new(List::::iter().map(|n| n.id().clone())) } @@ -228,17 +222,11 @@ impl frame_election_provider_support::VoterListProvider List::::insert(voter, weight); } - /// Hook for updating a voter in the list (unused). fn on_update(voter: &T::AccountId, new_weight: VoteWeight) { Pallet::::do_rebag(voter, new_weight); } - /// Hook for removing a voter from the list. fn on_remove(voter: &T::AccountId) { List::::remove(voter) } - - fn sanity_check() -> Result<(), &'static str> { - List::::sanity_check() - } } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index c9e47e191a329..9f241f6d05110 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -15,11 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! List implementation. +//! Basic implementation of a doubly-linked list use crate::Config; use codec::{Decode, Encode}; -use frame_election_provider_support::{StakingVoteWeight, VoteWeight}; +use frame_election_provider_support::{VoteWeight, VoteWeightProvider}; use frame_support::{ensure, traits::Get, DefaultNoBound}; use sp_runtime::SaturatedConversion; use sp_std::{ @@ -70,13 +70,12 @@ pub struct List(PhantomData); impl List { /// Remove all data associated with the voter list from storage. - // #[cfg(test)] // TODO this is used for regenerate - // pub fn clear() { - // crate::CounterForVoters::::kill(); - // crate::VoterBagFor::::remove_all(None); - // crate::VoterBags::::remove_all(None); - // crate::VoterNodes::::remove_all(None); - // } + pub fn clear() { + crate::CounterForVoters::::kill(); + crate::VoterBagFor::::remove_all(None); + crate::VoterBags::::remove_all(None); + crate::VoterNodes::::remove_all(None); + } /// Migrate the voter list from one set of thresholds to another. /// @@ -117,7 +116,7 @@ impl List { if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's // no point iterating through bag 10 twice. - continue + continue; } if let Some(bag) = Bag::::get(affected_bag) { @@ -128,7 +127,7 @@ impl List { // a removed bag means that all members of that bag must be rebagged for removed_bag in old_set.difference(&new_set).copied() { if !affected_old_bags.insert(removed_bag) { - continue + continue; } if let Some(bag) = Bag::::get(removed_bag) { @@ -137,7 +136,7 @@ impl List { } // migrate the - let weight_of = T::StakingVoteWeight::staking_vote_weight; + let weight_of = T::VoteWeightProvider::vote_weight; Self::remove_many(affected_accounts.iter().map(|voter| voter)); let num_affected = affected_accounts.len() as u32; Self::insert_many(affected_accounts.into_iter(), weight_of); @@ -173,18 +172,13 @@ impl List { /// consensus. /// /// Returns the number of voters migrated. - // pub fn regenerate(weight_of: dyn FnOnce(&T::AccountId) -> VoteWeight) -> u32 { - // Self::clear(); - - // let nominators_iter = staking::Nominators::::iter().map(|(id, _)| Voter::nominator(id)); - // let validators_iter = staking::Validators::::iter().map(|(id, _)| Voter::validator(id)); - - // nominators_iter.chain(validators_iter).for_each(|v| { - // let weight = weight_of(v); - // Self::insert(v, weight_of); - // }); - // // TODO: use the new insert_at. it should work - // } + pub fn regenerate( + all: impl IntoIterator, + weight_of: Box VoteWeight>, + ) { + Self::clear(); + Self::insert_many(all, weight_of); + } /// Decode the length of the voter list. pub fn decode_len() -> Option { @@ -234,11 +228,10 @@ impl List { } /// Insert a new voter into the appropriate bag in the voter list. - // WARNING: This does not check if the inserted AccountId is duplicate with - // others in any bag. + // WARNING: This does not check if the inserted AccountId is duplicate with others in any bag. pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { - // TODO: can check if this voter exists as a node by checking if `voter` exists - // in the nodes map and return early if it does. + // TODO: can check if this voter exists as a node by checking if `voter` exists in the nodes + // map and return early if it does. let bag_weight = notional_bag_for::(weight); crate::log!( @@ -480,7 +473,7 @@ impl Bag { // this should never happen, but this check prevents a worst case infinite loop debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return + return; }; } diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 4d13ca4b25c02..0c7869e38b8c0 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -1,24 +1,18 @@ use super::*; use frame_election_provider_support::VoteWeight; use frame_support::parameter_types; -use std::cell::RefCell; -type AccountId = u32; -type Balance = u32; +pub type AccountId = u32; +pub type Balance = u32; -thread_local! { - static VOTE_WEIGHT: RefCell = RefCell::new(Default::default()) -} - -/// Set a mock return value for `StakingVoteWeight::staking_vote_weight`. -pub(crate) fn set_staking_vote_weight(weight: VoteWeight) { - VOTE_WEIGHT.with(|w| *w.borrow_mut() = weight); +parameter_types! { + pub static NextVoteWeight: VoteWeight = 0; } pub struct StakingMock; -impl frame_election_provider_support::StakingVoteWeight for StakingMock { - fn staking_vote_weight(_who: &AccountId) -> VoteWeight { - VOTE_WEIGHT.with(|h| h.borrow().clone()) +impl frame_election_provider_support::VoteWeightProvider for StakingMock { + fn vote_weight(who: &AccountId) -> VoteWeight { + NextVoteWeight::get() } } @@ -58,7 +52,7 @@ parameter_types! { impl crate::Config for Runtime { type Event = Event; type BagThresholds = BagThresholds; - type StakingVoteWeight = StakingMock; + type VoteWeightProvider = StakingMock; } type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; @@ -70,16 +64,18 @@ frame_support::construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic, { System: frame_system::{Pallet, Call, Storage, Event, Config}, - PalletBagsList: crate::{Pallet, Call, Storage, Event}, + BagsList: crate::{Pallet, Call, Storage, Event}, } ); pub(crate) mod ext_builder { + use frame_support::RuntimeDebugNoBound; + use super::*; /// Default AccountIds and their weights. - const GENESIS_IDS: [(AccountId, VoteWeight); 4] = - [(31, 10), (11, 1_000), (21, 1_000), (101, 1_000)]; + const GENESIS_IDS: [(AccountId, VoteWeight); 4] = [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; + #[derive(Default)] pub(crate) struct ExtBuilder { ids: Vec<(AccountId, VoteWeight)>, @@ -108,7 +104,10 @@ pub(crate) mod ext_builder { } pub(crate) fn build_and_execute(self, test: impl FnOnce() -> ()) { - self.build().execute_with(test); + self.build().execute_with(|| { + test(); + List::::sanity_check().expect("Sanity check post condition failed") + }) } } } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 3720cc813e83f..66121eb75e129 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,106 +1,117 @@ use frame_support::assert_ok; use super::*; -use frame_election_provider_support::VoterListProvider; +use frame_election_provider_support::SortedListProvider; use list::Bag; use mock::{ext_builder::*, test_utils::*, *}; mod extrinsics { use super::*; + #[test] fn rebag_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1000, vec![2, 3, 4])]); // increase vote weight and implicitly rebag to the level of non-existent bag - set_staking_vote_weight(2_000); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); + NextVoteWeight::set(2000); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); // decrease weight within the range of the current bag - set_staking_vote_weight(1_001); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); + NextVoteWeight::set(1001); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // does not change bags - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); // reduce weight to the level of a non-existent bag - set_staking_vote_weight(1_001); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); + NextVoteWeight::set(30); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // creates the bag and moves the voter into it - assert_eq!(get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (30, vec![42]), (1000, vec![2, 3, 4])]); // increase weight to a pre-existing bag - set_staking_vote_weight(1_001); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 42)); + NextVoteWeight::set(500); + assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // moves the voter to that bag - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 42])]); }); } - // Rebagging the tail of a bag results in the old bag having a new tail and an overall correct state. + // Rebagging the tail of a bag results in the old bag having a new tail and an overall correct + // state. #[test] fn rebag_tail_works() { ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + // when - set_staking_vote_weight(10); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); + NextVoteWeight::set(10); + assert_ok!(BagsList::rebag(Origin::signed(0), 4)); // then - assert_eq!(get_bags(), vec![(10, vec![31, 101]), (1000, vec![11, 21])]); - assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(11), Some(21), 1_000)); + assert_eq!(get_bags(), vec![(10, vec![1, 4]), (1000, vec![2, 3])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(3), 1_000)); + + // when + assert_ok!(BagsList::rebag(Origin::signed(0), 3)); + + // then + assert_eq!(get_bags(), vec![(10, vec![1, 4, 3]), (1000, vec![2])]); + + assert_eq!(Bag::::get(10).unwrap(), Bag::new(Some(1), Some(3), 10)); + // TODO: This might be wrong, should be None. + assert_eq!(Bag::::get(1000).unwrap(), Bag::new(Some(2), Some(2), 1000)); + assert_eq!(get_voter_list_as_ids(), vec![2u32, 1, 4, 3]); // when - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); + assert_ok!(BagsList::rebag(Origin::signed(0), 2)); // then - assert_eq!(get_bags(), vec![(10, vec![31, 101, 21]), (1000, vec![11])]); + assert_eq!(get_bags(), vec![(10, vec![1, 4, 3, 2])]); }); } - // Rebagging the head of a bag results in the old bag having a new head and an overall correct state. + // Rebagging the head of a bag results in the old bag having a new head and an overall correct + // state. #[test] fn rebag_head_works() { ExtBuilder::default().build_and_execute(|| { // when - set_staking_vote_weight(10); - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 11)); + NextVoteWeight::set(10); + assert_ok!(BagsList::rebag(Origin::signed(0), 2)); // then - assert_eq!(get_bags(), vec![(10, vec![31, 11]), (1000, vec![21, 101])]); - assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(21), Some(101), 1_000)); + assert_eq!(get_bags(), vec![(10, vec![1, 2]), (1000, vec![3, 4])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(3), Some(4), 1_000)); // when - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 21)); + assert_ok!(BagsList::rebag(Origin::signed(0), 3)); // then - assert_eq!(get_bags(), vec![(10, vec![31, 11, 21]), (1000, vec![101])]); - assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(101), Some(101), 1_000)); + assert_eq!(get_bags(), vec![(10, vec![1, 2, 3]), (1000, vec![4])]); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(4), Some(4), 1_000)); // when - assert_ok!(PalletBagsList::rebag(Origin::signed(0), 101)); + assert_ok!(BagsList::rebag(Origin::signed(0), 4)); // then - assert_eq!(get_bags(), vec![(10, vec![31, 11, 21, 101])]); + assert_eq!(get_bags(), vec![(10, vec![1, 2, 3, 4])]); assert_eq!(Bag::::get(1_000), None); }); } } -mod bags_list_voter_list_provider { +mod sorted_list_provider { use super::*; #[test] - fn get_voters_works() { + fn iter_works() { ExtBuilder::default().build_and_execute(|| { - let expected = vec![11, 21, 101, 31]; - for (i, id) in BagsListVoterListProvider::::get_voters().enumerate() { + let expected = vec![2, 3, 4, 1]; + for (i, id) in >::iter().enumerate() { assert_eq!(id, expected[i]) } }); @@ -109,218 +120,170 @@ mod bags_list_voter_list_provider { #[test] fn count_works() { ExtBuilder::default().build_and_execute(|| { - assert_eq!(BagsListVoterListProvider::::count(), 4); + assert_eq!(>::count(), 4); - BagsListVoterListProvider::::on_insert(201, 0); - assert_eq!(BagsListVoterListProvider::::count(), 5); + >::on_insert(201, 0); + assert_eq!(>::count(), 5); - BagsListVoterListProvider::::on_remove(&201); - assert_eq!(BagsListVoterListProvider::::count(), 4); + >::on_remove(&201); + assert_eq!(>::count(), 4); - BagsListVoterListProvider::::on_remove(&31); - assert_eq!(BagsListVoterListProvider::::count(), 3); + >::on_remove(&1); + assert_eq!(>::count(), 3); - BagsListVoterListProvider::::on_remove(&21); - assert_eq!(BagsListVoterListProvider::::count(), 3); + >::on_remove(&2); + assert_eq!(>::count(), 2); }); } #[test] fn on_insert_works() { - // when - BagsListVoterListProvider::::on_insert(71, 1_000); - - // then - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71])]); - assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - vec![11, 21, 101, 71, 31] - ); - assert_eq!(BagsListVoterListProvider::::count(), 5); - assert_ok!(BagsListVoterListProvider::::sanity_check()); - - // when - List::::insert(81, 1_001); - - // then - assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - vec![81, 11, 21, 101, 71, 31] - ); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71]), (2_000, vec![81])] - ); - assert_eq!(BagsListVoterListProvider::::count(), 6); - assert_ok!(BagsListVoterListProvider::::sanity_check()); + ExtBuilder::default().build_and_execute(|| { + // when + >::on_insert(71, 1_000); + + // then + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 71])]); + assert_eq!( + >::iter().collect::>(), + vec![2, 3, 4, 71, 1] + ); + assert_eq!(>::count(), 5); + assert_ok!(List::::sanity_check()); + + // when + List::::insert(81, 1_001); + + // then + assert_eq!( + get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 71]), (2000, vec![81])] + ); + assert_eq!( + >::iter().collect::>(), + vec![81, 2, 3, 4, 71, 1] + ); + assert_eq!(>::count(), 6); + }) } #[test] fn on_update_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given - assert_eq!( - get_bags(), - vec![(10, vec![31]), (20, vec![42]), (1_000, vec![11, 21, 101])] - ); + assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])]); // update weight to the level of non-existent bag - BagsListVoterListProvider::::on_update(&42, 2_000); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (2_000, vec![42])] - ); + >::on_update(&42, 2_000); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - vec![42, 11, 21, 101, 31] + >::iter().collect::>(), + vec![42, 2, 3, 4, 1] ); - assert_eq!(BagsListVoterListProvider::::count(), 5); - assert_ok!(BagsListVoterListProvider::::sanity_check()); + assert_eq!(>::count(), 5); + assert_ok!(List::::sanity_check()); // decrease weight within the range of the current bag - BagsListVoterListProvider::::on_update(&42, 1_001); + >::on_update(&42, 1_001); // does not change bags + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); assert_eq!( - get_bags(), - vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (2_000, vec![42])] - ); - assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - vec![42, 11, 21, 101, 31] + >::iter().collect::>(), + vec![42, 2, 3, 4, 1] ); - assert_eq!(BagsListVoterListProvider::::count(), 5); - assert_ok!(BagsListVoterListProvider::::sanity_check()); + assert_eq!(>::count(), 5); + assert_ok!(List::::sanity_check()); - // reduce weight to the level of a non-existent bag - BagsListVoterListProvider::::on_update(&42, VoteWeight::MAX); + // increase weight to the level of a non-existent bag + >::on_update(&42, VoteWeight::MAX); // creates the bag and moves the voter into it assert_eq!( get_bags(), - vec![(10, vec![31]), (1_000, vec![11, 21, 101]), (VoteWeight::MAX, vec![42])] + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (VoteWeight::MAX, vec![42])] ); assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - vec![42, 11, 21, 101, 31] + >::iter().collect::>(), + vec![42, 2, 3, 4, 1] ); - assert_eq!(BagsListVoterListProvider::::count(), 5); - assert_ok!(BagsListVoterListProvider::::sanity_check()); + assert_eq!(>::count(), 5); + assert_ok!(List::::sanity_check()); - // increase weight to a pre-existing bag - BagsListVoterListProvider::::on_update(&42, 999); - // moves the voter to that bag - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42])]); + // decrease the weight to a pre-existing bag + >::on_update(&42, 999); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - vec![11, 21, 101, 42, 31] + >::iter().collect::>(), + vec![2, 3, 4, 42, 1] ); - assert_eq!(BagsListVoterListProvider::::count(), 5); - assert_ok!(BagsListVoterListProvider::::sanity_check()); + assert_eq!(>::count(), 5); }); } #[test] fn on_remove_works() { - let check_storage = |id, counter, accounts, bags| { + let ensure_left = |id, counter| { assert!(!VoterBagFor::::contains_key(id)); assert!(!VoterNodes::::contains_key(id)); - assert_eq!(BagsListVoterListProvider::::count(), counter); + assert_eq!(>::count(), counter); assert_eq!(CounterForVoters::::get(), counter); assert_eq!(VoterBagFor::::iter().count() as u32, counter); assert_eq!(VoterNodes::::iter().count() as u32, counter); - assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - accounts - ); - assert_eq!(get_bags(), bags); }; ExtBuilder::default().build_and_execute(|| { // when removing a non-existent voter - assert!(!BagsListVoterListProvider::::get_voters() - .collect::>() - .contains(&42)); + assert!(!get_voter_list_as_ids().contains(&42)); assert!(!VoterNodes::::contains_key(42)); - BagsListVoterListProvider::::on_remove(&42); + >::on_remove(&42); // then nothing changes - assert_eq!( - BagsListVoterListProvider::::get_voters().collect::>(), - vec![11, 21, 101, 31] - ); - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(get_voter_list_as_ids(), vec![2, 3, 4, 1]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // when removing a node from a bag with multiple nodes - BagsListVoterListProvider::::on_remove(&11); + >::on_remove(&2); // then - assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); - check_storage( - 11, - 3, - vec![21, 101, 31], // list - vec![(10, vec![31]), (1_000, vec![21, 101])], // bags - ); + assert_eq!(get_voter_list_as_ids(), vec![3, 4, 1]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + ensure_left(2, 3); // when removing a node from a bag with only one node: - BagsListVoterListProvider::::on_remove(&31); + >::on_remove(&1); // then - assert_eq!(get_voter_list_as_ids(), vec![21, 101]); - check_storage( - 31, - 2, - vec![21, 101], // list - vec![(1_000, vec![21, 101])], // bags - ); - assert!(!VoterBags::::contains_key(10)); // bag 10 is removed + assert_eq!(get_voter_list_as_ids(), vec![3, 4]); + assert_eq!(get_bags(), vec![(1_000, vec![3, 4])]); + ensure_left(1, 2); // remove remaining voters to make sure storage cleans up as expected - BagsListVoterListProvider::::on_remove(&21); - check_storage( - 21, - 1, - vec![101], // list - vec![(1_000, vec![101])], // bags - ); + >::on_remove(&4); + assert_eq!(get_voter_list_as_ids(), vec![3]); + ensure_left(4, 1); - BagsListVoterListProvider::::on_remove(&101); - check_storage( - 101, - 0, - Vec::::new(), // list - vec![], // bags - ); - assert!(!VoterBags::::contains_key(1_000)); // bag 1_000 is removed - - // bags are deleted via removals - assert_eq!(VoterBags::::iter().count(), 0); + >::on_remove(&3); + assert_eq!(get_voter_list_as_ids(), Vec::::new()); + ensure_left(3, 0); }); } #[test] fn sanity_check_works() { ExtBuilder::default().build_and_execute(|| { - assert_ok!(BagsListVoterListProvider::::sanity_check()); + assert_ok!(List::::sanity_check()); }); // make sure there are no duplicates. ExtBuilder::default().build_and_execute(|| { - BagsListVoterListProvider::::on_insert(11, 10); - assert_eq!( - BagsListVoterListProvider::::sanity_check(), - Err("duplicate identified") - ); - + >::on_insert(1, 10); + assert_eq!(List::::sanity_check(), Err("duplicate identified")); }); // ensure count is in sync with `CounterForVoters`. ExtBuilder::default().build_and_execute(|| { crate::CounterForVoters::::mutate(|counter| *counter += 1); assert_eq!(crate::CounterForVoters::::get(), 5); - assert_eq!( - BagsListVoterListProvider::::sanity_check(), - Err("iter_count != voter_count") - ); + assert_eq!(List::::sanity_check(), Err("iter_count != voter_count")); }); } } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index c01170e0c5625..9f77b88a1b718 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -293,25 +293,37 @@ impl ElectionProvider for () { } } -// TODO not sure where to put these, but put them here just because this is where -// export voteweight from into the staking pallet -/// Trait to be implemented by a voter list provider. -pub trait VoterListProvider { +/// A utility trait for something to implement `ElectionDataProvider` in a sensible way. +/// +/// This is generic over the `AccountId`'d sole, and it can represent a validator, a nominator, or +/// any other entity. +/// +/// On the contrary, to simplify the trait, the `VoteWeight` is hardcoded as the weight of each +/// entity. The weights are scending, the higher, the better. In the long term, if this trait ends +/// up having use cases outside of the election context, it is easy enough to make it generic over +/// the `VoteWeight`. +/// +/// Something that implements this trait will do a best-effort sort over voters, and thus can be +/// used on the implementing side of `ElectionDataProvider`. +pub trait SortedListProvider { /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters() -> Box>; + fn iter() -> Box>; /// get the current count of voters. fn count() -> u32; - // Hook for inserting a validator. - fn on_insert(voter: AccountId, weight: VoteWeight); // TODO - /// Hook for updating the list when a voter is added, their voter type is changed, - /// or their weight changes. + // Hook for inserting a voter. + fn on_insert(voter: AccountId, weight: VoteWeight); + /// Hook for updating a single voter. fn on_update(voter: &AccountId, weight: VoteWeight); /// Hook for removing a voter from the list. fn on_remove(voter: &AccountId); /// Sanity check internal state of list. Only meant for debug compilation. + #[cfg(test)] fn sanity_check() -> Result<(), &'static str>; } -pub trait StakingVoteWeight { - fn staking_vote_weight(who: &AccountId) -> VoteWeight; +/// Something that can provide the `VoteWeight` of an account. Similar to [`ElectionProvider`] and +/// [`ElectionDataProvider`], this should typically be implementing by whoever is supposed to *use* +/// `SortedListProvider`. +pub trait VoteWeightProvider { + fn vote_weight(who: &AccountId) -> VoteWeight; } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 79fee78b79716..6e8c36b26ddb5 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -494,7 +494,7 @@ impl } if unlocking_balance >= value { - break + break; } } @@ -799,7 +799,7 @@ where /// A simple voter list implementation that does not require any additional pallets. pub struct StakingVoterListStub(sp_std::marker::PhantomData); -impl frame_election_provider_support::VoterListProvider +impl frame_election_provider_support::SortedListProvider for StakingVoterListStub { /// Returns iterator over voter list, which can have `take` called on it. diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 1452a9ff18335..0cebfdd07af91 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -284,8 +284,8 @@ impl crate::pallet::pallet::Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); type VoterBagThresholds = VoterBagThresholds; - // type VoterListProvider = pallet_voter_bags::VoterBagsVoterListProvider; - type VoterListProvider = staking::VoterList; + // type SortedListProvider = pallet_voter_bags::VoterBagsVoterListProvider; + type SortedListProvider = staking::VoterList; } impl frame_system::offchain::SendTransactionTypes for Test @@ -576,7 +576,7 @@ fn check_nominators() { e.others.iter().filter(|e| e.who == nominator).collect::>(); let len = individual.len(); match len { - 0 => { /* not supporting this validator at all. */ }, + 0 => { /* not supporting this validator at all. */ } 1 => sum += individual[0].value, _ => panic!("nominator cannot back a validator more than once."), }; @@ -760,9 +760,9 @@ pub(crate) fn on_offence_in_era( for &(bonded_era, start_session) in bonded_eras.iter() { if bonded_era == era { let _ = Staking::on_offence(offenders, slash_fraction, start_session); - return + return; } else if bonded_era > era { - break + break; } } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 0749aba8bcd74..283d07b015aba 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -18,7 +18,7 @@ //! Implementations for the Staking FRAME Pallet. use frame_election_provider_support::{ - data_provider, ElectionProvider, Supports, VoteWeight, VoterListProvider, + data_provider, ElectionProvider, SortedListProvider, Supports, VoteWeight, }; use frame_support::{ pallet_prelude::*, @@ -136,7 +136,7 @@ impl Pallet { // Nothing to do if they have no reward points. if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()); } // This is the fraction of the total reward that the validator and the @@ -227,8 +227,9 @@ impl Pallet { Self::update_ledger(&controller, &l); r }), - RewardDestination::Account(dest_account) => - Some(T::Currency::deposit_creating(&dest_account, amount)), + RewardDestination::Account(dest_account) => { + Some(T::Currency::deposit_creating(&dest_account, amount)) + } RewardDestination::None => None, } } @@ -256,14 +257,14 @@ impl Pallet { _ => { // Either `Forcing::ForceNone`, // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None - }, + return None; + } } // New era. let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() && - matches!(ForceEra::::get(), Forcing::ForceNew) + if maybe_new_era_validators.is_some() + && matches!(ForceEra::::get(), Forcing::ForceNew) { ForceEra::::put(Forcing::NotForcing); } @@ -444,12 +445,12 @@ impl Pallet { // TODO: this should be simplified #8911 CurrentEra::::put(0); ErasStartSessionIndex::::insert(&0, &start_session_index); - }, + } _ => (), } Self::deposit_event(Event::StakingElectionFailed); - return None + return None; } Self::deposit_event(Event::StakersElected); @@ -657,7 +658,7 @@ impl Pallet { // let slashing_spans = >::iter().collect::>(); let unfiltered_voters: Vec = - T::VoterListProvider::get_voters().take(wanted_voters).collect(); + T::SortedListProvider::get_voters().take(wanted_voters).collect(); // TODO: filter slashing spans // concatenate account ids to voter data. unimplemented!() @@ -682,8 +683,8 @@ impl Pallet { CounterForNominators::::mutate(|x| x.saturating_inc()) } Nominators::::insert(who, nominations); - T::VoterListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); - debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); + T::SortedListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } /// This function will remove a nominator from the `Nominators` storage map, @@ -698,8 +699,8 @@ impl Pallet { if Nominators::::contains_key(who) { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); - T::VoterListProvider::on_remove(who); - debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); + T::SortedListProvider::on_remove(who); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); true } else { false @@ -719,8 +720,8 @@ impl Pallet { CounterForValidators::::mutate(|x| x.saturating_inc()) } Validators::::insert(who, prefs); - T::VoterListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); - debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); + T::SortedListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } /// This function will remove a validator from the `Validators` storage map, @@ -735,8 +736,8 @@ impl Pallet { if Validators::::contains_key(who) { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); - T::VoterListProvider::on_remove(who); - debug_assert_eq!(T::VoterListProvider::sanity_check(), Ok(())); + T::SortedListProvider::on_remove(who); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); true } else { false @@ -765,7 +766,7 @@ impl debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); // debug_assert_eq!( // voter_count, - // T::VoterListProvider::get_voters().count(), + // T::SortedListProvider::get_voters().count(), // "voter_count must be accurate", // ); @@ -783,7 +784,7 @@ impl let target_count = CounterForValidators::::get() as usize; if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big") + return Err("Target snapshot too big"); } let weight = ::DbWeight::get().reads(target_count as u64); @@ -1047,7 +1048,7 @@ where add_db_reads_writes(1, 0); if active_era.is_none() { // This offence need not be re-submitted. - return consumed_weight + return consumed_weight; } active_era.expect("value checked not to be `None`; qed").index }; @@ -1093,7 +1094,7 @@ where // Skip if the validator is invulnerable. if invulnerables.contains(stash) { - continue + continue; } let unapplied = slashing::compute_slash::(slashing::SlashParams { diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 0836fc98f0f10..e5203f0064723 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -17,7 +17,7 @@ //! Staking FRAME Pallet. -use frame_election_provider_support::{VoteWeight, VoterListProvider}; +use frame_election_provider_support::{SortedListProvider, VoteWeight}; use frame_support::{ pallet_prelude::*, traits::{ @@ -196,7 +196,7 @@ pub mod pallet { #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; - type VoterListProvider: VoterListProvider; + type SortedListProvider: SortedListProvider; } #[pallet::extra_constants] @@ -568,7 +568,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue + continue; } match status { StakerStatus::Validator => { @@ -580,7 +580,7 @@ pub mod pallet { } else { num_voters += 1; } - }, + } StakerStatus::Nominator(votes) => { if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), @@ -590,14 +590,14 @@ pub mod pallet { } else { num_voters += 1; } - }, + } _ => (), }; } // all voters are inserted sanely. assert_eq!( - T::VoterListProvider::count(), + T::SortedListProvider::count(), num_voters, "not all genesis stakers were inserted into bags, something is wrong." ); @@ -839,7 +839,7 @@ pub mod pallet { Error::::InsufficientBond ); - T::VoterListProvider::on_update(&stash, Self::weight_of_fn()(&ledger.stash)); + T::SortedListProvider::on_update(&stash, Self::weight_of_fn()(&ledger.stash)); Self::deposit_event(Event::::Bonded(stash.clone(), extra)); Self::update_ledger(&controller, &ledger); } @@ -901,7 +901,10 @@ pub mod pallet { let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); Self::update_ledger(&controller, &ledger); - T::VoterListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); + T::SortedListProvider::on_update( + &ledger.stash, + Self::weight_of_fn()(&ledger.stash), + ); Self::deposit_event(Event::::Unbonded(ledger.stash, value)); } Ok(()) @@ -1394,11 +1397,11 @@ pub mod pallet { Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); Self::update_ledger(&controller, &ledger); - T::VoterListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. + T::SortedListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. Ok(Some( - 35 * WEIGHT_PER_MICROS + - 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + - T::DbWeight::get().reads_writes(3, 2), + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), ) .into()) } From 894da3f3bb6aab5ae115c1e89727424e5386a958 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 4 Aug 2021 19:25:07 +0200 Subject: [PATCH 105/241] push a lot of changes --- frame/bags-list/src/benchmarks.rs | 108 ++++++++ frame/bags-list/src/lib.rs | 3 + frame/bags-list/src/list/mod.rs | 57 ++-- frame/bags-list/src/list/tests.rs | 307 ++++++++++----------- frame/bags-list/src/mock.rs | 8 +- frame/bags-list/src/tests.rs | 15 +- frame/election-provider-support/src/lib.rs | 1 - frame/staking/src/benchmarking.rs | 121 +------- frame/staking/src/lib.rs | 21 -- frame/staking/src/mock.rs | 66 +---- frame/staking/src/pallet/impls.rs | 98 +++++-- frame/staking/src/pallet/mod.rs | 64 +---- frame/staking/src/testing_utils.rs | 6 +- frame/staking/src/tests.rs | 128 ++------- 14 files changed, 410 insertions(+), 593 deletions(-) create mode 100644 frame/bags-list/src/benchmarks.rs diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs new file mode 100644 index 0000000000000..7407bee1135b8 --- /dev/null +++ b/frame/bags-list/src/benchmarks.rs @@ -0,0 +1,108 @@ +// rebag { +// // The most expensive case for this call: +// // +// // - It doesn't matter where in the origin bag the stash lies; the number of reads and +// // writes is constant. We can use the case that the stash is the only one in the origin +// // bag, for simplicity. +// // - The destination bag is not empty, because then we need to update the `next` pointer +// // of the previous node in addition to the work we do otherwise. + +// use crate::voter_bags::{Bag, Node}; + +// let make_validator = |n: u32, balance_factor: u32| -> Result<(T::AccountId, T::AccountId), &'static str> { +// let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default())?; +// whitelist_account!(controller); + +// let prefs = ValidatorPrefs::default(); +// // bond the full value of the stash +// let free_balance = T::Currency::free_balance(&stash); +// Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), free_balance)?; +// Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; + +// Ok((stash, controller)) +// }; + +// // Clean up any existing state. +// clear_validators_and_nominators::(); + +// let thresholds = T::VoterBagThresholds::get(); + +// // stash controls the node account +// let bag0_thresh = thresholds[0]; +// let (stash, controller) = make_validator(USER_SEED, bag0_thresh as u32)?; + +// // create another validator with more stake +// let bag2_thresh = thresholds[2]; +// let (other_stash, _) = make_validator(USER_SEED + 1, bag2_thresh as u32)?; + +// // update the stash account's value/weight +// // +// // note that we have to manually update the ledger; if we were to just call +// // `Staking::::bond_extra`, then it would implicitly rebag. We want to separate that step +// // so we can measure it in isolation. +// let other_free_balance = T::Currency::free_balance(&other_stash); +// T::Currency::make_free_balance_be(&stash, other_free_balance); +// let controller = Staking::::bonded(&stash).ok_or("stash had no controller")?; +// let mut ledger = Staking::::ledger(&controller).ok_or("controller had no ledger")?; +// let extra = other_free_balance.checked_sub(&ledger.total).ok_or("balance did not increase")?; +// ledger.total += extra; +// ledger.active += extra; +// Staking::::update_ledger(&controller, &ledger); + +// // verify preconditions +// let weight_of = Staking::::weight_of_fn(); +// let node = Node::::from_id(&stash).ok_or("node not found for stash")?; +// ensure!( +// node.is_misplaced(&weight_of), +// "rebagging only makes sense when a node is misplaced", +// ); +// ensure!( +// { +// let origin_bag = Bag::::get(node.bag_upper).ok_or("origin bag not found")?; +// origin_bag.iter().count() == 1 +// }, +// "stash should be the only node in origin bag", +// ); +// let other_node = Node::::from_id(&other_stash).ok_or("node not found for other_stash")?; +// ensure!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); +// ensure!( +// { +// let destination_bag = Bag::::get(node.proper_bag_for()).ok_or("destination bag not found")?; +// destination_bag.iter().count() != 0 +// }, +// "destination bag should not be empty", +// ); +// drop(node); + +// // caller will call rebag +// let caller = whitelisted_caller(); +// // ensure it's distinct from the other accounts +// ensure!(caller != stash, "caller must not be the same as the stash"); +// ensure!(caller != controller, "caller must not be the same as the controller"); +// }: _(RawOrigin::Signed(caller), stash.clone()) +// verify { +// let node = Node::::from_id(&stash).ok_or("node not found for stash")?; +// ensure!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); +// } + +// regenerate { +// // number of validator intention. +// let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; +// // number of nominator intention. +// let n in (MAX_NOMINATORS / 2) .. MAX_NOMINATORS; + +// clear_validators_and_nominators::(); +// ensure!( +// create_validators_with_nominators_for_era::( +// v, +// n, +// T::MAX_NOMINATIONS as usize, +// true, +// None, +// ).is_ok(), +// "creating validators and nominators failed", +// ); +// }: { +// let migrated = VoterList::::regenerate(); +// ensure!(v + n == migrated, "didn't migrate right amount of voters"); +// } diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 96c8fd20945c7..a5639b63f72e1 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -229,4 +229,7 @@ impl SortedListProvider for Pallet { fn on_remove(voter: &T::AccountId) { List::::remove(voter) } + fn sanity_check() -> Result<(), &'static str> { + List::::sanity_check() + } } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 9f241f6d05110..09121d902d4df 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -20,8 +20,7 @@ use crate::Config; use codec::{Decode, Encode}; use frame_election_provider_support::{VoteWeight, VoteWeightProvider}; -use frame_support::{ensure, traits::Get, DefaultNoBound}; -use sp_runtime::SaturatedConversion; +use frame_support::{traits::Get, DefaultNoBound}; use sp_std::{ boxed::Box, collections::{btree_map::BTreeMap, btree_set::BTreeSet}, @@ -96,6 +95,7 @@ impl List { /// and the new. /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the /// new threshold set. + #[allow(dead_code)] pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { // we can't check all preconditions, but we can check one debug_assert!( @@ -172,6 +172,7 @@ impl List { /// consensus. /// /// Returns the number of voters migrated. + #[allow(dead_code)] pub fn regenerate( all: impl IntoIterator, weight_of: Box VoteWeight>, @@ -180,17 +181,6 @@ impl List { Self::insert_many(all, weight_of); } - /// Decode the length of the voter list. - pub fn decode_len() -> Option { - let maybe_len = crate::CounterForVoters::::try_get().ok().map(|n| n.saturated_into()); - debug_assert_eq!( - maybe_len.unwrap_or_default(), - crate::VoterNodes::::iter().count(), - "stored length must match count of nodes", - ); - maybe_len - } - /// Iterate over all nodes in all bags in the voter list. /// /// Full iteration can be expensive; it's recommended to limit the number of items with @@ -317,12 +307,12 @@ impl List { bag.remove_node(&node); bag.put(); } else { - debug_assert!(false, "every node must have an extant bag associated with it"); crate::log!( - error, - "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", - node.id, - ); + error, + "Node for voter {:?} did not have a bag; VoterBags is in an inconsistent state", + node.id, + ); + debug_assert!(false, "every node must have an extant bag associated with it"); } // put the voter into the appropriate new bag @@ -346,7 +336,8 @@ impl List { /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `List`. - pub(crate) fn sanity_check() -> Result<(), &'static str> { + pub(crate) fn sanity_check() -> Result<(), String> { + use frame_support::ensure; let mut seen_in_list = BTreeSet::new(); ensure!( Self::iter().map(|node| node.id).all(|voter| seen_in_list.insert(voter)), @@ -355,11 +346,10 @@ impl List { let iter_count = Self::iter().collect::>().len() as u32; let stored_count = crate::CounterForVoters::::get(); - ensure!(iter_count == stored_count, "iter_count != voter_count"); - - // let validators = staking::CounterForValidators::::get(); TOOD can we just remove? - // let nominators = staking::CounterForNominators::::get(); - // ensure!(validators + nominators == stored_count, "validators + nominators != voters"); + ensure!( + iter_count == stored_count, + format!("iter_count {} != stored_count {}", iter_count, stored_count) + ); let _ = T::BagThresholds::get() .into_iter() @@ -470,6 +460,7 @@ impl Bag { fn insert_node(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { + // DOCUMENT_ME // this should never happen, but this check prevents a worst case infinite loop debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); @@ -477,8 +468,8 @@ impl Bag { }; } + // update this node now let id = node.id.clone(); - node.prev = self.tail.clone(); node.next = None; node.put(); @@ -488,13 +479,17 @@ impl Bag { old_tail.next = Some(id.clone()); old_tail.put(); } + self.tail = Some(id.clone()); - // update the internal bag links + // ensure head exist. This is typically only set when the length of the bag is just 1, i.e. + // if this is the first insertion into the bag. In this case, both head and tail should + // point to the same voter node. if self.head.is_none() { self.head = Some(id.clone()); + debug_assert!(self.iter().count() == 1); } - self.tail = Some(id.clone()); + // update the voter's bag index. crate::VoterBagFor::::insert(id, self.bag_upper); } @@ -536,7 +531,7 @@ impl Bag { /// * Ensures tail has no next. /// * Ensures there are no loops, traversal from head to tail is correct. fn sanity_check(&self) -> Result<(), &'static str> { - ensure!( + frame_support::ensure!( self.head() .map(|head| head.prev().is_none()) // if there is no head, then there must not be a tail, meaning that the bag is @@ -545,7 +540,7 @@ impl Bag { "head has a prev" ); - ensure!( + frame_support::ensure!( self.tail() .map(|tail| tail.next().is_none()) // if there is no tail, then there must not be a head, meaning that the bag is @@ -555,7 +550,7 @@ impl Bag { ); let mut seen_in_bag = BTreeSet::new(); - ensure!( + frame_support::ensure!( self.iter() .map(|node| node.id) // each voter is only seen once, thus there is no cycle within a bag @@ -579,8 +574,6 @@ pub struct Node { /// The bag index is not stored in storage, but injected during all fetch operations. #[codec(skip)] pub(crate) bag_upper: VoteWeight, - // TODO maybe - // - store voter data here i.e targets } impl Node { diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index fc7c8dd1e73e9..554493a489c82 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -6,7 +6,7 @@ use crate::{ use frame_support::{assert_ok, assert_storage_noop}; #[test] -fn setup_works() { +fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { // syntactic sugar to create a raw node let node = |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; @@ -16,27 +16,29 @@ fn setup_works() { assert_eq!(VoterNodes::::iter().count(), 4); assert_eq!(VoterBags::::iter().count(), 2); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + // the state of the bags is as expected assert_eq!( VoterBags::::get(10).unwrap(), - Bag:: { head: Some(31), tail: Some(31), bag_upper: 0 } + Bag:: { head: Some(1), tail: Some(1), bag_upper: 0 } ); assert_eq!( VoterBags::::get(1_000).unwrap(), - Bag:: { head: Some(11), tail: Some(101), bag_upper: 0 } + Bag:: { head: Some(2), tail: Some(4), bag_upper: 0 } ); - assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); - assert_eq!(VoterNodes::::get(11).unwrap(), node(11, None, Some(21))); + assert_eq!(VoterBagFor::::get(2).unwrap(), 1000); + assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3))); - assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); - assert_eq!(VoterNodes::::get(21).unwrap(), node(21, Some(11), Some(101))); + assert_eq!(VoterBagFor::::get(3).unwrap(), 1000); + assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4))); - assert_eq!(VoterBagFor::::get(31).unwrap(), 10); - assert_eq!(VoterNodes::::get(31).unwrap(), node(31, None, None)); + assert_eq!(VoterBagFor::::get(4).unwrap(), 1000); + assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None)); - assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); - assert_eq!(VoterNodes::::get(101).unwrap(), node(101, Some(21), None)); + assert_eq!(VoterBagFor::::get(1).unwrap(), 10); + assert_eq!(VoterNodes::::get(1).unwrap(), node(1, None, None)); // non-existent id does not have a storage footprint assert_eq!(VoterBagFor::::get(41), None); @@ -45,11 +47,9 @@ fn setup_works() { // iteration of the bags would yield: assert_eq!( List::::iter().map(|n| *n.id()).collect::>(), - vec![11, 21, 101, 31], + vec![2, 3, 4, 1], // ^^ note the order of insertion in genesis! ); - - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); }); } @@ -58,16 +58,20 @@ fn notional_bag_for_works() { // under a threshold gives the next threshold. assert_eq!(notional_bag_for::(0), 10); assert_eq!(notional_bag_for::(9), 10); - assert_eq!(notional_bag_for::(11), 20); // at a threshold gives that threshold. assert_eq!(notional_bag_for::(10), 10); + // above the threshold. + assert_eq!(notional_bag_for::(11), 20); + let max_explicit_threshold = *::BagThresholds::get().last().unwrap(); assert_eq!(max_explicit_threshold, 10_000); // if the max explicit threshold is less than VoteWeight::MAX, assert!(VoteWeight::MAX > max_explicit_threshold); + // anything above it will belong to the VoteWeight::MAX bag. + assert_eq!(notional_bag_for::(max_explicit_threshold), max_explicit_threshold); assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); } @@ -75,21 +79,18 @@ fn notional_bag_for_works() { fn remove_last_voter_in_bags_cleans_bag() { ExtBuilder::default().build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); - // Bump 31 to a bigger bag - List::::remove(&31); - List::::insert(31, 10_000); + // bump 1 to a bigger bag + List::::remove(&1); + List::::insert(1, 10_000); // then the bag with bound 10 is wiped from storage. - assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101]), (10_000, vec![31])]); + assert_eq!(get_bags(), vec![(1_000, vec![2, 3, 4]), (10_000, vec![1])]); - // and can be recreated again as needed + // and can be recreated again as needed. List::::insert(77, 10); - assert_eq!( - get_bags(), - vec![(10, vec![77]), (1_000, vec![11, 21, 101]), (10_000, vec![31])] - ); + assert_eq!(get_bags(), vec![(10, vec![77]), (1_000, vec![2, 3, 4]), (10_000, vec![1])]); }); } @@ -99,35 +100,34 @@ mod voter_list { #[test] fn iteration_is_semi_sorted() { ExtBuilder::default() - .add_ids(vec![(51, 2_000), (61, 2_000)]) + .add_ids(vec![(5, 2_000), (6, 2_000)]) .build_and_execute(|| { // given assert_eq!( get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![5, 6])] ); // then assert_eq!( get_voter_list_as_ids(), vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, // last bag. + 5, 6, // best bag + 2, 3, 4, // middle bag + 1, // last bag. ] ); // when adding a voter that has a higher weight than pre-existing voters in the bag - List::::insert(71, 10); + List::::insert(7, 10); // then assert_eq!( get_voter_list_as_ids(), vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, - 71, // last bag; the new voter is last, because it is order of insertion + 5, 6, // best bag + 2, 3, 4, // middle bag + 1, 7, // last bag; new voter is last. ] ); }) @@ -137,12 +137,12 @@ mod voter_list { #[test] fn take_works() { ExtBuilder::default() - .add_ids(vec![(51, 2_000), (61, 2_000)]) + .add_ids(vec![(5, 2_000), (6, 2_000)]) .build_and_execute(|| { // given assert_eq!( get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![5, 6])] ); // when @@ -153,8 +153,8 @@ mod voter_list { assert_eq!( iteration, vec![ - 51, 61, // best bag, fully iterated - 11, 21, // middle bag, partially iterated + 5, 6, // best bag, fully iterated + 2, 3, // middle bag, partially iterated ] ); }) @@ -164,36 +164,30 @@ mod voter_list { fn insert_works() { ExtBuilder::default().build_and_execute(|| { // when inserting into an existing bag - List::::insert(71, 1_000); + List::::insert(5, 1_000); // then - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 71, 31]); - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5])]); + assert_eq!(get_voter_list_as_ids(), vec![2, 3, 4, 5, 1]); // when inserting into a non-existent bag - List::::insert(81, 1_001); + List::::insert(6, 1_001); // then - assert_eq!(get_voter_list_as_ids(), vec![81, 11, 21, 101, 71, 31]); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1_000, vec![11, 21, 101, 71]), (2_000, vec![81])] - ); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5]), (2000, vec![6])]); + assert_eq!(get_voter_list_as_ids(), vec![6, 2, 3, 4, 5, 1]); }); } #[test] fn remove_works() { use crate::{CounterForVoters, VoterBags, VoterNodes}; - - let check_storage = |id, counter, voters, bags| { + let ensure_left = |id, counter| { assert!(!VoterBagFor::::contains_key(id)); assert!(!VoterNodes::::contains_key(id)); assert_eq!(CounterForVoters::::get(), counter); assert_eq!(VoterBagFor::::iter().count() as u32, counter); assert_eq!(VoterNodes::::iter().count() as u32, counter); - assert_eq!(get_voter_list_as_ids(), voters); - assert_eq!(get_bags(), bags); }; ExtBuilder::default().build_and_execute(|| { @@ -203,52 +197,36 @@ mod voter_list { List::::remove(&42); // then nothing changes - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(get_voter_list_as_ids(), vec![2, 3, 4, 1]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); assert_eq!(CounterForVoters::::get(), 4); // when removing a node from a bag with multiple nodes - List::::remove(&11); + List::::remove(&2); // then - assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); - check_storage( - 11, - 3, - vec![21, 101, 31], // voter list - vec![(10, vec![31]), (1_000, vec![21, 101])], // bags - ); + assert_eq!(get_voter_list_as_ids(), vec![3, 4, 1]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![3, 4])]); + ensure_left(2, 3); // when removing a node from a bag with only one node: - List::::remove(&31); + List::::remove(&1); // then - assert_eq!(get_voter_list_as_ids(), vec![21, 101]); - check_storage( - 31, - 2, - vec![21, 101], // voter list - vec![(1_000, vec![21, 101])], // bags - ); - assert!(!VoterBags::::contains_key(10)); // bag 10 is removed + assert_eq!(get_voter_list_as_ids(), vec![3, 4]); + assert_eq!(get_bags(), vec![(1000, vec![3, 4])]); + ensure_left(1, 2); + // bag 10 is removed + assert!(!VoterBags::::contains_key(10)); // remove remaining voters to make sure storage cleans up as expected - List::::remove(&21); - check_storage( - 21, - 1, - vec![101], // voter list - vec![(1_000, vec![101])], // bags - ); + List::::remove(&3); + ensure_left(3, 1); + assert_eq!(get_voter_list_as_ids(), vec![4]); - List::::remove(&101); - check_storage( - 101, - 0, - Vec::::new(), // voter list - vec![], // bags - ); - assert!(!VoterBags::::contains_key(1_000)); // bag 1_000 is removed + List::::remove(&4); + ensure_left(4, 0); + assert_eq!(get_voter_list_as_ids(), Vec::::new()); // bags are deleted via removals assert_eq!(VoterBags::::iter().count(), 0); @@ -258,51 +236,45 @@ mod voter_list { #[test] fn update_position_for_works() { ExtBuilder::default().build_and_execute(|| { - // given a correctly placed account 31 - let node_31 = Node::::from_id(&31).unwrap(); - assert!(!node_31.is_misplaced(10)); + // given a correctly placed account 1 + let node = Node::::from_id(&1).unwrap(); + assert!(!node.is_misplaced(10)); - // when account 31's weight becomes 20, it is then misplaced. - let weight_20 = 20; - assert!(node_31.is_misplaced(weight_20)); + // .. it is invalid with weight 20 + assert!(node.is_misplaced(20)); // then updating position moves it to the correct bag - assert_eq!(List::::update_position_for(node_31, weight_20), Some((10, 20))); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + assert_eq!(List::::update_position_for(node, 20), Some((10, 20))); - assert_eq!(get_bags(), vec![(20, vec![31]), (1_000, vec![11, 21, 101])]); - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + assert_eq!(get_bags(), vec![(20, vec![1]), (1_000, vec![2, 3, 4])]); - // and if you try and update the position with no change in weight - let node_31 = Node::::from_id(&31).unwrap(); + // get the new updated node; try and update the position with no change in weight. + let node = Node::::from_id(&1).unwrap(); + // TODO: we can pass a ref to node to this function as well. assert_storage_noop!(assert_eq!( - List::::update_position_for(node_31, weight_20), - None, + List::::update_position_for(node.clone(), 20), + None )); - // when account 31 needs to be moved to an existing higher bag - let weight_500 = 500; + // then move it to bag 1000 by giving it weight 500. + assert_eq!(List::::update_position_for(node.clone(), 500), Some((20, 1_000))); + assert_eq!(get_bags(), vec![(1_000, vec![2, 3, 4, 1])]); - // then updating positions moves it to the correct bag - let node_31 = Node::::from_id(&31).unwrap(); - assert_eq!( - List::::update_position_for(node_31, weight_500), - Some((20, 1_000)) - ); - assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101, 31])]); - assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); - - // when account 31 has a higher but within its current bag - let weight_1000 = 1_000; - - // then nothing changes - let node_31 = Node::::from_id(&31).unwrap(); + // moving withing that bag again is a noop + let node = Node::::from_id(&1).unwrap(); assert_storage_noop!(assert_eq!( - List::::update_position_for(node_31, weight_1000), + List::::update_position_for(node.clone(), 750), + None, + )); + assert_storage_noop!(assert_eq!( + List::::update_position_for(node, 1000), None, )); }); } } + mod bags { use super::*; @@ -310,8 +282,8 @@ mod bags { fn get_works() { ExtBuilder::default().build_and_execute(|| { let check_bag = |bag_upper, head, tail, ids| { + // @zeke TODO: why? assert_storage_noop!(Bag::::get(bag_upper)); - let bag = Bag::::get(bag_upper).unwrap(); let bag_ids = bag.iter().map(|n| *n.id()).collect::>(); @@ -319,30 +291,27 @@ mod bags { assert_eq!(bag_ids, ids); }; - // given uppers of bags that exist. - let existing_bag_uppers = vec![10, 1_000]; + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); // we can fetch them - check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); - // (getting the same bag twice has the same results) - check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); - check_bag(existing_bag_uppers[1], Some(11), Some(101), vec![11, 21, 101]); + check_bag(10, Some(1), Some(1), vec![1]); + check_bag(1000, Some(2), Some(4), vec![2, 3, 4]); // and all other uppers don't get bags. ::BagThresholds::get() .iter() .chain(iter::once(&VoteWeight::MAX)) - .filter(|bag_upper| !existing_bag_uppers.contains(bag_upper)) + .filter(|bag_upper| !vec![10, 1000].contains(bag_upper)) .for_each(|bag_upper| { assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); assert!(!VoterBags::::contains_key(*bag_upper)); }); // when we make a pre-existing bag empty - List::::remove(&31); + List::::remove(&1); // then - assert_eq!(Bag::::get(existing_bag_uppers[0]), None) + assert_eq!(Bag::::get(10), None) }); } @@ -357,7 +326,7 @@ mod bags { #[test] fn insert_node_happy_paths_works() { - ExtBuilder::default().build_and_execute(|| { + ExtBuilder::default().build_and_execute_no_post_check(|| { let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; // when inserting into a bag with 1 node @@ -365,37 +334,37 @@ mod bags { // (note: bags api does not care about balance or ledger) bag_10.insert_node(node(42, bag_10.bag_upper)); // then - assert_eq!(bag_as_ids(&bag_10), vec![31, 42]); + assert_eq!(bag_as_ids(&bag_10), vec![1, 42]); // when inserting into a bag with 3 nodes let mut bag_1000 = Bag::::get(1_000).unwrap(); bag_1000.insert_node(node(52, bag_1000.bag_upper)); // then - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 52]); + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 52]); // when inserting into a new bag let mut bag_20 = Bag::::get_or_make(20); - bag_20.insert_node(node(71, bag_20.bag_upper)); + bag_20.insert_node(node(62, bag_20.bag_upper)); // then - assert_eq!(bag_as_ids(&bag_20), vec![71]); + assert_eq!(bag_as_ids(&bag_20), vec![62]); // when inserting a node pointing to the accounts not in the bag let node_61 = Node:: { id: 61, prev: Some(21), next: Some(101), bag_upper: 20 }; bag_20.insert_node(node_61); // then ids are in order - assert_eq!(bag_as_ids(&bag_20), vec![71, 61]); + assert_eq!(bag_as_ids(&bag_20), vec![62, 61]); // and when the node is re-fetched all the info is correct assert_eq!( Node::::get(20, &61).unwrap(), - Node:: { id: 61, prev: Some(71), next: None, bag_upper: 20 } + Node:: { id: 61, prev: Some(62), next: None, bag_upper: 20 } ); // state of all bags is as expected bag_20.put(); // need to put this bag so its in the storage map assert_eq!( get_bags(), - vec![(10, vec![31, 42]), (20, vec![71, 61]), (1_000, vec![11, 21, 101, 52])] + vec![(10, vec![1, 42]), (20, vec![62, 61]), (1_000, vec![2, 3, 4, 52])] ); }); } @@ -404,52 +373,60 @@ mod bags { #[test] fn insert_node_bad_paths_documented() { let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; - ExtBuilder::default().build_and_execute(|| { - // when inserting a node with both prev & next pointing at an account in the bag - // and an incorrect bag_upper + ExtBuilder::default().build_and_execute_no_post_check(|| { + // when inserting a node with both prev & next pointing at an account in an incorrect + // bag. let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(42, Some(11), Some(11), 0)); + bag_1000.insert_node(node(42, Some(1), Some(1), 0)); + + // then the proper perv and next is set. + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 42]); - // then the ids are in the correct order - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 42]); // and when the node is re-fetched all the info is correct assert_eq!( Node::::get(1_000, &42).unwrap(), - node(42, Some(101), None, bag_1000.bag_upper) + node(42, Some(4), None, bag_1000.bag_upper) ); }); - ExtBuilder::default().build_and_execute(|| { - // given 21 is in in bag_1000 (and not a tail node) + ExtBuilder::default().build_and_execute_no_post_check(|| { + // given 3 is in in bag_1000 (and not a tail node) let mut bag_1000 = Bag::::get(1_000).unwrap(); - assert_eq!(bag_as_ids(&bag_1000)[1], 21); + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); - // when inserting a node with duplicate id 21 - bag_1000.insert_node(node(21, None, None, bag_1000.bag_upper)); + // when inserting a node with duplicate id 3 + bag_1000.insert_node(node(3, None, None, bag_1000.bag_upper)); // then all the nodes after the duplicate are lost (because it is set as the tail) - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21]); - // and the re-fetched node has an **incorrect** prev pointer. + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3]); + // also in the full iteration, 2 and 3 are from the 1000 bag and 1 from bag 10. + assert_eq!(get_voter_list_as_ids(), vec![2, 3, 1]); + + // and the last accessible node has an **incorrect** prev pointer. + // TODO: consider doing a check on insert, it is cheap. assert_eq!( - Node::::get(1_000, &21).unwrap(), - node(21, Some(101), None, bag_1000.bag_upper) + Node::::get(1_000, &3).unwrap(), + node(3, Some(4), None, bag_1000.bag_upper) ); }); - ExtBuilder::default().build_and_execute(|| { + ExtBuilder::default().build_and_execute_no_post_check(|| { // when inserting a duplicate id of the head let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(11, None, None, 0)); + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); + bag_1000.insert_node(node(2, None, None, 0)); // then all nodes after the head are lost - assert_eq!(bag_as_ids(&bag_1000), vec![11]); + assert_eq!(bag_as_ids(&bag_1000), vec![2]); + // and the re-fetched node has bad pointers assert_eq!( - Node::::get(1_000, &11).unwrap(), - node(11, Some(101), None, bag_1000.bag_upper) // ^^^ despite being the bags head, it has a prev + Node::::get(1_000, &2).unwrap(), + node(2, Some(4), None, bag_1000.bag_upper) ); + // ^^^ despite being the bags head, it has a prev - assert_eq!(bag_1000, Bag { head: Some(11), tail: Some(11), bag_upper: 1_000 }) + assert_eq!(bag_1000, Bag { head: Some(2), tail: Some(2), bag_upper: 1_000 }) }); } @@ -461,12 +438,12 @@ mod bags { let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; // given - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])],); let mut bag_1000 = Bag::::get(1_000).unwrap(); // when inserting a duplicate id that is already the tail - assert_eq!(bag_1000.tail, Some(101)); - bag_1000.insert_node(node(101, None, None, bag_1000.bag_upper)); // panics + assert_eq!(bag_1000.tail, Some(4)); + bag_1000.insert_node(node(4, None, None, bag_1000.bag_upper)); // panics }); } @@ -612,16 +589,16 @@ mod node { #[test] fn is_misplaced_works() { ExtBuilder::default().build_and_execute(|| { - let node_31 = Node::::get(10, &31).unwrap(); + let node = Node::::get(10, &1).unwrap(); // given - assert_eq!(node_31.bag_upper, 10); + assert_eq!(node.bag_upper, 10); // then - assert!(!node_31.is_misplaced(0)); - assert!(!node_31.is_misplaced(9)); - assert!(!node_31.is_misplaced(10)); - assert!(node_31.is_misplaced(11)); + assert!(!node.is_misplaced(0)); + assert!(!node.is_misplaced(9)); + assert!(!node.is_misplaced(10)); + assert!(node.is_misplaced(11)); }); } diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 0c7869e38b8c0..9b295bde5821d 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -11,7 +11,7 @@ parameter_types! { pub struct StakingMock; impl frame_election_provider_support::VoteWeightProvider for StakingMock { - fn vote_weight(who: &AccountId) -> VoteWeight { + fn vote_weight(_: &AccountId) -> VoteWeight { NextVoteWeight::get() } } @@ -69,8 +69,6 @@ frame_support::construct_runtime!( ); pub(crate) mod ext_builder { - use frame_support::RuntimeDebugNoBound; - use super::*; /// Default AccountIds and their weights. @@ -109,6 +107,10 @@ pub(crate) mod ext_builder { List::::sanity_check().expect("Sanity check post condition failed") }) } + + pub(crate) fn build_and_execute_no_post_check(self, test: impl FnOnce() -> ()) { + self.build().execute_with(test) + } } } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 66121eb75e129..5140b84a0ea56 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -269,21 +269,24 @@ mod sorted_list_provider { #[test] fn sanity_check_works() { - ExtBuilder::default().build_and_execute(|| { + ExtBuilder::default().build_and_execute_no_post_check(|| { assert_ok!(List::::sanity_check()); }); // make sure there are no duplicates. - ExtBuilder::default().build_and_execute(|| { - >::on_insert(1, 10); - assert_eq!(List::::sanity_check(), Err("duplicate identified")); + ExtBuilder::default().build_and_execute_no_post_check(|| { + >::on_insert(2, 10); + assert_eq!(List::::sanity_check(), Err("duplicate identified".to_string())); }); // ensure count is in sync with `CounterForVoters`. - ExtBuilder::default().build_and_execute(|| { + ExtBuilder::default().build_and_execute_no_post_check(|| { crate::CounterForVoters::::mutate(|counter| *counter += 1); assert_eq!(crate::CounterForVoters::::get(), 5); - assert_eq!(List::::sanity_check(), Err("iter_count != voter_count")); + assert_eq!( + List::::sanity_check(), + Err("iter_count 4 != stored_count 5".to_string()) + ); }); } } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 9f77b88a1b718..c87edbfb3a174 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -317,7 +317,6 @@ pub trait SortedListProvider { /// Hook for removing a voter from the list. fn on_remove(voter: &AccountId); /// Sanity check internal state of list. Only meant for debug compilation. - #[cfg(test)] fn sanity_check() -> Result<(), &'static str>; } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index ff4a8ba986b00..e63856906b842 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -26,13 +26,12 @@ use frame_support::{ traits::{Currency, Get, Imbalance}, }; use sp_runtime::{ - traits::{CheckedSub, StaticLookup, Zero}, + traits::{StaticLookup, Zero}, Perbill, Percent, }; use sp_staking::SessionIndex; use sp_std::prelude::*; -use crate::voter_bags::VoterList; pub use frame_benchmarking::{ account, benchmarks, impl_benchmark_test_suite, whitelist_account, whitelisted_caller, }; @@ -49,7 +48,7 @@ const MAX_SLASHES: u32 = 1000; // read and write operations. fn add_slashing_spans(who: &T::AccountId, spans: u32) { if spans == 0 { - return + return; } // For the first slashing span, we initialize @@ -111,11 +110,6 @@ pub fn create_validator_with_nominators( assert_eq!(new_validators.len(), 1); assert_eq!(new_validators[0], v_stash, "Our validator was not selected!"); - assert_eq!( - VoterList::::decode_len().unwrap_or_default() as u32, - CounterForNominators::::get() + CounterForValidators::::get(), - "ensure storage has been mutated coherently", - ); assert_ne!(CounterForValidators::::get(), 0); assert_ne!(CounterForNominators::::get(), 0); @@ -609,7 +603,7 @@ benchmarks! { let num_voters = (v + n) as usize; }: { - let voters = >::get_npos_voters(None, num_voters); + let voters = >::get_npos_voters(None); assert_eq!(voters.len(), num_voters); } @@ -658,115 +652,6 @@ benchmarks! { verify { assert!(!Validators::::contains_key(controller)); } - - rebag { - // The most expensive case for this call: - // - // - It doesn't matter where in the origin bag the stash lies; the number of reads and - // writes is constant. We can use the case that the stash is the only one in the origin - // bag, for simplicity. - // - The destination bag is not empty, because then we need to update the `next` pointer - // of the previous node in addition to the work we do otherwise. - - use crate::voter_bags::{Bag, Node}; - - let make_validator = |n: u32, balance_factor: u32| -> Result<(T::AccountId, T::AccountId), &'static str> { - let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default())?; - whitelist_account!(controller); - - let prefs = ValidatorPrefs::default(); - // bond the full value of the stash - let free_balance = T::Currency::free_balance(&stash); - Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), free_balance)?; - Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; - - Ok((stash, controller)) - }; - - // Clean up any existing state. - clear_validators_and_nominators::(); - - let thresholds = T::VoterBagThresholds::get(); - - // stash controls the node account - let bag0_thresh = thresholds[0]; - let (stash, controller) = make_validator(USER_SEED, bag0_thresh as u32)?; - - // create another validator with more stake - let bag2_thresh = thresholds[2]; - let (other_stash, _) = make_validator(USER_SEED + 1, bag2_thresh as u32)?; - - // update the stash account's value/weight - // - // note that we have to manually update the ledger; if we were to just call - // `Staking::::bond_extra`, then it would implicitly rebag. We want to separate that step - // so we can measure it in isolation. - let other_free_balance = T::Currency::free_balance(&other_stash); - T::Currency::make_free_balance_be(&stash, other_free_balance); - let controller = Staking::::bonded(&stash).ok_or("stash had no controller")?; - let mut ledger = Staking::::ledger(&controller).ok_or("controller had no ledger")?; - let extra = other_free_balance.checked_sub(&ledger.total).ok_or("balance did not increase")?; - ledger.total += extra; - ledger.active += extra; - Staking::::update_ledger(&controller, &ledger); - - // verify preconditions - let weight_of = Staking::::weight_of_fn(); - let node = Node::::from_id(&stash).ok_or("node not found for stash")?; - ensure!( - node.is_misplaced(&weight_of), - "rebagging only makes sense when a node is misplaced", - ); - ensure!( - { - let origin_bag = Bag::::get(node.bag_upper).ok_or("origin bag not found")?; - origin_bag.iter().count() == 1 - }, - "stash should be the only node in origin bag", - ); - let other_node = Node::::from_id(&other_stash).ok_or("node not found for other_stash")?; - ensure!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); - ensure!( - { - let destination_bag = Bag::::get(node.proper_bag_for()).ok_or("destination bag not found")?; - destination_bag.iter().count() != 0 - }, - "destination bag should not be empty", - ); - drop(node); - - // caller will call rebag - let caller = whitelisted_caller(); - // ensure it's distinct from the other accounts - ensure!(caller != stash, "caller must not be the same as the stash"); - ensure!(caller != controller, "caller must not be the same as the controller"); - }: _(RawOrigin::Signed(caller), stash.clone()) - verify { - let node = Node::::from_id(&stash).ok_or("node not found for stash")?; - ensure!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); - } - - regenerate { - // number of validator intention. - let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; - // number of nominator intention. - let n in (MAX_NOMINATORS / 2) .. MAX_NOMINATORS; - - clear_validators_and_nominators::(); - ensure!( - create_validators_with_nominators_for_era::( - v, - n, - T::MAX_NOMINATIONS as usize, - true, - None, - ).is_ok(), - "creating validators and nominators failed", - ); - }: { - let migrated = VoterList::::regenerate(); - ensure!(v + n == migrated, "didn't migrate right amount of voters"); - } } #[cfg(test)] diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 6e8c36b26ddb5..f0ab0e6f7560b 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -291,7 +291,6 @@ pub mod weights; mod pallet; use codec::{Decode, Encode, HasCompact}; -use frame_election_provider_support::VoteWeight; use frame_support::{ traits::{Currency, Get}, weights::Weight, @@ -796,23 +795,3 @@ where R::is_known_offence(offenders, time_slot) } } - -/// A simple voter list implementation that does not require any additional pallets. -pub struct StakingVoterListStub(sp_std::marker::PhantomData); -impl frame_election_provider_support::SortedListProvider - for StakingVoterListStub -{ - /// Returns iterator over voter list, which can have `take` called on it. - fn get_voters() -> Box> { - Box::new(Nominators::::iter().map(|(n, _)| n)) - } - fn count() -> u32 { - CounterForNominators::::get() - } - fn on_insert(_voter: T::AccountId, _weight: VoteWeight) {} - fn on_update(_voter: &T::AccountId, _weight: VoteWeight) {} - fn on_remove(_voter: &T::AccountId) {} - fn sanity_check() -> Result<(), &'static str> { - Ok(()) - } -} diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 0cebfdd07af91..9241b67bc14f1 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,9 +17,9 @@ //! Test utilities -use crate as staking; -use crate::{voter_bags::VoterList, *}; +use crate::{self as pallet_staking, *}; use frame_election_provider_support::onchain; +use frame_election_provider_support::SortedListProvider; use frame_support::{ assert_ok, parameter_types, traits::{ @@ -104,7 +104,7 @@ frame_support::construct_runtime!( Authorship: pallet_authorship::{Pallet, Call, Storage, Inherent}, Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - Staking: staking::{Pallet, Call, Config, Storage, Event}, + Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, // VoterBags: pallet_voter_bags::{Pallet, Call, Storage, Event}, } @@ -283,9 +283,7 @@ impl crate::pallet::pallet::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); - type VoterBagThresholds = VoterBagThresholds; - // type SortedListProvider = pallet_voter_bags::VoterBagsVoterListProvider; - type SortedListProvider = staking::VoterList; + type SortedListProvider = UseNominatorsMap; } impl frame_system::offchain::SendTransactionTypes for Test @@ -459,7 +457,7 @@ impl ExtBuilder { (101, 100, balance_factor * 500, StakerStatus::::Nominator(nominated)), ]; } - let _ = staking::GenesisConfig:: { + let _ = pallet_staking::GenesisConfig:: { stakers, validator_count: self.validator_count, minimum_validator_count: self.minimum_validator_count, @@ -504,23 +502,13 @@ impl ExtBuilder { ext.execute_with(test); ext.execute_with(post_conditions); } - /// WARNING: This should only be use for testing `VoterList` api or lower. - pub fn build_and_execute_without_check_count(self, test: impl FnOnce() -> ()) { - let mut ext = self.build(); - ext.execute_with(test); - ext.execute_with(post_conditions_without_check_count); - } } fn post_conditions() { - post_conditions_without_check_count(); - check_count(); -} - -fn post_conditions_without_check_count() { check_nominators(); check_exposures(); check_ledgers(); + check_count(); } fn check_count() { @@ -529,8 +517,9 @@ fn check_count() { assert_eq!(nominator_count, CounterForNominators::::get()); assert_eq!(validator_count, CounterForValidators::::get()); - let voters_count = CounterForVoters::::get(); - assert_eq!(voters_count, nominator_count + validator_count); + // the voters that the voter list is storing for us. + let external_voters = ::SortedListProvider::count(); + assert_eq!(external_voters, nominator_count); } fn check_ledgers() { @@ -833,7 +822,7 @@ macro_rules! assert_session_era { }; } -pub(crate) fn staking_events() -> Vec> { +pub(crate) fn staking_events() -> Vec> { System::events() .into_iter() .map(|r| r.event) @@ -844,38 +833,3 @@ pub(crate) fn staking_events() -> Vec> { pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { (Balances::free_balance(who), Balances::reserved_balance(who)) } - -use crate::voter_bags::Bag; -/// Returns the nodes of all non-empty bags. -pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { - VoterBagThresholds::get() - .into_iter() - .filter_map(|t| { - Bag::::get(*t) - .map(|bag| (*t, bag.iter().map(|n| n.voter().id).collect::>())) - }) - .collect::>() -} - -pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { - bag.iter().map(|n| n.voter().id).collect::>() -} - -pub(crate) fn get_voter_list_as_ids() -> Vec { - VoterList::::iter().map(|n| n.voter().id).collect::>() -} - -pub(crate) fn get_voter_list_as_voters() -> Vec> { - VoterList::::iter().map(|node| node.voter().clone()).collect::>() -} - -// Useful for when you want to change the effectively bonded value but you don't want to use -// the bond extrinsics because they implicitly rebag. -pub(crate) fn set_ledger_and_free_balance(account: &AccountId, value: Balance) { - Balances::make_free_balance_be(account, value); - let controller = Staking::bonded(account).unwrap(); - let mut ledger = Staking::ledger(&controller).unwrap(); - ledger.total = value; - ledger.active = value; - Staking::update_ledger(&controller, &ledger); -} diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 283d07b015aba..22aab837f4c18 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -651,17 +651,54 @@ impl Pallet { /// `voter_count` voters. Use with care. pub fn get_npos_voters( maybe_max_len: Option, - voter_count: usize, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { - let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); - // Collect all slashing spans into a BTreeMap for further queries. - // let slashing_spans = >::iter().collect::>(); - - let unfiltered_voters: Vec = - T::SortedListProvider::get_voters().take(wanted_voters).collect(); - // TODO: filter slashing spans - // concatenate account ids to voter data. - unimplemented!() + // TODO: weight this function and see how many can fit in a block. + let weight_of = Self::weight_of_fn(); + let nominator_count = CounterForNominators::::get() as usize; + let validator_count = CounterForValidators::::get() as usize; + let all_voter_count = validator_count.saturating_add(nominator_count); + + let total_len = maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count); + let mut all_voters = Vec::<_>::with_capacity(total_len); + + // first, grab all validators. + for (validator, _) in >::iter() { + // Append self vote. + let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]); + all_voters.push(self_vote); + } + + // see how many voters we can grab from nominator sorted list, and grab them. + let nominators_quota = total_len.saturating_sub(validator_count); + let slashing_spans = >::iter().collect::>(); + for nominator in T::SortedListProvider::iter().take(nominators_quota) { + if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = + >::get(&nominator) + { + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + if !targets.len().is_zero() { + all_voters.push((nominator.clone(), weight_of(&nominator), targets)) + } + } else { + log!(warn, "invalid item in `SortedListProvider`: {:?}", nominator) + } + } + + // all_voters should have not re-allocated. + debug_assert!(all_voters.capacity() == all_voters.len()); + + log!( + info, + "generated {} npos voters, {} from validators and {} nominators", + all_voter_count, + validator_count, + nominators_quota + ); + all_voters } /// This is a very expensive function and result should be cached versus being called multiple times. @@ -720,8 +757,6 @@ impl Pallet { CounterForValidators::::mutate(|x| x.saturating_inc()) } Validators::::insert(who, prefs); - T::SortedListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } /// This function will remove a validator from the `Validators` storage map, @@ -736,8 +771,6 @@ impl Pallet { if Validators::::contains_key(who) { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); - T::SortedListProvider::on_remove(who); - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); true } else { false @@ -759,16 +792,15 @@ impl ) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec)>, Weight)> { let nominator_count = CounterForNominators::::get(); let validator_count = CounterForValidators::::get(); - let voter_count = nominator_count.saturating_add(validator_count) as usize; // check a few counters one last time... debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); - // debug_assert_eq!( - // voter_count, - // T::SortedListProvider::get_voters().count(), - // "voter_count must be accurate", - // ); + debug_assert_eq!( + CounterForNominators::::get(), + T::SortedListProvider::count(), + "voter_count must be accurate", + ); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( @@ -777,7 +809,7 @@ impl slashing_span_count as u32, ); - Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) + Ok((Self::get_npos_voters(maybe_max_len), weight)) } fn targets(maybe_max_len: Option) -> data_provider::Result<(Vec, Weight)> { @@ -1143,3 +1175,27 @@ where consumed_weight } } + +/// A simple voter list implementation that does not require any additional pallets. +pub struct UseNominatorsMap(sp_std::marker::PhantomData); +impl SortedListProvider for UseNominatorsMap { + /// Returns iterator over voter list, which can have `take` called on it. + fn iter() -> Box> { + Box::new(Nominators::::iter().map(|(n, _)| n)) + } + fn count() -> u32 { + CounterForNominators::::get() + } + fn on_insert(_voter: T::AccountId, _weight: VoteWeight) { + // nothing to do on update. + } + fn on_update(_voter: &T::AccountId, _weight: VoteWeight) { + // nothing to do on update. + } + fn on_remove(_voter: &T::AccountId) { + // nothing to do on update. + } + fn sanity_check() -> Result<(), &'static str> { + Ok(()) + } +} diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index e5203f0064723..cfa2ce1a3bdfe 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -143,60 +143,11 @@ pub mod pallet { #[pallet::constant] type MaxNominatorRewardedPerValidator: Get; + /// Something that can provide a sorted list of voters is a somewhat sorted way. + type SortedListProvider: SortedListProvider; + /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; - - /// The list of thresholds separating the various voter bags. - /// - /// Voters are separated into unsorted bags according to their vote weight. This specifies - /// the thresholds separating the bags. A voter's bag is the largest bag for which the - /// voter's weight is less than or equal to its upper threshold. - /// - /// When voters are iterated, higher bags are iterated completely before lower bags. This - /// means that iteration is _semi-sorted_: voters of higher weight tend to come before - /// voters of lower weight, but peer voters within a particular bag are sorted in insertion - /// order. - /// - /// # Expressing the constant - /// - /// This constant must be sorted in strictly increasing order. Duplicate items are not - /// permitted. - /// - /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be - /// specified within the bag. For any two threshold lists, if one ends with - /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists - /// will behave identically. - /// - /// # Calculation - /// - /// It is recommended to generate the set of thresholds in a geometric series, such that - /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * - /// constant_ratio).max(threshold[k] + 1)` for all `k`. - /// - /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use - /// them, the `make-bags` feature must be enabled. - /// - /// # Examples - /// - /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, - /// and iteration is strictly in insertion order. - /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is equal to 2. - /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is approximately equal - /// to 1.248. - /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will - /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. - /// - /// # Migration - /// - /// In the event that this list ever changes, a copy of the old bags list must be retained. - /// With that `VoterList::migrate` can be called, which will perform the appropriate - /// migration. - #[pallet::constant] - type VoterBagThresholds: Get<&'static [VoteWeight]>; - - type SortedListProvider: SortedListProvider; } #[pallet::extra_constants] @@ -545,7 +496,6 @@ pub mod pallet { MinNominatorBond::::put(self.min_nominator_bond); MinValidatorBond::::put(self.min_validator_bond); - let mut num_voters: u32 = 0; for &(ref stash, ref controller, balance, ref status) in &self.stakers { log!( trace, @@ -577,8 +527,6 @@ pub mod pallet { Default::default(), ) { log!(warn, "failed to validate staker at genesis: {:?}.", why); - } else { - num_voters += 1; } } StakerStatus::Nominator(votes) => { @@ -587,18 +535,16 @@ pub mod pallet { votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), ) { log!(warn, "failed to nominate staker at genesis: {:?}.", why); - } else { - num_voters += 1; } } _ => (), }; } - // all voters are inserted sanely. + // all voters are reported to the `SortedListProvider`. assert_eq!( T::SortedListProvider::count(), - num_voters, + CounterForNominators::::get(), "not all genesis stakers were inserted into bags, something is wrong." ); } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 44bd84b9a167f..c9b39e9c0db13 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -27,7 +27,6 @@ use rand_chacha::{ }; use sp_io::hashing::blake2_256; -use crate::voter_bags::VoterList; use frame_support::{pallet_prelude::*, traits::Currency}; use sp_runtime::{traits::StaticLookup, Perbill}; use sp_std::prelude::*; @@ -40,7 +39,6 @@ pub fn clear_validators_and_nominators() { CounterForValidators::::kill(); Nominators::::remove_all(None); CounterForNominators::::kill(); - VoterList::::clear(); } /// Grab a funded user. @@ -74,7 +72,7 @@ pub fn create_stash_controller( amount, destination, )?; - return Ok((stash, controller)) + return Ok((stash, controller)); } /// Create a stash and controller pair, where the controller is dead, and payouts go to controller. @@ -96,7 +94,7 @@ pub fn create_stash_and_dead_controller( amount, destination, )?; - return Ok((stash, controller)) + return Ok((stash, controller)); } /// create `max` validators. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index c295c311ea8b1..29f8e415f6ffb 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -18,8 +18,7 @@ //! Tests for the module. use super::{Event, *}; -use crate::voter_bags::VoterList; -use frame_election_provider_support::{ElectionProvider, Support}; +use frame_election_provider_support::{ElectionProvider, SortedListProvider, Support}; use frame_support::{ assert_noop, assert_ok, dispatch::WithPostDispatchInfo, @@ -178,11 +177,6 @@ fn basic_setup_works() { // New era is not being forced assert_eq!(Staking::force_era(), Forcing::NotForcing); - - // check the bags - assert_eq!(CounterForVoters::::get(), 4); - - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); }); } @@ -282,9 +276,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 + - part_for_100_from_10 * total_payout_0 * 2 / 3 + - part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * total_payout_0 * 2 / 3 + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -320,9 +314,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 + - part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) + - part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -3833,95 +3827,6 @@ fn on_finalize_weight_is_nonzero() { }) } -// end-to-end nodes of the voter bags operation. -mod voter_bags { - use super::Origin; - use crate::{mock::*, ValidatorPrefs}; - use frame_support::{assert_ok, traits::Currency}; - - #[test] - fn insert_and_remove_works() { - // we test insert/remove indirectly via `validate`, `nominate`, and chill - ExtBuilder::default().build_and_execute(|| { - // given - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); - - // `bond` - bond(42, 43, 2_000); - // does not insert the voter - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); - - // `validate` - assert_ok!(Staking::validate(Origin::signed(43).into(), ValidatorPrefs::default())); - // moves the voter into a bag - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); - - // `nominate`-ing, but not changing active stake (which implicitly calls remove) - assert_ok!(Staking::nominate(Origin::signed(43), vec![11])); - // does not change the voters position - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); - - // `chill` - assert_ok!(Staking::chill(Origin::signed(43))); - // removes the voter - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); - }); - } - - #[test] - fn rebag_works() { - ExtBuilder::default().build_and_execute(|| { - // add a nominator to genesis state - bond_nominator(42, 43, 20, vec![11]); - Balances::make_free_balance_be(&42, 2_000); - - // given - assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); - - // increase stake and implicitly rebag with `bond_extra` to the level of non-existent bag - assert_ok!(Staking::bond_extra(Origin::signed(42), 1_980)); // 20 + 1_980 = 2_000 - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); - - // decrease stake within the range of the current bag - assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 - // does not change bags - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] - ); - - // reduce stake to the level of a non-existent bag - assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 - // creates the bag and moves the voter into it - assert_eq!( - get_bags(), - vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] - ); - - // increase stake by `rebond`-ing to the level of a pre-existing bag - assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 - // moves the voter to that bag - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); - - // TODO test rebag directly - }); - } - - // #[test] TODO - // fn rebag_head_works() { - // // rebagging the head of a bag results in the old bag having a new head and an overall correct state. - // } -} - mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; @@ -3929,8 +3834,8 @@ mod election_data_provider { #[test] fn targets_2sec_block() { let mut validators = 1000; - while ::WeightInfo::get_npos_targets(validators) < - 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_targets(validators) + < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { validators += 1; } @@ -3947,8 +3852,8 @@ mod election_data_provider { let slashing_spans = validators; let mut nominators = 1000; - while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) < - 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) + < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { nominators += 1; } @@ -4021,7 +3926,7 @@ mod election_data_provider { fn respects_snapshot_len_limits() { ExtBuilder::default().validator_pool(true).build_and_execute(|| { // sum of all validators and nominators who'd be voters. - assert_eq!(VoterList::::decode_len().unwrap(), 5); + assert_eq!(::SortedListProvider::count(), 5); // if limits is less.. assert_eq!(Staking::voters(Some(1)).unwrap().0.len(), 1); @@ -4409,3 +4314,12 @@ mod election_data_provider { }) } } + +mod sorted_list_provider { + #[test] + fn iter_works() { + // TODO: not sure if this is really needed. What should I check? if it returns the correct + // type, then it is correct from my PoV. + todo!() + } +} From 8757ac9caa61c15c4531a236426a91d9bd451783 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 4 Aug 2021 23:14:48 +0200 Subject: [PATCH 106/241] my last changes --- frame/staking/src/mock.rs | 13 ------------- frame/staking/src/pallet/impls.rs | 7 ++----- frame/staking/src/pallet/mod.rs | 4 +--- frame/staking/src/tests.rs | 2 +- 4 files changed, 4 insertions(+), 22 deletions(-) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 9241b67bc14f1..0c5c9b0c3fbc6 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -106,7 +106,6 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, - // VoterBags: pallet_voter_bags::{Pallet, Call, Storage, Event}, } ); @@ -251,18 +250,6 @@ impl onchain::Config for Test { type DataProvider = Staking; } -/// Thresholds used for bags. -const THRESHOLDS: [VoteWeight; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; - -parameter_types! { - pub const VoterBagThresholds: &'static [VoteWeight] = &THRESHOLDS; -} - -// impl pallet_voter_bags::Config for Test { -// type Event = Event; -// type BVoterBagThresholds = VoterBagThresholds; -// } - impl crate::pallet::pallet::Config for Test { const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 22aab837f4c18..a54c5185bba57 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -18,7 +18,7 @@ //! Implementations for the Staking FRAME Pallet. use frame_election_provider_support::{ - data_provider, ElectionProvider, SortedListProvider, Supports, VoteWeight, + data_provider, ElectionDataProvider, ElectionProvider, SortedListProvider, Supports, VoteWeight, }; use frame_support::{ pallet_prelude::*, @@ -778,10 +778,7 @@ impl Pallet { } } -impl - frame_election_provider_support::ElectionDataProvider> - for Pallet -{ +impl ElectionDataProvider> for Pallet { const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; fn desired_targets() -> data_provider::Result<(u32, Weight)> { Ok((Self::validator_count(), ::DbWeight::get().reads(1))) diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index cfa2ce1a3bdfe..41c74ed930b54 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -545,7 +545,7 @@ pub mod pallet { assert_eq!( T::SortedListProvider::count(), CounterForNominators::::get(), - "not all genesis stakers were inserted into bags, something is wrong." + "not all genesis stakers were inserted into sorted list provider, something is wrong." ); } } @@ -587,8 +587,6 @@ pub mod pallet { Chilled(T::AccountId), /// The stakers' rewards are getting paid. \[era_index, validator_stash\] PayoutStarted(EraIndex, T::AccountId), - /// Moved an account from one bag to another. \[who, from, to\]. - Rebagged(T::AccountId, VoteWeight, VoteWeight), } #[pallet::error] diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 29f8e415f6ffb..e31aaff64b97d 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -550,8 +550,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 800, own: 1000, others: vec![ - IndividualExposure { who: 1, value: 400 }, IndividualExposure { who: 3, value: 400 }, + IndividualExposure { who: 1, value: 400 }, ] }, ); From 3ff99e1db93a94a79cfa7ecad22e7559ddfecc95 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 14:47:20 -0700 Subject: [PATCH 107/241] all bags-list test work; fmt --- frame/bags-list/src/list/mod.rs | 14 ++- frame/bags-list/src/list/tests.rs | 185 ++++++++++++++++------------- frame/bags-list/src/tests.rs | 76 +++++------- frame/staking/src/benchmarking.rs | 2 +- frame/staking/src/lib.rs | 2 +- frame/staking/src/mock.rs | 9 +- frame/staking/src/pallet/impls.rs | 25 ++-- frame/staking/src/pallet/mod.rs | 12 +- frame/staking/src/testing_utils.rs | 4 +- frame/staking/src/tests.rs | 20 ++-- 10 files changed, 179 insertions(+), 170 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 09121d902d4df..ba08ef445687b 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -116,7 +116,7 @@ impl List { if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's // no point iterating through bag 10 twice. - continue; + continue } if let Some(bag) = Bag::::get(affected_bag) { @@ -127,7 +127,7 @@ impl List { // a removed bag means that all members of that bag must be rebagged for removed_bag in old_set.difference(&new_set).copied() { if !affected_old_bags.insert(removed_bag) { - continue; + continue } if let Some(bag) = Bag::::get(removed_bag) { @@ -222,6 +222,7 @@ impl List { pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { // TODO: can check if this voter exists as a node by checking if `voter` exists in the nodes // map and return early if it does. + // OR create a can_insert let bag_weight = notional_bag_for::(weight); crate::log!( @@ -336,7 +337,7 @@ impl List { /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `List`. - pub(crate) fn sanity_check() -> Result<(), String> { + pub(crate) fn sanity_check() -> Result<(), &'static str> { use frame_support::ensure; let mut seen_in_list = BTreeSet::new(); ensure!( @@ -348,7 +349,10 @@ impl List { let stored_count = crate::CounterForVoters::::get(); ensure!( iter_count == stored_count, - format!("iter_count {} != stored_count {}", iter_count, stored_count) + // TODO @kian how strongly do you feel about this String? + // afaict its non-trivial to get this work with compile flags etc. + // format!("iter_count {} != stored_count {}", iter_count, stored_count) + "iter_count != stored_count", ); let _ = T::BagThresholds::get() @@ -464,7 +468,7 @@ impl Bag { // this should never happen, but this check prevents a worst case infinite loop debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return; + return }; } diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 554493a489c82..aab4befdc2d3d 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -451,134 +451,155 @@ mod bags { fn remove_node_happy_paths_works() { ExtBuilder::default() .add_ids(vec![ - (51, 1_000), - (61, 1_000), - (71, 10), - (81, 10), - (91, 2_000), - (161, 2_000), - (171, 2_000), - (181, 2_000), - (191, 2_000), + (11, 10), + (12, 10), + (13, 1_000), + (14, 1_000), + (15, 2_000), + (16, 2_000), + (17, 2_000), + (18, 2_000), + (19, 2_000), ]) - .build_and_execute(|| { + .build_and_execute_no_post_check(|| { let mut bag_10 = Bag::::get(10).unwrap(); let mut bag_1000 = Bag::::get(1_000).unwrap(); let mut bag_2000 = Bag::::get(2_000).unwrap(); // given - assert_eq!(bag_as_ids(&bag_10), vec![31, 71, 81]); - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 51, 61]); - assert_eq!(bag_as_ids(&bag_2000), vec![91, 161, 171, 181, 191]); - - // remove node that is not pointing at head or tail - let node_101 = Node::::get(bag_1000.bag_upper, &101).unwrap(); - let node_101_pre_remove = node_101.clone(); - bag_1000.remove_node(&node_101); - assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 51, 61]); + assert_eq!(bag_as_ids(&bag_10), vec![1, 11, 12]); + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 13, 14]); + assert_eq!(bag_as_ids(&bag_2000), vec![15, 16, 17, 18, 19]); + + // when removing a node that is not pointing at the head or tail + let node_4 = Node::::get(bag_1000.bag_upper, &4).unwrap(); + let node_4_pre_remove = node_4.clone(); + bag_1000.remove_node(&node_4); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 13, 14]); assert_ok!(bag_1000.sanity_check()); - // node isn't mutated when its removed - assert_eq!(node_101, node_101_pre_remove); + // and the node isn't mutated when its removed + assert_eq!(node_4, node_4_pre_remove); - // remove head when its not pointing at tail - let node_11 = Node::::get(bag_1000.bag_upper, &11).unwrap(); - bag_1000.remove_node(&node_11); - assert_eq!(bag_as_ids(&bag_1000), vec![21, 51, 61]); + // when removing a head that is not pointing at the tail + let node_2 = Node::::get(bag_1000.bag_upper, &2).unwrap(); + bag_1000.remove_node(&node_2); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![3, 13, 14]); assert_ok!(bag_1000.sanity_check()); - // remove tail when its not pointing at head - let node_61 = Node::::get(bag_1000.bag_upper, &61).unwrap(); - bag_1000.remove_node(&node_61); - assert_eq!(bag_as_ids(&bag_1000), vec![21, 51]); + // when removing a tail that is not pointing at the head + let node_14 = Node::::get(bag_1000.bag_upper, &14).unwrap(); + bag_1000.remove_node(&node_14); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![3, 13]); assert_ok!(bag_1000.sanity_check()); - // remove tail when its pointing at head - let node_51 = Node::::get(bag_1000.bag_upper, &51).unwrap(); - bag_1000.remove_node(&node_51); - assert_eq!(bag_as_ids(&bag_1000), vec![21]); + // when removing a tail that is pointing at the head + let node_13 = Node::::get(bag_1000.bag_upper, &13).unwrap(); + bag_1000.remove_node(&node_13); + + // then + assert_eq!(bag_as_ids(&bag_1000), vec![3]); assert_ok!(bag_1000.sanity_check()); - // remove node that is head & tail - let node_21 = Node::::get(bag_1000.bag_upper, &21).unwrap(); - bag_1000.remove_node(&node_21); - bag_1000.put(); // put into storage so get returns the updated bag + // when removing a node that is both the head & tail + let node_3 = Node::::get(bag_1000.bag_upper, &3).unwrap(); + bag_1000.remove_node(&node_3); + bag_1000.put(); // put into storage so `get` returns the updated bag + + // then assert_eq!(Bag::::get(1_000), None); - // remove node that is pointing at head and tail - let node_71 = Node::::get(bag_10.bag_upper, &71).unwrap(); - bag_10.remove_node(&node_71); - assert_eq!(bag_as_ids(&bag_10), vec![31, 81]); + // when removing a node that is pointing at both the head & tail + let node_11 = Node::::get(bag_10.bag_upper, &11).unwrap(); + bag_10.remove_node(&node_11); + + // then + assert_eq!(bag_as_ids(&bag_10), vec![1, 12]); assert_ok!(bag_10.sanity_check()); - // remove head when pointing at tail - let node_31 = Node::::get(bag_10.bag_upper, &31).unwrap(); - bag_10.remove_node(&node_31); - assert_eq!(bag_as_ids(&bag_10), vec![81]); + // when removing a head that is pointing at the tail + let node_1 = Node::::get(bag_10.bag_upper, &1).unwrap(); + bag_10.remove_node(&node_1); + + // then + assert_eq!(bag_as_ids(&bag_10), vec![12]); assert_ok!(bag_10.sanity_check()); - bag_10.put(); // since we updated the bag's head/tail, we need to write this storage + // and since we updated the bag's head/tail, we need to write this storage so we + // can correctly `get` it again in later checks + bag_10.put(); - // remove node that is pointing at head, but not tail - let node_161 = Node::::get(bag_2000.bag_upper, &161).unwrap(); - bag_2000.remove_node(&node_161); - assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 181, 191]); + // when removing a node that is pointing at the head but not the tail + let node_16 = Node::::get(bag_2000.bag_upper, &16).unwrap(); + bag_2000.remove_node(&node_16); + + // then + assert_eq!(bag_as_ids(&bag_2000), vec![15, 17, 18, 19]); assert_ok!(bag_2000.sanity_check()); - // remove node that is pointing at tail, but not head - let node_181 = Node::::get(bag_2000.bag_upper, &181).unwrap(); - bag_2000.remove_node(&node_181); - assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 191]); + // when removing a node that is pointing at tail, but not head + let node_18 = Node::::get(bag_2000.bag_upper, &18).unwrap(); + bag_2000.remove_node(&node_18); + + // then + assert_eq!(bag_as_ids(&bag_2000), vec![15, 17, 19]); assert_ok!(bag_2000.sanity_check()); - // state of all bags is as expected - assert_eq!(get_bags(), vec![(10, vec![81]), (2_000, vec![91, 171, 191])]); + // finally, when read from storage the state of all bags is as expected + assert_eq!(get_bags(), vec![(10, vec![12]), (2_000, vec![15, 17, 19])]); }); } #[test] fn remove_node_bad_paths_documented() { - ExtBuilder::default().build_and_execute(|| { - // removing a node that is in the bag but has the wrong upper works. - - let bad_upper_node_11 = Node:: { - id: 11, + ExtBuilder::default().build_and_execute_no_post_check(|| { + let bad_upper_node_2 = Node:: { + id: 2, prev: None, - next: Some(21), + next: Some(3), bag_upper: 10, // should be 1_000 }; let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.remove_node(&bad_upper_node_11); + + // when removing a node that is in the bag but has the wrong upper + bag_1000.remove_node(&bad_upper_node_2); bag_1000.put(); - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![21, 101])]); + // then the node is no longer in any bags + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + // .. and the bag it was removed from let bag_1000 = Bag::::get(1_000).unwrap(); + // is sane assert_ok!(bag_1000.sanity_check()); - assert_eq!(bag_1000.head, Some(21)); - assert_eq!(bag_1000.tail, Some(101)); + // and has the correct head and tail + assert_eq!(bag_1000.head, Some(3)); + assert_eq!(bag_1000.tail, Some(4)); }); - ExtBuilder::default().build_and_execute(|| { - // removing a node that is in another bag, will mess up the - // other bag. + // Removing a node that is in another bag, will mess up the other bag. + ExtBuilder::default().build_and_execute_no_post_check(|| { + // given a tail node is in bag 1_000 + let node_4 = Node::::get(1_000, &4).unwrap(); - let node_101 = Node::::get(1_000, &101).unwrap(); + // when we remove it from bag 10 let mut bag_10 = Bag::::get(10).unwrap(); - bag_10.remove_node(&node_101); // node_101 is in bag 1_000 + bag_10.remove_node(&node_4); // node_101 is in bag 1_000 bag_10.put(); - // the node was removed from its actual bag, bag_1000. - assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21])]); - - // the bag remove was called on is ok. + // then bag remove was called on is ok, let bag_10 = Bag::::get(10).unwrap(); - assert_eq!(bag_10.tail, Some(31)); - assert_eq!(bag_10.head, Some(31)); + assert_eq!(bag_10.tail, Some(1)); + assert_eq!(bag_10.head, Some(1)); // but the bag that the node belonged to is in an invalid state let bag_1000 = Bag::::get(1_000).unwrap(); // because it still has the removed node as its tail. - assert_eq!(bag_1000.tail, Some(101)); - assert_eq!(bag_1000.head, Some(11)); - assert_ok!(bag_1000.sanity_check()); + assert_eq!(bag_1000.tail, Some(4)); + assert_eq!(bag_1000.head, Some(2)); }); } } @@ -594,10 +615,12 @@ mod node { // given assert_eq!(node.bag_upper, 10); - // then + // then within bag 10 its not misplaced, assert!(!node.is_misplaced(0)); assert!(!node.is_misplaced(9)); assert!(!node.is_misplaced(10)); + + // and out of bag 10 it is misplaced assert!(node.is_misplaced(11)); }); } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 5140b84a0ea56..84e2fbcf714d2 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -14,27 +14,32 @@ mod extrinsics { // given assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1000, vec![2, 3, 4])]); - // increase vote weight and implicitly rebag to the level of non-existent bag + // when increasing vote weight to the level of non-existent bag NextVoteWeight::set(2000); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); + + // then a new bag is created and the voter moves into it assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); - // decrease weight within the range of the current bag + // when decreasing weight within the range of the current bag NextVoteWeight::set(1001); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); - // does not change bags + + // then the voter does not move assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); - // reduce weight to the level of a non-existent bag + // when reducing weight to the level of a non-existent bag NextVoteWeight::set(30); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); - // creates the bag and moves the voter into it + + // then a new bag is created and the voter moves into it assert_eq!(get_bags(), vec![(10, vec![1]), (30, vec![42]), (1000, vec![2, 3, 4])]); - // increase weight to a pre-existing bag + // when increasing weight to the level of a pre-existing bag NextVoteWeight::set(500); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); - // moves the voter to that bag + + // then the voter moves into that bag assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 42])]); }); } @@ -62,7 +67,6 @@ mod extrinsics { assert_eq!(get_bags(), vec![(10, vec![1, 4, 3]), (1000, vec![2])]); assert_eq!(Bag::::get(10).unwrap(), Bag::new(Some(1), Some(3), 10)); - // TODO: This might be wrong, should be None. assert_eq!(Bag::::get(1000).unwrap(), Bag::new(Some(2), Some(2), 1000)); assert_eq!(get_voter_list_as_ids(), vec![2u32, 1, 4, 3]); @@ -71,6 +75,7 @@ mod extrinsics { // then assert_eq!(get_bags(), vec![(10, vec![1, 4, 3, 2])]); + assert_eq!(Bag::::get(1000), None); }); } @@ -120,19 +125,23 @@ mod sorted_list_provider { #[test] fn count_works() { ExtBuilder::default().build_and_execute(|| { + // given assert_eq!(>::count(), 4); + // when inserting >::on_insert(201, 0); + // then the count goes up assert_eq!(>::count(), 5); + // when removing >::on_remove(&201); + // then the count goes down assert_eq!(>::count(), 4); - >::on_remove(&1); - assert_eq!(>::count(), 3); - - >::on_remove(&2); - assert_eq!(>::count(), 2); + // when updating + >::on_update(&201, VoteWeight::MAX); + // then the count stays the same + assert_eq!(>::count(), 4); }); } @@ -140,28 +149,25 @@ mod sorted_list_provider { fn on_insert_works() { ExtBuilder::default().build_and_execute(|| { // when - >::on_insert(71, 1_000); + >::on_insert(6, 1_000); // then - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 71])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6])]); assert_eq!( >::iter().collect::>(), - vec![2, 3, 4, 71, 1] + vec![2, 3, 4, 6, 1] ); assert_eq!(>::count(), 5); assert_ok!(List::::sanity_check()); // when - List::::insert(81, 1_001); + List::::insert(7, 1_001); // then - assert_eq!( - get_bags(), - vec![(10, vec![1]), (1_000, vec![2, 3, 4, 71]), (2000, vec![81])] - ); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2000, vec![7])]); assert_eq!( >::iter().collect::>(), - vec![81, 2, 3, 4, 71, 1] + vec![7, 2, 3, 4, 6, 1] ); assert_eq!(>::count(), 6); }) @@ -176,23 +182,13 @@ mod sorted_list_provider { // update weight to the level of non-existent bag >::on_update(&42, 2_000); assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); - assert_eq!( - >::iter().collect::>(), - vec![42, 2, 3, 4, 1] - ); assert_eq!(>::count(), 5); - assert_ok!(List::::sanity_check()); // decrease weight within the range of the current bag >::on_update(&42, 1_001); // does not change bags assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); - assert_eq!( - >::iter().collect::>(), - vec![42, 2, 3, 4, 1] - ); assert_eq!(>::count(), 5); - assert_ok!(List::::sanity_check()); // increase weight to the level of a non-existent bag >::on_update(&42, VoteWeight::MAX); @@ -201,20 +197,11 @@ mod sorted_list_provider { get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (VoteWeight::MAX, vec![42])] ); - assert_eq!( - >::iter().collect::>(), - vec![42, 2, 3, 4, 1] - ); assert_eq!(>::count(), 5); - assert_ok!(List::::sanity_check()); // decrease the weight to a pre-existing bag >::on_update(&42, 999); assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); - assert_eq!( - >::iter().collect::>(), - vec![2, 3, 4, 42, 1] - ); assert_eq!(>::count(), 5); }); } @@ -276,17 +263,14 @@ mod sorted_list_provider { // make sure there are no duplicates. ExtBuilder::default().build_and_execute_no_post_check(|| { >::on_insert(2, 10); - assert_eq!(List::::sanity_check(), Err("duplicate identified".to_string())); + assert_eq!(List::::sanity_check(), Err("duplicate identified")); }); // ensure count is in sync with `CounterForVoters`. ExtBuilder::default().build_and_execute_no_post_check(|| { crate::CounterForVoters::::mutate(|counter| *counter += 1); assert_eq!(crate::CounterForVoters::::get(), 5); - assert_eq!( - List::::sanity_check(), - Err("iter_count 4 != stored_count 5".to_string()) - ); + assert_eq!(List::::sanity_check(), Err("iter_count != stored_count")); }); } } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index e63856906b842..53f0f7c8431c5 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -48,7 +48,7 @@ const MAX_SLASHES: u32 = 1000; // read and write operations. fn add_slashing_spans(who: &T::AccountId, spans: u32) { if spans == 0 { - return; + return } // For the first slashing span, we initialize diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index f0ab0e6f7560b..3ddfa4d4d5e7e 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -493,7 +493,7 @@ impl } if unlocking_balance >= value { - break; + break } } diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 9241b67bc14f1..35ffed2698da0 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -18,8 +18,7 @@ //! Test utilities use crate::{self as pallet_staking, *}; -use frame_election_provider_support::onchain; -use frame_election_provider_support::SortedListProvider; +use frame_election_provider_support::{onchain, SortedListProvider}; use frame_support::{ assert_ok, parameter_types, traits::{ @@ -565,7 +564,7 @@ fn check_nominators() { e.others.iter().filter(|e| e.who == nominator).collect::>(); let len = individual.len(); match len { - 0 => { /* not supporting this validator at all. */ } + 0 => { /* not supporting this validator at all. */ }, 1 => sum += individual[0].value, _ => panic!("nominator cannot back a validator more than once."), }; @@ -749,9 +748,9 @@ pub(crate) fn on_offence_in_era( for &(bonded_era, start_session) in bonded_eras.iter() { if bonded_era == era { let _ = Staking::on_offence(offenders, slash_fraction, start_session); - return; + return } else if bonded_era > era { - break; + break } } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 22aab837f4c18..952e6bef082dd 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -136,7 +136,7 @@ impl Pallet { // Nothing to do if they have no reward points. if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()); + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) } // This is the fraction of the total reward that the validator and the @@ -227,9 +227,8 @@ impl Pallet { Self::update_ledger(&controller, &l); r }), - RewardDestination::Account(dest_account) => { - Some(T::Currency::deposit_creating(&dest_account, amount)) - } + RewardDestination::Account(dest_account) => + Some(T::Currency::deposit_creating(&dest_account, amount)), RewardDestination::None => None, } } @@ -257,14 +256,14 @@ impl Pallet { _ => { // Either `Forcing::ForceNone`, // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None; - } + return None + }, } // New era. let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() - && matches!(ForceEra::::get(), Forcing::ForceNew) + if maybe_new_era_validators.is_some() && + matches!(ForceEra::::get(), Forcing::ForceNew) { ForceEra::::put(Forcing::NotForcing); } @@ -445,12 +444,12 @@ impl Pallet { // TODO: this should be simplified #8911 CurrentEra::::put(0); ErasStartSessionIndex::::insert(&0, &start_session_index); - } + }, _ => (), } Self::deposit_event(Event::StakingElectionFailed); - return None; + return None } Self::deposit_event(Event::StakersElected); @@ -816,7 +815,7 @@ impl let target_count = CounterForValidators::::get() as usize; if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big"); + return Err("Target snapshot too big") } let weight = ::DbWeight::get().reads(target_count as u64); @@ -1080,7 +1079,7 @@ where add_db_reads_writes(1, 0); if active_era.is_none() { // This offence need not be re-submitted. - return consumed_weight; + return consumed_weight } active_era.expect("value checked not to be `None`; qed").index }; @@ -1126,7 +1125,7 @@ where // Skip if the validator is invulnerable. if invulnerables.contains(stash) { - continue; + continue } let unapplied = slashing::compute_slash::(slashing::SlashParams { diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index cfa2ce1a3bdfe..74dfb68c6d2c2 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -518,7 +518,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue; + continue } match status { StakerStatus::Validator => { @@ -528,7 +528,7 @@ pub mod pallet { ) { log!(warn, "failed to validate staker at genesis: {:?}.", why); } - } + }, StakerStatus::Nominator(votes) => { if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), @@ -536,7 +536,7 @@ pub mod pallet { ) { log!(warn, "failed to nominate staker at genesis: {:?}.", why); } - } + }, _ => (), }; } @@ -1345,9 +1345,9 @@ pub mod pallet { Self::update_ledger(&controller, &ledger); T::SortedListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. Ok(Some( - 35 * WEIGHT_PER_MICROS - + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) - + T::DbWeight::get().reads_writes(3, 2), + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), ) .into()) } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index c9b39e9c0db13..795c066d09bb3 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -72,7 +72,7 @@ pub fn create_stash_controller( amount, destination, )?; - return Ok((stash, controller)); + return Ok((stash, controller)) } /// Create a stash and controller pair, where the controller is dead, and payouts go to controller. @@ -94,7 +94,7 @@ pub fn create_stash_and_dead_controller( amount, destination, )?; - return Ok((stash, controller)); + return Ok((stash, controller)) } /// create `max` validators. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 29f8e415f6ffb..0727a1504aa7e 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -276,9 +276,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * total_payout_0 * 2 / 3 - + part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * total_payout_0 * 2 / 3 + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -314,9 +314,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) - + part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -3834,8 +3834,8 @@ mod election_data_provider { #[test] fn targets_2sec_block() { let mut validators = 1000; - while ::WeightInfo::get_npos_targets(validators) - < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_targets(validators) < + 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { validators += 1; } @@ -3852,8 +3852,8 @@ mod election_data_provider { let slashing_spans = validators; let mut nominators = 1000; - while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) - < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) < + 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { nominators += 1; } From e7d40fccf8596adf58a2fbf9536824b658be2200 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:10:17 -0700 Subject: [PATCH 108/241] Beautify some tests --- frame/bags-list/src/tests.rs | 64 ++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 84e2fbcf714d2..417a7b3a3f6c4 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -151,24 +151,27 @@ mod sorted_list_provider { // when >::on_insert(6, 1_000); - // then + // then the bags assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6])]); + // and list correctly include the new id, assert_eq!( >::iter().collect::>(), vec![2, 3, 4, 6, 1] ); + // and the count is incremented. assert_eq!(>::count(), 5); - assert_ok!(List::::sanity_check()); // when List::::insert(7, 1_001); - // then + // then the bags assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2000, vec![7])]); + // and list correctly include the new id, assert_eq!( >::iter().collect::>(), vec![7, 2, 3, 4, 6, 1] ); + // and the count is incremented. assert_eq!(>::count(), 6); }) } @@ -178,30 +181,56 @@ mod sorted_list_provider { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])]); + assert_eq!(>::count(), 5); - // update weight to the level of non-existent bag + // when increasing weight to the level of non-existent bag >::on_update(&42, 2_000); + + // then the bag is created with the voter in it, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); - assert_eq!(>::count(), 5); + // and the id position is updated in the list. + assert_eq!( + >::iter().collect::>(), + vec![42, 2, 3, 4, 1] + ); - // decrease weight within the range of the current bag + // when decreasing weight within the range of the current bag >::on_update(&42, 1_001); - // does not change bags + + // then the voter does not change bags, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); - assert_eq!(>::count(), 5); + // or change position in the list. + assert_eq!( + >::iter().collect::>(), + vec![42, 2, 3, 4, 1] + ); - // increase weight to the level of a non-existent bag + // when increasing weight to the level of a non-existent bag >::on_update(&42, VoteWeight::MAX); - // creates the bag and moves the voter into it + + // the the new bag is created with the voter in it, assert_eq!( get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (VoteWeight::MAX, vec![42])] ); - assert_eq!(>::count(), 5); + // and the id position is updated in the list. + assert_eq!( + >::iter().collect::>(), + vec![42, 2, 3, 4, 1] + ); + + // when decreasing the weight to a pre-existing bag + >::on_update(&42, 1_000); - // decrease the weight to a pre-existing bag - >::on_update(&42, 999); + // then voter is moved to the correct bag (as the last member), assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); + // and the id position is updated in the list. + assert_eq!( + >::iter().collect::>(), + vec![2, 3, 4, 42, 1] + ); + + // since we have only called on_update, the `count` has not changed. assert_eq!(>::count(), 5); }); } @@ -219,12 +248,10 @@ mod sorted_list_provider { ExtBuilder::default().build_and_execute(|| { // when removing a non-existent voter - assert!(!get_voter_list_as_ids().contains(&42)); assert!(!VoterNodes::::contains_key(42)); >::on_remove(&42); // then nothing changes - assert_eq!(get_voter_list_as_ids(), vec![2, 3, 4, 1]); assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // when removing a node from a bag with multiple nodes @@ -235,7 +262,7 @@ mod sorted_list_provider { assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); ensure_left(2, 3); - // when removing a node from a bag with only one node: + // when removing a node from a bag with only one node >::on_remove(&1); // then @@ -243,12 +270,13 @@ mod sorted_list_provider { assert_eq!(get_bags(), vec![(1_000, vec![3, 4])]); ensure_left(1, 2); - // remove remaining voters to make sure storage cleans up as expected + // when removing all remaining voters >::on_remove(&4); assert_eq!(get_voter_list_as_ids(), vec![3]); ensure_left(4, 1); - >::on_remove(&3); + + // then storage is completely cleaned up assert_eq!(get_voter_list_as_ids(), Vec::::new()); ensure_left(3, 0); }); From 93a26f620ae060e1169099b1ca5432bc67a10476 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:36:50 -0700 Subject: [PATCH 109/241] Doc comment for bags-list --- frame/bags-list/src/lib.rs | 27 +++++++++++++++++++++------ frame/bags-list/src/tests.rs | 4 ++-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index a5639b63f72e1..fe75fd82c9c9a 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -15,11 +15,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Implement a data structure for pallet-staking designed for the properties that: +//! A semi-sorted list, where items hold an `id` and `weight`. +//! TODO items will also have a generic `data` field //! -//! - It's efficient to insert or remove a voter -//! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of -//! voters doesn't particularly matter. +//! # Goals +//! +//! The data structure exposed by this pallet aims to be optimized for +//! - insertions and removal +//! - iteration over the top* N items by weight, where the precise ordering of items doesn't +//! particularly matter. +//! +//! # Details +//! +//! - items are kept in bags, which are delineated by their range of weight. (See [`BagThresholds`]) +//! - for iteration bags are chained together from highest to lowest and elements within the bag +//! are iterated from head to tail. +//! - items within a bag are in order of insertion. Thus removing an item and re-inserting it +//! will worsen its position in list iteration; this reduces incentives for some types +//! of spam that involve consistently removing and inserting for better position. Further, +//! ordering granularity is thus dictated by range between each bag threshold. +//! - if an items weight changes to a value no longer within the range of its current bag its +//! position will need to be updated by an external actor with rebag or removal & insertion. use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; use frame_system::ensure_signed; @@ -119,8 +135,6 @@ pub mod pallet { /// migration. #[pallet::constant] type BagThresholds: Get<&'static [VoteWeight]>; - - // TODO Node.id type could be made generic? } /// How many voters are registered. @@ -229,6 +243,7 @@ impl SortedListProvider for Pallet { fn on_remove(voter: &T::AccountId) { List::::remove(voter) } + fn sanity_check() -> Result<(), &'static str> { List::::sanity_check() } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 417a7b3a3f6c4..63377d6fff6a0 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -205,7 +205,7 @@ mod sorted_list_provider { vec![42, 2, 3, 4, 1] ); - // when increasing weight to the level of a non-existent bag + // when increasing weight to the level of a non-existent bag with the max threshold >::on_update(&42, VoteWeight::MAX); // the the new bag is created with the voter in it, @@ -276,7 +276,7 @@ mod sorted_list_provider { ensure_left(4, 1); >::on_remove(&3); - // then storage is completely cleaned up + // then the storage is completely cleaned up assert_eq!(get_voter_list_as_ids(), Vec::::new()); ensure_left(3, 0); }); From 569064a14d6df635440547f5af54378486ac20a6 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 16:13:05 -0700 Subject: [PATCH 110/241] Add insert warnings --- frame/bags-list/src/benchmarks.rs | 121 +++++------------------------- frame/bags-list/src/lib.rs | 14 +++- frame/bags-list/src/list/mod.rs | 55 ++++++++------ 3 files changed, 65 insertions(+), 125 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 7407bee1135b8..ddafa070ed672 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -1,108 +1,27 @@ -// rebag { -// // The most expensive case for this call: -// // -// // - It doesn't matter where in the origin bag the stash lies; the number of reads and -// // writes is constant. We can use the case that the stash is the only one in the origin -// // bag, for simplicity. -// // - The destination bag is not empty, because then we need to update the `next` pointer -// // of the previous node in addition to the work we do otherwise. +//! Benchmarks for the bags list pallet. -// use crate::voter_bags::{Bag, Node}; +use super::*; +use frame_benchmarking::{account, benchmarks}; -// let make_validator = |n: u32, balance_factor: u32| -> Result<(T::AccountId, T::AccountId), &'static str> { -// let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default())?; -// whitelist_account!(controller); -// let prefs = ValidatorPrefs::default(); -// // bond the full value of the stash -// let free_balance = T::Currency::free_balance(&stash); -// Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), free_balance)?; -// Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; +benchmarks! { + rebag { + // The most expensive case for this rebag-ing: + // + // - The node to be rebagged should exist as a non-terminal node in a bag with at least + // 2 other nodes so both its prev and next are nodes that will need be updated + // when it is removed. + // TODO once we optimize for not writing the bag being removed we will have the case + // of removing head/tail from a bag with at least 2 nodes + // - The destination bag is not empty, because then we need to update the `next` pointer + // of the previous node in addition to the work we do otherwise. -// Ok((stash, controller)) -// }; + let origin_bag_thresh = 10; + let dest_bag_thresh = 1_000; -// // Clean up any existing state. -// clear_validators_and_nominators::(); + let origin_head = account("origin") -// let thresholds = T::VoterBagThresholds::get(); + Pallet::::on_insert(42, ) + } +} -// // stash controls the node account -// let bag0_thresh = thresholds[0]; -// let (stash, controller) = make_validator(USER_SEED, bag0_thresh as u32)?; - -// // create another validator with more stake -// let bag2_thresh = thresholds[2]; -// let (other_stash, _) = make_validator(USER_SEED + 1, bag2_thresh as u32)?; - -// // update the stash account's value/weight -// // -// // note that we have to manually update the ledger; if we were to just call -// // `Staking::::bond_extra`, then it would implicitly rebag. We want to separate that step -// // so we can measure it in isolation. -// let other_free_balance = T::Currency::free_balance(&other_stash); -// T::Currency::make_free_balance_be(&stash, other_free_balance); -// let controller = Staking::::bonded(&stash).ok_or("stash had no controller")?; -// let mut ledger = Staking::::ledger(&controller).ok_or("controller had no ledger")?; -// let extra = other_free_balance.checked_sub(&ledger.total).ok_or("balance did not increase")?; -// ledger.total += extra; -// ledger.active += extra; -// Staking::::update_ledger(&controller, &ledger); - -// // verify preconditions -// let weight_of = Staking::::weight_of_fn(); -// let node = Node::::from_id(&stash).ok_or("node not found for stash")?; -// ensure!( -// node.is_misplaced(&weight_of), -// "rebagging only makes sense when a node is misplaced", -// ); -// ensure!( -// { -// let origin_bag = Bag::::get(node.bag_upper).ok_or("origin bag not found")?; -// origin_bag.iter().count() == 1 -// }, -// "stash should be the only node in origin bag", -// ); -// let other_node = Node::::from_id(&other_stash).ok_or("node not found for other_stash")?; -// ensure!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); -// ensure!( -// { -// let destination_bag = Bag::::get(node.proper_bag_for()).ok_or("destination bag not found")?; -// destination_bag.iter().count() != 0 -// }, -// "destination bag should not be empty", -// ); -// drop(node); - -// // caller will call rebag -// let caller = whitelisted_caller(); -// // ensure it's distinct from the other accounts -// ensure!(caller != stash, "caller must not be the same as the stash"); -// ensure!(caller != controller, "caller must not be the same as the controller"); -// }: _(RawOrigin::Signed(caller), stash.clone()) -// verify { -// let node = Node::::from_id(&stash).ok_or("node not found for stash")?; -// ensure!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); -// } - -// regenerate { -// // number of validator intention. -// let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; -// // number of nominator intention. -// let n in (MAX_NOMINATORS / 2) .. MAX_NOMINATORS; - -// clear_validators_and_nominators::(); -// ensure!( -// create_validators_with_nominators_for_era::( -// v, -// n, -// T::MAX_NOMINATIONS as usize, -// true, -// None, -// ).is_ok(), -// "creating validators and nominators failed", -// ); -// }: { -// let migrated = VoterList::::regenerate(); -// ensure!(v + n == migrated, "didn't migrate right amount of voters"); -// } diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index fe75fd82c9c9a..f24ec39774a0e 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -18,10 +18,16 @@ //! A semi-sorted list, where items hold an `id` and `weight`. //! TODO items will also have a generic `data` field //! +//! # ⚠️ WARNING ⚠️ +//! +//! Do not insert an id that already exists in the list; doing so can result in catastrophic failure +//! of your blockchain, including entering into an infinite loop during block execution. +//! //! # Goals //! -//! The data structure exposed by this pallet aims to be optimized for -//! - insertions and removal +//! The data structure exposed by this pallet aims to be optimized for: +//! +//! - insertions and removals //! - iteration over the top* N items by weight, where the precise ordering of items doesn't //! particularly matter. //! @@ -232,6 +238,10 @@ impl SortedListProvider for Pallet { CounterForVoters::::get() } + /// # ⚠️ WARNING ⚠️ + /// + /// Do not insert an id that already exists in the list; doing so can result in catastrophic + /// failure of your blockchain, including entering into an infinite loop during block execution. fn on_insert(voter: T::AccountId, weight: VoteWeight) { List::::insert(voter, weight); } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index ba08ef445687b..b313bdb3ffa5b 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -69,13 +69,29 @@ pub struct List(PhantomData); impl List { /// Remove all data associated with the voter list from storage. - pub fn clear() { + // TODO @kian can we just move this code block to withing `generate` ? + fn clear() { crate::CounterForVoters::::kill(); crate::VoterBagFor::::remove_all(None); crate::VoterBags::::remove_all(None); crate::VoterNodes::::remove_all(None); } + /// Regenerate voter data from the given ids. + /// + /// This is expensive and should only ever be performed during a migration, never during + /// consensus. + /// + /// Returns the number of voters migrated. + #[allow(dead_code)] + pub fn regenerate( + all: impl IntoIterator, + weight_of: Box VoteWeight>, + ) { + Self::clear(); + Self::insert_many(all, weight_of); + } + /// Migrate the voter list from one set of thresholds to another. /// /// This should only be called as part of an intentional migration; it's fairly expensive. @@ -166,21 +182,6 @@ impl List { num_affected } - /// Regenerate voter data from the `Nominators` and `Validators` storage items. - /// - /// This is expensive and should only ever be performed during a migration, never during - /// consensus. - /// - /// Returns the number of voters migrated. - #[allow(dead_code)] - pub fn regenerate( - all: impl IntoIterator, - weight_of: Box VoteWeight>, - ) { - Self::clear(); - Self::insert_many(all, weight_of); - } - /// Iterate over all nodes in all bags in the voter list. /// /// Full iteration can be expensive; it's recommended to limit the number of items with @@ -204,9 +205,15 @@ impl List { iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } - /// Insert several voters into the appropriate bags in the voter list. + /// Insert several voters into the appropriate bags in the voter list. Does not check for + /// duplicates. /// /// This is more efficient than repeated calls to `Self::insert`. + /// + /// # ⚠️ WARNING ⚠️ + /// + /// Do not insert an id that already exists in the list; doing so can result in catastrophic + /// failure of your blockchain, including entering into an infinite loop during block execution. fn insert_many( voters: impl IntoIterator, weight_of: impl Fn(&T::AccountId) -> VoteWeight, @@ -217,8 +224,12 @@ impl List { }); } - /// Insert a new voter into the appropriate bag in the voter list. - // WARNING: This does not check if the inserted AccountId is duplicate with others in any bag. + /// Insert a new voter into the appropriate bag in the voter list. Does not check for duplicates. + /// + /// # ⚠️ WARNING ⚠️ + /// + /// Do not insert an id that already exists in the list; doing so can result in catastrophic + /// failure of your blockchain, including entering into an infinite loop during block execution. pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { // TODO: can check if this voter exists as a node by checking if `voter` exists in the nodes // map and return early if it does. @@ -244,14 +255,14 @@ impl List { } /// Remove a voter (by id) from the voter list. - pub fn remove(voter: &T::AccountId) { + pub(crate) fn remove(voter: &T::AccountId) { Self::remove_many(sp_std::iter::once(voter)); } /// Remove many voters (by id) from the voter list. /// /// This is more efficient than repeated calls to `Self::remove`. - pub fn remove_many<'a>(voters: impl IntoIterator) { + fn remove_many<'a>(voters: impl IntoIterator) { let mut bags = BTreeMap::new(); let mut count = 0; @@ -292,7 +303,7 @@ impl List { /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient /// to call [`self.remove_many`] followed by [`self.insert_many`]. - pub fn update_position_for( + pub(crate) fn update_position_for( mut node: Node, new_weight: VoteWeight, ) -> Option<(VoteWeight, VoteWeight)> { From c128ffdfe681edb4eda241696d102f4c337bd8ac Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 18:07:03 -0700 Subject: [PATCH 111/241] Setup initial benchmark --- frame/bags-list/src/benchmarks.rs | 94 ++++++++++++++++++++++++++++--- frame/bags-list/src/lib.rs | 3 + frame/bags-list/src/list/mod.rs | 3 +- frame/bags-list/src/mock.rs | 1 + 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index ddafa070ed672..37cbd2f2f6097 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -1,12 +1,29 @@ //! Benchmarks for the bags list pallet. use super::*; -use frame_benchmarking::{account, benchmarks}; +use crate::list::{Bag, List}; +use frame_benchmarking::{account, whitelisted_caller}; +use frame_support::traits::Get; +use frame_system::RawOrigin; +// use frame_election_provider_support::VoteWeight; +fn get_bags() -> Vec<(VoteWeight, Vec)> { + T::BagThresholds::get() + .into_iter() + .filter_map(|t| { + Bag::::get(*t) + .map(|bag| (*t, bag.iter().map(|n| n.id().clone()).collect::>())) + }) + .collect::>() +} + +fn get_list_as_ids() -> Vec { + List::::iter().map(|n| n.id().clone()).collect::>() +} -benchmarks! { - rebag { - // The most expensive case for this rebag-ing: +frame_benchmarking::benchmarks! { + rebag_middle { + // An expensive case for rebag-ing: // // - The node to be rebagged should exist as a non-terminal node in a bag with at least // 2 other nodes so both its prev and next are nodes that will need be updated @@ -16,12 +33,73 @@ benchmarks! { // - The destination bag is not empty, because then we need to update the `next` pointer // of the previous node in addition to the work we do otherwise. - let origin_bag_thresh = 10; - let dest_bag_thresh = 1_000; + // clear any pre-existing storage. + List::::clear(); + + // define our origin and destination thresholds. + let origin_bag_thresh = T::BagThresholds::get()[0]; + let dest_bag_thresh = T::BagThresholds::get()[1]; - let origin_head = account("origin") + // seed items in the origin bag. + let origin_head: T::AccountId = account("origin_head", 0, 0); + List::::insert(origin_head.clone(), origin_bag_thresh); - Pallet::::on_insert(42, ) + let origin_middle: T::AccountId = account("origin_middle", 0, 0); + List::::insert(origin_middle.clone(), origin_bag_thresh); + + let origin_tail: T::AccountId = account("origin_tail", 0, 0); + List::::insert(origin_tail.clone(), origin_bag_thresh); + + // seed items in the destination bag. + let dest_head: T::AccountId = account("dest_head", 0, 0); + List::::insert(dest_head.clone(), dest_bag_thresh); + + // check that the list + assert_eq!( + get_list_as_ids::(), + vec![dest_head.clone(), origin_head.clone(), origin_middle.clone(), origin_tail.clone()] + ); + // and the ags are in the expected state after insertions. + assert_eq!( + get_bags::(), + vec![(origin_bag_thresh, vec![origin_head.clone(), origin_middle.clone(), origin_tail.clone()]), (dest_bag_thresh, vec![dest_head.clone()])] + ); + }: { + // TODO need to figure out how to mock a return value for VoteWeightProvider::vote_weight + // and then call the real rebag extrinsic + Pallet::::do_rebag(&origin_middle, dest_bag_thresh) + } + verify { + // check that the list + assert_eq!( + List::::iter().map(|n| n.id().clone()).collect::>(), + vec![dest_head.clone(), origin_middle.clone(), origin_head.clone(), origin_tail.clone()] + ); + // and the bags have updated as expected. + assert_eq!( + get_bags::(), + vec![(origin_bag_thresh, vec![origin_head, origin_tail]), (dest_bag_thresh, vec![dest_head, origin_middle])] + ); } + + // rebag_head { TODO once we have optimized so we only write the origin bag if head tail was updated + // An expensive case for rebagging + // + // - the node being rebagged is a head node, so origin bag must be updated with a new head + // and second node in the bag must have its prev pointer updated. + // - The destination bag is not empty, because then we need to update the `next` pointer + // of the previous node in addition to the work we do otherwise. + + // } _(Origin::signed(white_listed_caller()), origin_head) + // verify { + + // } } + +use frame_benchmarking::impl_benchmark_test_suite; +impl_benchmark_test_suite!( + Pallet, + crate::mock::ext_builder::ExtBuilder::default().build(), + crate::mock::Runtime, +); diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index f24ec39774a0e..a0465a0837be9 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -46,7 +46,10 @@ use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; use frame_system::ensure_signed; +#[cfg(feature = "runtime-benchmarks")] +mod benchmarks; mod list; +// #[cfg(any(feature = "runtime-benchmarks", test))] #[cfg(test)] mod mock; #[cfg(test)] diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index b313bdb3ffa5b..0786ce16fdf14 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -69,8 +69,7 @@ pub struct List(PhantomData); impl List { /// Remove all data associated with the voter list from storage. - // TODO @kian can we just move this code block to withing `generate` ? - fn clear() { + pub(crate) fn clear() { crate::CounterForVoters::::kill(); crate::VoterBagFor::::remove_all(None); crate::VoterBags::::remove_all(None); diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 9b295bde5821d..3f3f9a0976acd 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -134,6 +134,7 @@ pub(crate) mod test_utils { bag.iter().map(|n| *n.id()).collect::>() } + // TODO change this to get_list_as_ids pub(crate) fn get_voter_list_as_ids() -> Vec { List::::iter().map(|n| *n.id()).collect::>() } From d093a72fdccb1c5a0d73a3897e11c12b02ecfecb Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 18:18:55 -0700 Subject: [PATCH 112/241] Wire up WeightInfo --- frame/bags-list/src/benchmarks.rs | 6 +++--- frame/bags-list/src/lib.rs | 4 ++-- frame/bags-list/src/mock.rs | 1 + frame/bags-list/src/weights.rs | 26 +++++++++++++++++++++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 37cbd2f2f6097..ba3b113d03613 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -2,10 +2,10 @@ use super::*; use crate::list::{Bag, List}; -use frame_benchmarking::{account, whitelisted_caller}; +// use frame_benchmarking::{account, whitelisted_caller}; +use frame_benchmarking::{account}; use frame_support::traits::Get; -use frame_system::RawOrigin; -// use frame_election_provider_support::VoteWeight; +// use frame_system::RawOrigin; fn get_bags() -> Vec<(VoteWeight, Vec)> { T::BagThresholds::get() diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index a0465a0837be9..52edd43ec7d41 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -90,7 +90,7 @@ pub mod pallet { type Event: From> + IsType<::Event>; /// Weight information for extrinsics in this pallet. - // type WeightInfo: WeightInfo; + type WeightInfo: weights::WeightInfo; // Something that implements `trait StakingVoteWeight`. type VoteWeightProvider: VoteWeightProvider; @@ -189,7 +189,7 @@ pub mod pallet { /// /// Anyone can call this function about any stash. // #[pallet::weight(T::WeightInfo::rebag())] - #[pallet::weight(123456789)] // TODO + #[pallet::weight(T::WeightInfo::rebag_middle())] pub fn rebag(origin: OriginFor, account: T::AccountId) -> DispatchResult { ensure_signed(origin)?; let weight = T::VoteWeightProvider::vote_weight(&account); diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 3f3f9a0976acd..58f3a075283cb 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -51,6 +51,7 @@ parameter_types! { impl crate::Config for Runtime { type Event = Event; + type WeightInfo = (); type BagThresholds = BagThresholds; type VoteWeightProvider = StakingMock; } diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 5afdff89ab389..f358005853b52 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -1 +1,25 @@ -pub trait WeightInfo {} +use frame_support::pallet_prelude::Weight; +use frame_support::weights::constants::RocksDbWeight; + +pub trait WeightInfo { + fn rebag_middle() -> Weight; +} + +pub struct SubstrateWeight(std::marker::PhantomData); +impl WeightInfo for SubstrateWeight { + fn rebag_middle() -> Weight { + // FAKE numbers + (84_501_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + } +} + +impl WeightInfo for () { + fn rebag_middle() -> Weight { + // FAKE numbers + (84_501_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + } +} \ No newline at end of file From 5758bd336d0aa8a928e93a8606e040d445e16b62 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 19:45:26 -0700 Subject: [PATCH 113/241] is_terminal wip; everything broken! --- frame/bags-list/src/benchmarks.rs | 3 +- frame/bags-list/src/list/mod.rs | 125 +++++++++++++++++++++++------- frame/bags-list/src/list/tests.rs | 83 +++++++++++++------- frame/bags-list/src/mock.rs | 2 +- frame/bags-list/src/tests.rs | 10 +-- frame/bags-list/src/weights.rs | 5 +- 6 files changed, 164 insertions(+), 64 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index ba3b113d03613..95abd48abec8c 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -3,7 +3,7 @@ use super::*; use crate::list::{Bag, List}; // use frame_benchmarking::{account, whitelisted_caller}; -use frame_benchmarking::{account}; +use frame_benchmarking::account; use frame_support::traits::Get; // use frame_system::RawOrigin; @@ -96,7 +96,6 @@ frame_benchmarking::benchmarks! { // } } - use frame_benchmarking::impl_benchmark_test_suite; impl_benchmark_test_suite!( Pallet, diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 0786ce16fdf14..e56b83d86a489 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -185,7 +185,7 @@ impl List { /// /// Full iteration can be expensive; it's recommended to limit the number of items with /// `.take(n)`. - pub fn iter() -> impl Iterator> { + pub(crate) fn iter() -> impl Iterator> { // We need a touch of special handling here: because we permit `T::BagThresholds` to // omit the final bound, we need to ensure that we explicitly include that threshold in the // list. @@ -245,7 +245,10 @@ impl List { let mut bag = Bag::::get_or_make(bag_weight); bag.insert(voter); - // TODO: why are we writing to the bag of the voter is neither head or tail???? + // new inserts are always the tail, so we must write the bag. + // TODO: @kianenigma maybe an optimization is to make the tail actually be a dummy node in storage + // then we could just update the tail dummy to point at the inserted node.. but then we have + // to write the tail dummy to storage... which doesn't seem any better bag.put(); crate::CounterForVoters::::mutate(|prev_count| { @@ -262,7 +265,7 @@ impl List { /// /// This is more efficient than repeated calls to `Self::remove`. fn remove_many<'a>(voters: impl IntoIterator) { - let mut bags = BTreeMap::new(); + let mut bags = BTreeMap::, bool)>::new(); let mut count = 0; for voter_id in voters.into_iter() { @@ -272,19 +275,31 @@ impl List { }; count += 1; - // clear the bag head/tail pointers as necessary - let bag = bags - .entry(node.bag_upper) - .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); - bag.remove_node(&node); + // check if node.is_terminal + + if !node.is_terminal { + // this node is not a head or a tail and thus the bag does not need to be updated + node.excise() + } else { + // this node is a head or tail, so the bag needs to be updated + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| (Bag::::get_or_make(node.bag_upper), false)); + if bag.0.remove_node(&node) { + // if the bag head or tail was updated, mark that the bag should be put. + bag.1 = true; + } + } // now get rid of the node itself crate::VoterNodes::::remove(voter_id); crate::VoterBagFor::::remove(voter_id); } - for (_, bag) in bags { - bag.put(); + for (_, (bag, should_put)) in bags { + if should_put { + bag.put(); + } } crate::CounterForVoters::::mutate(|prev_count| { @@ -314,9 +329,14 @@ impl List { // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 // clear the old bag head/tail pointers as necessary - if let Some(mut bag) = Bag::::get(node.bag_upper) { - bag.remove_node(&node); - bag.put(); + if !node.is_terminal { + // this node is not a head or a tail, so we do not need to update its current bag. + node.excise(); + } else if let Some(mut bag) = Bag::::get(node.bag_upper) { + if bag.remove_node(&node) { + // only right the bag to storage if the bag head OR tail changed. + bag.put(); + } } else { crate::log!( error, @@ -461,7 +481,13 @@ impl Bag { /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. fn insert(&mut self, id: T::AccountId) { - self.insert_node(Node:: { id, prev: None, next: None, bag_upper: self.bag_upper }); + self.insert_node(Node:: { + id, + prev: None, + next: None, + bag_upper: self.bag_upper, + is_terminal: false, + }); } /// Insert a voter node into this bag. @@ -482,20 +508,22 @@ impl Bag { }; } - // update this node now + // update this node now, treating as the new tail. let id = node.id.clone(); node.prev = self.tail.clone(); node.next = None; + node.is_terminal = true; node.put(); // update the previous tail if let Some(mut old_tail) = self.tail() { old_tail.next = Some(id.clone()); + old_tail.is_terminal = false; old_tail.put(); } self.tail = Some(id.clone()); - // ensure head exist. This is typically only set when the length of the bag is just 1, i.e. + // ensure head exist. This is only set when the length of the bag is just 1, i.e. // if this is the first insertion into the bag. In this case, both head and tail should // point to the same voter node. if self.head.is_none() { @@ -507,7 +535,7 @@ impl Bag { crate::VoterBagFor::::insert(id, self.bag_upper); } - /// Remove a voter node from this bag. + /// Remove a voter node from this bag. Returns true iff the bag's head or tail is updated. /// /// This is private on purpose because it doesn't check whether this bag contains the voter in /// the first place. Generally, use [`List::remove`] instead. @@ -515,25 +543,48 @@ impl Bag { /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` /// to update storage for the bag and `node`. - fn remove_node(&mut self, node: &Node) { - // Update previous node. + // + // TODO, @kianenigma if decide to keep is_terminal then we don't need to return a bool here + // afaict keeping is_terminal is a better optimization because in some case we don't need + // to fetch the bag remove is called on. + fn remove_node(&mut self, node: &Node) -> bool { + let mut removed = false; + + // clear the bag head/tail pointers as necessary + if self.tail.as_ref() == Some(&node.id) { + self.tail = node.prev.clone(); + removed = true; + } + if self.head.as_ref() == Some(&node.id) { + self.head = node.next.clone(); + removed = true; + } + + // set neighbors to point to each other. Note that instead of using node.excise here we + // recreate some of the logic but update the neighboring nodes `is_terminal` in case they + // become a head or tail. if let Some(mut prev) = node.prev() { prev.next = node.next.clone(); + + if self.tail.as_ref() == Some(&prev.id) { + // if prev is the new tail, we mark it as terminal + prev.is_terminal = true; + } + prev.put(); } - // Update next node. if let Some(mut next) = node.next() { next.prev = node.prev.clone(); + + if self.head.as_ref() == Some(&next.id) { + // if next is the new head, we mark it as terminal + next.is_terminal = true; + } + next.put(); } - // clear the bag head/tail pointers as necessary - if self.head.as_ref() == Some(&node.id) { - self.head = node.next.clone(); - } - if self.tail.as_ref() == Some(&node.id) { - self.tail = node.prev.clone(); - } + removed } /// Sanity check this bag. @@ -622,6 +673,22 @@ impl Node { crate::VoterNodes::::insert(self.id.clone(), self); } + /// Update neighboring nodes to point to reach other. + /// + /// Does _not_ update storage, so the user may need to call `self.put`. + fn excise(&self) { + // Update previous node. + if let Some(mut prev) = self.prev() { + prev.next = self.next.clone(); + prev.put(); + } + // Update next self. + if let Some(mut next) = self.next() { + next.prev = self.prev.clone(); + next.put(); + } + } + /// Get the previous node in the bag. fn prev(&self) -> Option> { self.prev.as_ref().and_then(|id| self.in_bag(id)) @@ -637,6 +704,10 @@ impl Node { notional_bag_for::(current_weight) != self.bag_upper } + fn is_terminal(&self) -> bool { + self.head.is_none() || self.tail.is_none() + } + /// Get the underlying voter. pub(crate) fn id(&self) -> &T::AccountId { &self.id diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index aab4befdc2d3d..cb56dd517874c 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -9,7 +9,8 @@ use frame_support::{assert_ok, assert_storage_noop}; fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { // syntactic sugar to create a raw node - let node = |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; + let node = + |id, prev, next, is_terminal| Node:: { id, prev, next, bag_upper: 0, is_terminal }; assert_eq!(CounterForVoters::::get(), 4); assert_eq!(VoterBagFor::::iter().count(), 4); @@ -29,13 +30,13 @@ fn basic_setup_works() { ); assert_eq!(VoterBagFor::::get(2).unwrap(), 1000); - assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3))); + assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3), true)); assert_eq!(VoterBagFor::::get(3).unwrap(), 1000); - assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4))); + assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4), false)); assert_eq!(VoterBagFor::::get(4).unwrap(), 1000); - assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None)); + assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None, true)); assert_eq!(VoterBagFor::::get(1).unwrap(), 10); assert_eq!(VoterNodes::::get(1).unwrap(), node(1, None, None)); @@ -110,7 +111,7 @@ mod voter_list { // then assert_eq!( - get_voter_list_as_ids(), + get_list_as_ids(), vec![ 5, 6, // best bag 2, 3, 4, // middle bag @@ -123,7 +124,7 @@ mod voter_list { // then assert_eq!( - get_voter_list_as_ids(), + get_list_as_ids(), vec![ 5, 6, // best bag 2, 3, 4, // middle bag @@ -168,14 +169,14 @@ mod voter_list { // then assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5])]); - assert_eq!(get_voter_list_as_ids(), vec![2, 3, 4, 5, 1]); + assert_eq!(get_list_as_ids(), vec![2, 3, 4, 5, 1]); // when inserting into a non-existent bag List::::insert(6, 1_001); // then assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5]), (2000, vec![6])]); - assert_eq!(get_voter_list_as_ids(), vec![6, 2, 3, 4, 5, 1]); + assert_eq!(get_list_as_ids(), vec![6, 2, 3, 4, 5, 1]); }); } @@ -197,7 +198,7 @@ mod voter_list { List::::remove(&42); // then nothing changes - assert_eq!(get_voter_list_as_ids(), vec![2, 3, 4, 1]); + assert_eq!(get_list_as_ids(), vec![2, 3, 4, 1]); assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); assert_eq!(CounterForVoters::::get(), 4); @@ -205,7 +206,7 @@ mod voter_list { List::::remove(&2); // then - assert_eq!(get_voter_list_as_ids(), vec![3, 4, 1]); + assert_eq!(get_list_as_ids(), vec![3, 4, 1]); assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![3, 4])]); ensure_left(2, 3); @@ -213,7 +214,7 @@ mod voter_list { List::::remove(&1); // then - assert_eq!(get_voter_list_as_ids(), vec![3, 4]); + assert_eq!(get_list_as_ids(), vec![3, 4]); assert_eq!(get_bags(), vec![(1000, vec![3, 4])]); ensure_left(1, 2); // bag 10 is removed @@ -222,11 +223,11 @@ mod voter_list { // remove remaining voters to make sure storage cleans up as expected List::::remove(&3); ensure_left(3, 1); - assert_eq!(get_voter_list_as_ids(), vec![4]); + assert_eq!(get_list_as_ids(), vec![4]); List::::remove(&4); ensure_left(4, 0); - assert_eq!(get_voter_list_as_ids(), Vec::::new()); + assert_eq!(get_list_as_ids(), Vec::::new()); // bags are deleted via removals assert_eq!(VoterBags::::iter().count(), 0); @@ -327,7 +328,13 @@ mod bags { #[test] fn insert_node_happy_paths_works() { ExtBuilder::default().build_and_execute_no_post_check(|| { - let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; + let node = |id, bag_upper| Node:: { + id, + prev: None, + next: None, + bag_upper, + is_terminal: false, + }; // when inserting into a bag with 1 node let mut bag_10 = Bag::::get(10).unwrap(); @@ -349,15 +356,26 @@ mod bags { assert_eq!(bag_as_ids(&bag_20), vec![62]); // when inserting a node pointing to the accounts not in the bag - let node_61 = - Node:: { id: 61, prev: Some(21), next: Some(101), bag_upper: 20 }; + let node_61 = Node:: { + id: 61, + prev: Some(21), + next: Some(101), + bag_upper: 20, + is_terminal: false, + }; bag_20.insert_node(node_61); // then ids are in order assert_eq!(bag_as_ids(&bag_20), vec![62, 61]); // and when the node is re-fetched all the info is correct assert_eq!( Node::::get(20, &61).unwrap(), - Node:: { id: 61, prev: Some(62), next: None, bag_upper: 20 } + Node:: { + id: 61, + prev: Some(62), + next: None, + bag_upper: 20, + is_terminal: false + } ); // state of all bags is as expected @@ -372,12 +390,18 @@ mod bags { // Document improper ways `insert_node` may be getting used. #[test] fn insert_node_bad_paths_documented() { - let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + let node = |id, prev, next, bag_upper, is_terminal| Node:: { + id, + prev, + next, + bag_upper, + is_terminal, + }; ExtBuilder::default().build_and_execute_no_post_check(|| { // when inserting a node with both prev & next pointing at an account in an incorrect // bag. let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(42, Some(1), Some(1), 0)); + bag_1000.insert_node(node(42, Some(1), Some(1), 0, false)); // then the proper perv and next is set. assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 42]); @@ -385,7 +409,7 @@ mod bags { // and when the node is re-fetched all the info is correct assert_eq!( Node::::get(1_000, &42).unwrap(), - node(42, Some(4), None, bag_1000.bag_upper) + node(42, Some(4), None, bag_1000.bag_upper, true) ); }); @@ -395,18 +419,18 @@ mod bags { assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); // when inserting a node with duplicate id 3 - bag_1000.insert_node(node(3, None, None, bag_1000.bag_upper)); + bag_1000.insert_node(node(3, None, None, bag_1000.bag_upper, false)); // then all the nodes after the duplicate are lost (because it is set as the tail) assert_eq!(bag_as_ids(&bag_1000), vec![2, 3]); // also in the full iteration, 2 and 3 are from the 1000 bag and 1 from bag 10. - assert_eq!(get_voter_list_as_ids(), vec![2, 3, 1]); + assert_eq!(get_list_as_ids(), vec![2, 3, 1]); // and the last accessible node has an **incorrect** prev pointer. // TODO: consider doing a check on insert, it is cheap. assert_eq!( Node::::get(1_000, &3).unwrap(), - node(3, Some(4), None, bag_1000.bag_upper) + node(3, Some(4), None, bag_1000.bag_upper, false) ); }); @@ -414,7 +438,7 @@ mod bags { // when inserting a duplicate id of the head let mut bag_1000 = Bag::::get(1_000).unwrap(); assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); - bag_1000.insert_node(node(2, None, None, 0)); + bag_1000.insert_node(node(2, None, None, 0, false)); // then all nodes after the head are lost assert_eq!(bag_as_ids(&bag_1000), vec![2]); @@ -422,7 +446,7 @@ mod bags { // and the re-fetched node has bad pointers assert_eq!( Node::::get(1_000, &2).unwrap(), - node(2, Some(4), None, bag_1000.bag_upper) + node(2, Some(4), None, bag_1000.bag_upper, true) ); // ^^^ despite being the bags head, it has a prev @@ -435,7 +459,13 @@ mod bags { #[should_panic = "system logic error: inserting a node who has the id of tail"] fn insert_node_duplicate_tail_panics_with_debug_assert() { ExtBuilder::default().build_and_execute(|| { - let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + let node = |id, prev, next, bag_upper| Node:: { + id, + prev, + next, + bag_upper, + is_terminal: false, + }; // given assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])],); @@ -562,6 +592,7 @@ mod bags { prev: None, next: Some(3), bag_upper: 10, // should be 1_000 + is_terminal: false, }; let mut bag_1000 = Bag::::get(1_000).unwrap(); diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 58f3a075283cb..a18c74fa47e58 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -136,7 +136,7 @@ pub(crate) mod test_utils { } // TODO change this to get_list_as_ids - pub(crate) fn get_voter_list_as_ids() -> Vec { + pub(crate) fn get_list_as_ids() -> Vec { List::::iter().map(|n| *n.id()).collect::>() } } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 63377d6fff6a0..61c6797e8db0b 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -68,7 +68,7 @@ mod extrinsics { assert_eq!(Bag::::get(10).unwrap(), Bag::new(Some(1), Some(3), 10)); assert_eq!(Bag::::get(1000).unwrap(), Bag::new(Some(2), Some(2), 1000)); - assert_eq!(get_voter_list_as_ids(), vec![2u32, 1, 4, 3]); + assert_eq!(get_list_as_ids(), vec![2u32, 1, 4, 3]); // when assert_ok!(BagsList::rebag(Origin::signed(0), 2)); @@ -258,7 +258,7 @@ mod sorted_list_provider { >::on_remove(&2); // then - assert_eq!(get_voter_list_as_ids(), vec![3, 4, 1]); + assert_eq!(get_list_as_ids(), vec![3, 4, 1]); assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); ensure_left(2, 3); @@ -266,18 +266,18 @@ mod sorted_list_provider { >::on_remove(&1); // then - assert_eq!(get_voter_list_as_ids(), vec![3, 4]); + assert_eq!(get_list_as_ids(), vec![3, 4]); assert_eq!(get_bags(), vec![(1_000, vec![3, 4])]); ensure_left(1, 2); // when removing all remaining voters >::on_remove(&4); - assert_eq!(get_voter_list_as_ids(), vec![3]); + assert_eq!(get_list_as_ids(), vec![3]); ensure_left(4, 1); >::on_remove(&3); // then the storage is completely cleaned up - assert_eq!(get_voter_list_as_ids(), Vec::::new()); + assert_eq!(get_list_as_ids(), Vec::::new()); ensure_left(3, 0); }); } diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index f358005853b52..284fa27afaa70 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -1,5 +1,4 @@ -use frame_support::pallet_prelude::Weight; -use frame_support::weights::constants::RocksDbWeight; +use frame_support::{pallet_prelude::Weight, weights::constants::RocksDbWeight}; pub trait WeightInfo { fn rebag_middle() -> Weight; @@ -22,4 +21,4 @@ impl WeightInfo for () { .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } -} \ No newline at end of file +} From bf9197ebed160674cd7413435b4937003c27d9d8 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 19:52:13 -0700 Subject: [PATCH 114/241] Is terminal working --- frame/bags-list/src/list/mod.rs | 38 ++++++------------------------- frame/bags-list/src/list/tests.rs | 30 +++++++++--------------- 2 files changed, 18 insertions(+), 50 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index e56b83d86a489..caeffefab85fb 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -277,7 +277,7 @@ impl List { // check if node.is_terminal - if !node.is_terminal { + if !node.is_terminal() { // this node is not a head or a tail and thus the bag does not need to be updated node.excise() } else { @@ -329,7 +329,7 @@ impl List { // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 // clear the old bag head/tail pointers as necessary - if !node.is_terminal { + if !node.is_terminal() { // this node is not a head or a tail, so we do not need to update its current bag. node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { @@ -486,7 +486,6 @@ impl Bag { prev: None, next: None, bag_upper: self.bag_upper, - is_terminal: false, }); } @@ -512,13 +511,11 @@ impl Bag { let id = node.id.clone(); node.prev = self.tail.clone(); node.next = None; - node.is_terminal = true; node.put(); // update the previous tail if let Some(mut old_tail) = self.tail() { old_tail.next = Some(id.clone()); - old_tail.is_terminal = false; old_tail.put(); } self.tail = Some(id.clone()); @@ -548,8 +545,10 @@ impl Bag { // afaict keeping is_terminal is a better optimization because in some case we don't need // to fetch the bag remove is called on. fn remove_node(&mut self, node: &Node) -> bool { - let mut removed = false; + // reassign neigbhoring nodes. + node.excise(); + let mut removed = false; // clear the bag head/tail pointers as necessary if self.tail.as_ref() == Some(&node.id) { self.tail = node.prev.clone(); @@ -560,30 +559,6 @@ impl Bag { removed = true; } - // set neighbors to point to each other. Note that instead of using node.excise here we - // recreate some of the logic but update the neighboring nodes `is_terminal` in case they - // become a head or tail. - if let Some(mut prev) = node.prev() { - prev.next = node.next.clone(); - - if self.tail.as_ref() == Some(&prev.id) { - // if prev is the new tail, we mark it as terminal - prev.is_terminal = true; - } - - prev.put(); - } - if let Some(mut next) = node.next() { - next.prev = node.prev.clone(); - - if self.head.as_ref() == Some(&next.id) { - // if next is the new head, we mark it as terminal - next.is_terminal = true; - } - - next.put(); - } - removed } @@ -704,8 +679,9 @@ impl Node { notional_bag_for::(current_weight) != self.bag_upper } + /// `true` when this voter is a bag head or tail. fn is_terminal(&self) -> bool { - self.head.is_none() || self.tail.is_none() + self.prev.is_none() || self.next.is_none() } /// Get the underlying voter. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index cb56dd517874c..52f63e6cb4d4a 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -10,7 +10,7 @@ fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { // syntactic sugar to create a raw node let node = - |id, prev, next, is_terminal| Node:: { id, prev, next, bag_upper: 0, is_terminal }; + |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; assert_eq!(CounterForVoters::::get(), 4); assert_eq!(VoterBagFor::::iter().count(), 4); @@ -30,13 +30,13 @@ fn basic_setup_works() { ); assert_eq!(VoterBagFor::::get(2).unwrap(), 1000); - assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3), true)); + assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3))); assert_eq!(VoterBagFor::::get(3).unwrap(), 1000); - assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4), false)); + assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4))); assert_eq!(VoterBagFor::::get(4).unwrap(), 1000); - assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None, true)); + assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None)); assert_eq!(VoterBagFor::::get(1).unwrap(), 10); assert_eq!(VoterNodes::::get(1).unwrap(), node(1, None, None)); @@ -283,8 +283,6 @@ mod bags { fn get_works() { ExtBuilder::default().build_and_execute(|| { let check_bag = |bag_upper, head, tail, ids| { - // @zeke TODO: why? - assert_storage_noop!(Bag::::get(bag_upper)); let bag = Bag::::get(bag_upper).unwrap(); let bag_ids = bag.iter().map(|n| *n.id()).collect::>(); @@ -333,7 +331,6 @@ mod bags { prev: None, next: None, bag_upper, - is_terminal: false, }; // when inserting into a bag with 1 node @@ -361,7 +358,6 @@ mod bags { prev: Some(21), next: Some(101), bag_upper: 20, - is_terminal: false, }; bag_20.insert_node(node_61); // then ids are in order @@ -374,7 +370,6 @@ mod bags { prev: Some(62), next: None, bag_upper: 20, - is_terminal: false } ); @@ -390,18 +385,17 @@ mod bags { // Document improper ways `insert_node` may be getting used. #[test] fn insert_node_bad_paths_documented() { - let node = |id, prev, next, bag_upper, is_terminal| Node:: { + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper, - is_terminal, }; ExtBuilder::default().build_and_execute_no_post_check(|| { // when inserting a node with both prev & next pointing at an account in an incorrect // bag. let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(42, Some(1), Some(1), 0, false)); + bag_1000.insert_node(node(42, Some(1), Some(1), 0)); // then the proper perv and next is set. assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 42]); @@ -409,7 +403,7 @@ mod bags { // and when the node is re-fetched all the info is correct assert_eq!( Node::::get(1_000, &42).unwrap(), - node(42, Some(4), None, bag_1000.bag_upper, true) + node(42, Some(4), None, bag_1000.bag_upper) ); }); @@ -419,7 +413,7 @@ mod bags { assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); // when inserting a node with duplicate id 3 - bag_1000.insert_node(node(3, None, None, bag_1000.bag_upper, false)); + bag_1000.insert_node(node(3, None, None, bag_1000.bag_upper)); // then all the nodes after the duplicate are lost (because it is set as the tail) assert_eq!(bag_as_ids(&bag_1000), vec![2, 3]); @@ -430,7 +424,7 @@ mod bags { // TODO: consider doing a check on insert, it is cheap. assert_eq!( Node::::get(1_000, &3).unwrap(), - node(3, Some(4), None, bag_1000.bag_upper, false) + node(3, Some(4), None, bag_1000.bag_upper) ); }); @@ -438,7 +432,7 @@ mod bags { // when inserting a duplicate id of the head let mut bag_1000 = Bag::::get(1_000).unwrap(); assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); - bag_1000.insert_node(node(2, None, None, 0, false)); + bag_1000.insert_node(node(2, None, None, 0)); // then all nodes after the head are lost assert_eq!(bag_as_ids(&bag_1000), vec![2]); @@ -446,7 +440,7 @@ mod bags { // and the re-fetched node has bad pointers assert_eq!( Node::::get(1_000, &2).unwrap(), - node(2, Some(4), None, bag_1000.bag_upper, true) + node(2, Some(4), None, bag_1000.bag_upper) ); // ^^^ despite being the bags head, it has a prev @@ -464,7 +458,6 @@ mod bags { prev, next, bag_upper, - is_terminal: false, }; // given @@ -592,7 +585,6 @@ mod bags { prev: None, next: Some(3), bag_upper: 10, // should be 1_000 - is_terminal: false, }; let mut bag_1000 = Bag::::get(1_000).unwrap(); From 7510ac7800cbc77c521f255b948b940f538906a1 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 19:56:15 -0700 Subject: [PATCH 115/241] add TODOs for remove_node --- frame/bags-list/src/list/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index caeffefab85fb..490f91f34e183 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -285,7 +285,7 @@ impl List { let bag = bags .entry(node.bag_upper) .or_insert_with(|| (Bag::::get_or_make(node.bag_upper), false)); - if bag.0.remove_node(&node) { + if bag.0.remove_node(&node) { // TODO remove node doesn't need this bc is_terminal // if the bag head or tail was updated, mark that the bag should be put. bag.1 = true; } @@ -333,7 +333,7 @@ impl List { // this node is not a head or a tail, so we do not need to update its current bag. node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { - if bag.remove_node(&node) { + if bag.remove_node(&node) { // TODO remove node doesn't need this bc is_terminal // only right the bag to storage if the bag head OR tail changed. bag.put(); } From 316eeb5e39e8e9d93a7395ce115891ccd74b105d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 4 Aug 2021 22:58:22 -0700 Subject: [PATCH 116/241] clean up remoe_node --- frame/bags-list/src/list/mod.rs | 53 ++++++++----------------------- frame/bags-list/src/list/tests.rs | 40 ++++------------------- 2 files changed, 20 insertions(+), 73 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 490f91f34e183..6e5bc25df4ae9 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -246,9 +246,6 @@ impl List { bag.insert(voter); // new inserts are always the tail, so we must write the bag. - // TODO: @kianenigma maybe an optimization is to make the tail actually be a dummy node in storage - // then we could just update the tail dummy to point at the inserted node.. but then we have - // to write the tail dummy to storage... which doesn't seem any better bag.put(); crate::CounterForVoters::::mutate(|prev_count| { @@ -265,7 +262,7 @@ impl List { /// /// This is more efficient than repeated calls to `Self::remove`. fn remove_many<'a>(voters: impl IntoIterator) { - let mut bags = BTreeMap::, bool)>::new(); + let mut bags = BTreeMap::new(); let mut count = 0; for voter_id in voters.into_iter() { @@ -278,17 +275,14 @@ impl List { // check if node.is_terminal if !node.is_terminal() { - // this node is not a head or a tail and thus the bag does not need to be updated + // this node is not a head or a tail and thus the bag does not need to be updated. node.excise() } else { // this node is a head or tail, so the bag needs to be updated let bag = bags .entry(node.bag_upper) - .or_insert_with(|| (Bag::::get_or_make(node.bag_upper), false)); - if bag.0.remove_node(&node) { // TODO remove node doesn't need this bc is_terminal - // if the bag head or tail was updated, mark that the bag should be put. - bag.1 = true; - } + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + bag.remove_node(&node); } // now get rid of the node itself @@ -296,10 +290,8 @@ impl List { crate::VoterBagFor::::remove(voter_id); } - for (_, (bag, should_put)) in bags { - if should_put { - bag.put(); - } + for (_, bag) in bags { + bag.put(); } crate::CounterForVoters::::mutate(|prev_count| { @@ -324,19 +316,14 @@ impl List { node.is_misplaced(new_weight).then(move || { let old_idx = node.bag_upper; - // TODO: there should be a way to move a non-head-tail node to another bag - // with just 1 bag read of the destination bag and zero writes - // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 - // clear the old bag head/tail pointers as necessary if !node.is_terminal() { - // this node is not a head or a tail, so we do not need to update its current bag. + // this node is not a head or a tail, so we can just cut it out of the list. node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { - if bag.remove_node(&node) { // TODO remove node doesn't need this bc is_terminal - // only right the bag to storage if the bag head OR tail changed. - bag.put(); - } + // this is a head or tail, so the bag must be updated. + bag.remove_node(&node); + bag.put(); } else { crate::log!( error, @@ -481,12 +468,7 @@ impl Bag { /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. fn insert(&mut self, id: T::AccountId) { - self.insert_node(Node:: { - id, - prev: None, - next: None, - bag_upper: self.bag_upper, - }); + self.insert_node(Node:: { id, prev: None, next: None, bag_upper: self.bag_upper }); } /// Insert a voter node into this bag. @@ -540,26 +522,17 @@ impl Bag { /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` /// to update storage for the bag and `node`. - // - // TODO, @kianenigma if decide to keep is_terminal then we don't need to return a bool here - // afaict keeping is_terminal is a better optimization because in some case we don't need - // to fetch the bag remove is called on. - fn remove_node(&mut self, node: &Node) -> bool { - // reassign neigbhoring nodes. + fn remove_node(&mut self, node: &Node) { + // reassign neighboring nodes. node.excise(); - let mut removed = false; // clear the bag head/tail pointers as necessary if self.tail.as_ref() == Some(&node.id) { self.tail = node.prev.clone(); - removed = true; } if self.head.as_ref() == Some(&node.id) { self.head = node.next.clone(); - removed = true; } - - removed } /// Sanity check this bag. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 52f63e6cb4d4a..a7f54447e0dda 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -9,8 +9,7 @@ use frame_support::{assert_ok, assert_storage_noop}; fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { // syntactic sugar to create a raw node - let node = - |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; + let node = |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; assert_eq!(CounterForVoters::::get(), 4); assert_eq!(VoterBagFor::::iter().count(), 4); @@ -245,7 +244,6 @@ mod voter_list { assert!(node.is_misplaced(20)); // then updating position moves it to the correct bag - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); assert_eq!(List::::update_position_for(node, 20), Some((10, 20))); assert_eq!(get_bags(), vec![(20, vec![1]), (1_000, vec![2, 3, 4])]); @@ -326,12 +324,7 @@ mod bags { #[test] fn insert_node_happy_paths_works() { ExtBuilder::default().build_and_execute_no_post_check(|| { - let node = |id, bag_upper| Node:: { - id, - prev: None, - next: None, - bag_upper, - }; + let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; // when inserting into a bag with 1 node let mut bag_10 = Bag::::get(10).unwrap(); @@ -353,24 +346,15 @@ mod bags { assert_eq!(bag_as_ids(&bag_20), vec![62]); // when inserting a node pointing to the accounts not in the bag - let node_61 = Node:: { - id: 61, - prev: Some(21), - next: Some(101), - bag_upper: 20, - }; + let node_61 = + Node:: { id: 61, prev: Some(21), next: Some(101), bag_upper: 20 }; bag_20.insert_node(node_61); // then ids are in order assert_eq!(bag_as_ids(&bag_20), vec![62, 61]); // and when the node is re-fetched all the info is correct assert_eq!( Node::::get(20, &61).unwrap(), - Node:: { - id: 61, - prev: Some(62), - next: None, - bag_upper: 20, - } + Node:: { id: 61, prev: Some(62), next: None, bag_upper: 20 } ); // state of all bags is as expected @@ -385,12 +369,7 @@ mod bags { // Document improper ways `insert_node` may be getting used. #[test] fn insert_node_bad_paths_documented() { - let node = |id, prev, next, bag_upper| Node:: { - id, - prev, - next, - bag_upper, - }; + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; ExtBuilder::default().build_and_execute_no_post_check(|| { // when inserting a node with both prev & next pointing at an account in an incorrect // bag. @@ -453,12 +432,7 @@ mod bags { #[should_panic = "system logic error: inserting a node who has the id of tail"] fn insert_node_duplicate_tail_panics_with_debug_assert() { ExtBuilder::default().build_and_execute(|| { - let node = |id, prev, next, bag_upper| Node:: { - id, - prev, - next, - bag_upper, - }; + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; // given assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])],); From d9540def5d750eaf0ac581a643d2e51cf5d1a952 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 11:08:39 +0200 Subject: [PATCH 117/241] Fix all staking tests --- frame/staking/src/pallet/impls.rs | 51 ++++++++++++++++--------------- frame/staking/src/pallet/mod.rs | 14 ++++----- frame/staking/src/tests.rs | 39 +++++++++++------------ 3 files changed, 50 insertions(+), 54 deletions(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 3439057acc2fc..25c5aa7177892 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -136,7 +136,7 @@ impl Pallet { // Nothing to do if they have no reward points. if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()); } // This is the fraction of the total reward that the validator and the @@ -227,8 +227,9 @@ impl Pallet { Self::update_ledger(&controller, &l); r }), - RewardDestination::Account(dest_account) => - Some(T::Currency::deposit_creating(&dest_account, amount)), + RewardDestination::Account(dest_account) => { + Some(T::Currency::deposit_creating(&dest_account, amount)) + } RewardDestination::None => None, } } @@ -256,14 +257,14 @@ impl Pallet { _ => { // Either `Forcing::ForceNone`, // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None - }, + return None; + } } // New era. let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() && - matches!(ForceEra::::get(), Forcing::ForceNew) + if maybe_new_era_validators.is_some() + && matches!(ForceEra::::get(), Forcing::ForceNew) { ForceEra::::put(Forcing::NotForcing); } @@ -444,12 +445,12 @@ impl Pallet { // TODO: this should be simplified #8911 CurrentEra::::put(0); ErasStartSessionIndex::::insert(&0, &start_session_index); - }, + } _ => (), } Self::deposit_event(Event::StakingElectionFailed); - return None + return None; } Self::deposit_event(Event::StakersElected); @@ -636,8 +637,8 @@ impl Pallet { /// Get all of the voters that are eligible for the npos election. /// - /// `voter_count` imposes an implicit cap on the number of voters returned; care should be taken - /// to ensure that it is accurate. + /// `voter_count` imposes a cap on the number of voters returned; care should be taken to ensure + /// that it is accurate. /// /// This will use all on-chain nominators, and all the validators will inject a self vote. /// @@ -645,30 +646,30 @@ impl Pallet { /// /// All nominations that have been submitted before the last non-zero slash of the validator are /// auto-chilled. - /// - /// Note that this is fairly expensive: it must iterate over the min of `maybe_max_len` and - /// `voter_count` voters. Use with care. pub fn get_npos_voters( maybe_max_len: Option, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { // TODO: weight this function and see how many can fit in a block. let weight_of = Self::weight_of_fn(); + let nominator_count = CounterForNominators::::get() as usize; let validator_count = CounterForValidators::::get() as usize; let all_voter_count = validator_count.saturating_add(nominator_count); - let total_len = maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count); - let mut all_voters = Vec::<_>::with_capacity(total_len); + let max_allowed_len = maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count); + drop(all_voter_count); + + let mut all_voters = Vec::<_>::with_capacity(max_allowed_len); - // first, grab all validators. - for (validator, _) in >::iter() { + // first, grab all validators, capped by the maximum allowed length. + for (validator, _) in >::iter().take(max_allowed_len) { // Append self vote. let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]); all_voters.push(self_vote); } - // see how many voters we can grab from nominator sorted list, and grab them. - let nominators_quota = total_len.saturating_sub(validator_count); + // .. and grab whatever we have left from nominators. + let nominators_quota = max_allowed_len.saturating_sub(validator_count); let slashing_spans = >::iter().collect::>(); for nominator in T::SortedListProvider::iter().take(nominators_quota) { if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = @@ -683,7 +684,7 @@ impl Pallet { all_voters.push((nominator.clone(), weight_of(&nominator), targets)) } } else { - log!(warn, "invalid item in `SortedListProvider`: {:?}", nominator) + log!(error, "invalid item in `SortedListProvider`: {:?}", nominator) } } @@ -693,7 +694,7 @@ impl Pallet { log!( info, "generated {} npos voters, {} from validators and {} nominators", - all_voter_count, + all_voters.len(), validator_count, nominators_quota ); @@ -812,7 +813,7 @@ impl ElectionDataProvider> for Pallet let target_count = CounterForValidators::::get() as usize; if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big") + return Err("Target snapshot too big"); } let weight = ::DbWeight::get().reads(target_count as u64); @@ -1076,7 +1077,7 @@ where add_db_reads_writes(1, 0); if active_era.is_none() { // This offence need not be re-submitted. - return consumed_weight + return consumed_weight; } active_era.expect("value checked not to be `None`; qed").index }; @@ -1122,7 +1123,7 @@ where // Skip if the validator is invulnerable. if invulnerables.contains(stash) { - continue + continue; } let unapplied = slashing::compute_slash::(slashing::SlashParams { diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 3df3eb6bde389..31c33fc5824a7 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -17,7 +17,7 @@ //! Staking FRAME Pallet. -use frame_election_provider_support::{SortedListProvider, VoteWeight}; +use frame_election_provider_support::SortedListProvider; use frame_support::{ pallet_prelude::*, traits::{ @@ -518,7 +518,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue + continue; } match status { StakerStatus::Validator => { @@ -528,7 +528,7 @@ pub mod pallet { ) { log!(warn, "failed to validate staker at genesis: {:?}.", why); } - }, + } StakerStatus::Nominator(votes) => { if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), @@ -536,7 +536,7 @@ pub mod pallet { ) { log!(warn, "failed to nominate staker at genesis: {:?}.", why); } - }, + } _ => (), }; } @@ -1343,9 +1343,9 @@ pub mod pallet { Self::update_ledger(&controller, &ledger); T::SortedListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. Ok(Some( - 35 * WEIGHT_PER_MICROS + - 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + - T::DbWeight::get().reads_writes(3, 2), + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), ) .into()) } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 07f62bbc641dd..fd165f52ac2e8 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -276,9 +276,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 + - part_for_100_from_10 * total_payout_0 * 2 / 3 + - part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * total_payout_0 * 2 / 3 + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -314,9 +314,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 + - part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) + - part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -561,8 +561,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 1200, own: 1000, others: vec![ - IndividualExposure { who: 1, value: 600 }, IndividualExposure { who: 3, value: 600 }, + IndividualExposure { who: 1, value: 600 }, ] }, ); @@ -3834,8 +3834,8 @@ mod election_data_provider { #[test] fn targets_2sec_block() { let mut validators = 1000; - while ::WeightInfo::get_npos_targets(validators) < - 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_targets(validators) + < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { validators += 1; } @@ -3852,8 +3852,8 @@ mod election_data_provider { let slashing_spans = validators; let mut nominators = 1000; - while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) < - 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) + < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { nominators += 1; } @@ -3925,8 +3925,12 @@ mod election_data_provider { #[test] fn respects_snapshot_len_limits() { ExtBuilder::default().validator_pool(true).build_and_execute(|| { - // sum of all validators and nominators who'd be voters. - assert_eq!(::SortedListProvider::count(), 5); + // sum of all nominators who'd be voters, plus the self votes. + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 5 + ); // if limits is less.. assert_eq!(Staking::voters(Some(1)).unwrap().0.len(), 1); @@ -4314,12 +4318,3 @@ mod election_data_provider { }) } } - -mod sorted_list_provider { - #[test] - fn iter_works() { - // TODO: not sure if this is really needed. What should I check? if it returns the correct - // type, then it is correct from my PoV. - todo!() - } -} From ce85c12a7b0818e097eeb81383cb0cbe37d8b2bb Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 11:52:07 +0200 Subject: [PATCH 118/241] retire VoterBagFor --- frame/bags-list/src/lib.rs | 37 ++++++++------- frame/bags-list/src/list/mod.rs | 75 +++++++++-------------------- frame/bags-list/src/list/tests.rs | 79 ++++++++++++++++--------------- frame/bags-list/src/tests.rs | 2 - 4 files changed, 84 insertions(+), 109 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 52edd43ec7d41..8b52897e620f5 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -15,8 +15,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! A semi-sorted list, where items hold an `id` and `weight`. -//! TODO items will also have a generic `data` field +//! A semi-sorted list, where items hold an `AccountId` based on some `VoteWeight`. The `AccountId` +//! can be synonym to a `Voter` and `VoteWeight` signifies the chance of each voter being included +//! in the final [`VoteWeightProvider::iter`]. +//! +//! It implements [`sp_election_provider_support::SortedListProvider`] to provide a semi-sorted list +//! of accounts to another pallet. It needs some other pallet to give it some information about the +//! weights of accounts via [`sp_election_provider_support::VoteWeightProvider`]. //! //! # ⚠️ WARNING ⚠️ //! @@ -27,21 +32,26 @@ //! //! The data structure exposed by this pallet aims to be optimized for: //! -//! - insertions and removals +//! - insertions and removals. //! - iteration over the top* N items by weight, where the precise ordering of items doesn't //! particularly matter. //! //! # Details //! -//! - items are kept in bags, which are delineated by their range of weight. (See [`BagThresholds`]) -//! - for iteration bags are chained together from highest to lowest and elements within the bag +//! - items are kept in bags, which are delineated by their range of weight (See [`BagThresholds`]). +//! - for iteration, bags are chained together from highest to lowest and elements within the bag //! are iterated from head to tail. -//! - items within a bag are in order of insertion. Thus removing an item and re-inserting it -//! will worsen its position in list iteration; this reduces incentives for some types -//! of spam that involve consistently removing and inserting for better position. Further, +//! - thus, items within a bag are iterated in order of insertion. Thus removing an item and +//! re-inserting it will worsen its position in list iteration; this reduces incentives for some +//! types of spam that involve consistently removing and inserting for better position. Further, //! ordering granularity is thus dictated by range between each bag threshold. //! - if an items weight changes to a value no longer within the range of its current bag its //! position will need to be updated by an external actor with rebag or removal & insertion. +// +//! ### Further Plans +//! +//! *new terminology*: fully decouple this pallet from voting names, use account id instead of +//! voter and priority instead of weight. use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; use frame_system::ensure_signed; @@ -92,7 +102,7 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: weights::WeightInfo; - // Something that implements `trait StakingVoteWeight`. + // Something that provides the weights of voters. type VoteWeightProvider: VoteWeightProvider; /// The list of thresholds separating the various voter bags. @@ -159,13 +169,6 @@ pub mod pallet { pub(crate) type VoterNodes = StorageMap<_, Twox64Concat, T::AccountId, list::Node>; - /// Which bag currently contains a particular voter. - /// - /// This may not be the appropriate bag for the voter's weight if they have been rewarded or - /// slashed. - #[pallet::storage] - pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, T::AccountId, VoteWeight>; // TODO: retire this storage item. - /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] @@ -223,7 +226,7 @@ impl Pallet { ) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = list::Node::::from_id(&account) + let maybe_movement = list::Node::::get(&account) .and_then(|node| List::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { Self::deposit_event(Event::::Rebagged(account.clone(), from, to)); diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 6e5bc25df4ae9..4553397d04ad7 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -48,11 +48,6 @@ fn notional_bag_for(weight: VoteWeight) -> VoteWeight { thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } -/// Find the upper threshold of the actual bag containing the current voter. -fn current_bag_for(id: &T::AccountId) -> Option { - crate::VoterBagFor::::try_get(id).ok() -} - /// Data structure providing efficient mostly-accurate selection of the top N voters by stake. /// /// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of @@ -71,7 +66,6 @@ impl List { /// Remove all data associated with the voter list from storage. pub(crate) fn clear() { crate::CounterForVoters::::kill(); - crate::VoterBagFor::::remove_all(None); crate::VoterBags::::remove_all(None); crate::VoterNodes::::remove_all(None); } @@ -131,7 +125,7 @@ impl List { if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's // no point iterating through bag 10 twice. - continue + continue; } if let Some(bag) = Bag::::get(affected_bag) { @@ -142,7 +136,7 @@ impl List { // a removed bag means that all members of that bag must be rebagged for removed_bag in old_set.difference(&new_set).copied() { if !affected_old_bags.insert(removed_bag) { - continue + continue; } if let Some(bag) = Bag::::get(removed_bag) { @@ -164,7 +158,7 @@ impl List { // lookups. for removed_bag in old_set.difference(&new_set).copied() { debug_assert!( - !crate::VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + !crate::VoterNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), "no voter should be present in a removed bag", ); crate::VoterBags::::remove(removed_bag); @@ -266,7 +260,7 @@ impl List { let mut count = 0; for voter_id in voters.into_iter() { - let node = match Node::::from_id(voter_id) { + let node = match Node::::get(voter_id) { Some(node) => node, None => continue, }; @@ -287,7 +281,6 @@ impl List { // now get rid of the node itself crate::VoterNodes::::remove(voter_id); - crate::VoterBagFor::::remove(voter_id); } for (_, bag) in bags { @@ -447,12 +440,12 @@ impl Bag { /// Get the head node in this bag. fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + self.head.as_ref().and_then(|id| Node::get(id)) } /// Get the tail node in this bag. fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + self.tail.as_ref().and_then(|id| Node::get(id)) } /// Iterate over the nodes in this bag. @@ -468,6 +461,7 @@ impl Bag { /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. fn insert(&mut self, id: T::AccountId) { + // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. self.insert_node(Node:: { id, prev: None, next: None, bag_upper: self.bag_upper }); } @@ -481,14 +475,17 @@ impl Bag { fn insert_node(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { - // DOCUMENT_ME // this should never happen, but this check prevents a worst case infinite loop debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return + return; }; } + // re-set the `bag_upper`. Regardless of whatever the node had previously, now it is going + // to be `self.bag_upper`. + node.bag_upper = self.bag_upper; + // update this node now, treating as the new tail. let id = node.id.clone(); node.prev = self.tail.clone(); @@ -502,16 +499,13 @@ impl Bag { } self.tail = Some(id.clone()); - // ensure head exist. This is only set when the length of the bag is just 1, i.e. - // if this is the first insertion into the bag. In this case, both head and tail should - // point to the same voter node. + // ensure head exist. This is only set when the length of the bag is just 1, i.e. if this is + // the first insertion into the bag. In this case, both head and tail should point to the + // same voter node. if self.head.is_none() { self.head = Some(id.clone()); debug_assert!(self.iter().count() == 1); } - - // update the voter's bag index. - crate::VoterBagFor::::insert(id, self.bag_upper); } /// Remove a voter node from this bag. Returns true iff the bag's head or tail is updated. @@ -520,8 +514,7 @@ impl Bag { /// the first place. Generally, use [`List::remove`] instead. /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` - /// to update storage for the bag and `node`. + /// `self.put()` and `VoterNodes::remove(voter_id)` to update storage for the bag and `node`. fn remove_node(&mut self, node: &Node) { // reassign neighboring nodes. node.excise(); @@ -579,41 +572,17 @@ impl Bag { #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] #[cfg_attr(test, derive(PartialEq, Clone))] -pub struct Node { +pub(crate) struct Node { id: T::AccountId, prev: Option, next: Option, - - /// The bag index is not stored in storage, but injected during all fetch operations. - #[codec(skip)] - pub(crate) bag_upper: VoteWeight, + bag_upper: VoteWeight, } impl Node { /// Get a node by bag idx and account id. - fn get(bag_upper: VoteWeight, account_id: &T::AccountId) -> Option> { - debug_assert!( - T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); - crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { - node.bag_upper = bag_upper; - node - }) - } - - /// Get a node by account id. - /// - /// Note that this must perform two storage lookups: one to identify which bag is appropriate, - /// and another to actually fetch the node. - pub(crate) fn from_id(account_id: &T::AccountId) -> Option> { - let bag = current_bag_for::(account_id)?; - Self::get(bag, account_id) - } - - /// Get a node by account id, assuming it's in the same bag as this node. - fn in_bag(&self, account_id: &T::AccountId) -> Option> { - Self::get(self.bag_upper, account_id) + pub(crate) fn get(account_id: &T::AccountId) -> Option> { + crate::VoterNodes::::try_get(account_id).ok() } /// Put the node back into storage. @@ -639,12 +608,12 @@ impl Node { /// Get the previous node in the bag. fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| self.in_bag(id)) + self.prev.as_ref().and_then(|id| Node::get(id)) } /// Get the next node in the bag. fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| self.in_bag(id)) + self.next.as_ref().and_then(|id| Node::get(id)) } /// `true` when this voter is in the wrong bag. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index a7f54447e0dda..7de8f50be4341 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::{ mock::{ext_builder::*, test_utils::*, *}, - CounterForVoters, VoterBagFor, VoterBags, VoterNodes, + CounterForVoters, VoterBags, VoterNodes, }; use frame_support::{assert_ok, assert_storage_noop}; @@ -9,10 +9,9 @@ use frame_support::{assert_ok, assert_storage_noop}; fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { // syntactic sugar to create a raw node - let node = |id, prev, next| Node:: { id, prev, next, bag_upper: 0 }; + let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; assert_eq!(CounterForVoters::::get(), 4); - assert_eq!(VoterBagFor::::iter().count(), 4); assert_eq!(VoterNodes::::iter().count(), 4); assert_eq!(VoterBags::::iter().count(), 2); @@ -28,20 +27,12 @@ fn basic_setup_works() { Bag:: { head: Some(2), tail: Some(4), bag_upper: 0 } ); - assert_eq!(VoterBagFor::::get(2).unwrap(), 1000); - assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3))); - - assert_eq!(VoterBagFor::::get(3).unwrap(), 1000); - assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4))); - - assert_eq!(VoterBagFor::::get(4).unwrap(), 1000); - assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None)); - - assert_eq!(VoterBagFor::::get(1).unwrap(), 10); - assert_eq!(VoterNodes::::get(1).unwrap(), node(1, None, None)); + assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3), 1000)); + assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1000)); + assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None, 1000)); + assert_eq!(VoterNodes::::get(1).unwrap(), node(1, None, None, 10)); // non-existent id does not have a storage footprint - assert_eq!(VoterBagFor::::get(41), None); assert_eq!(VoterNodes::::get(41), None); // iteration of the bags would yield: @@ -183,16 +174,13 @@ mod voter_list { fn remove_works() { use crate::{CounterForVoters, VoterBags, VoterNodes}; let ensure_left = |id, counter| { - assert!(!VoterBagFor::::contains_key(id)); assert!(!VoterNodes::::contains_key(id)); assert_eq!(CounterForVoters::::get(), counter); - assert_eq!(VoterBagFor::::iter().count() as u32, counter); assert_eq!(VoterNodes::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { // when removing a non-existent voter - assert!(!VoterBagFor::::contains_key(42)); assert!(!VoterNodes::::contains_key(42)); List::::remove(&42); @@ -237,7 +225,7 @@ mod voter_list { fn update_position_for_works() { ExtBuilder::default().build_and_execute(|| { // given a correctly placed account 1 - let node = Node::::from_id(&1).unwrap(); + let node = Node::::get(&1).unwrap(); assert!(!node.is_misplaced(10)); // .. it is invalid with weight 20 @@ -249,7 +237,7 @@ mod voter_list { assert_eq!(get_bags(), vec![(20, vec![1]), (1_000, vec![2, 3, 4])]); // get the new updated node; try and update the position with no change in weight. - let node = Node::::from_id(&1).unwrap(); + let node = Node::::get(&1).unwrap(); // TODO: we can pass a ref to node to this function as well. assert_storage_noop!(assert_eq!( List::::update_position_for(node.clone(), 20), @@ -261,7 +249,7 @@ mod voter_list { assert_eq!(get_bags(), vec![(1_000, vec![2, 3, 4, 1])]); // moving withing that bag again is a noop - let node = Node::::from_id(&1).unwrap(); + let node = Node::::get(&1).unwrap(); assert_storage_noop!(assert_eq!( List::::update_position_for(node.clone(), 750), None, @@ -321,6 +309,23 @@ mod bags { }); } + #[test] + fn insert_node_sets_proper_bag() { + ExtBuilder::default().build_and_execute_no_post_check(|| { + let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; + + assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + + let mut bag_10 = Bag::::get(10).unwrap(); + bag_10.insert_node(node(42, 5)); + + assert_eq!( + VoterNodes::::get(&42).unwrap(), + Node { bag_upper: 10, prev: Some(1), next: None, id: 42 } + ); + }); + } + #[test] fn insert_node_happy_paths_works() { ExtBuilder::default().build_and_execute_no_post_check(|| { @@ -353,7 +358,7 @@ mod bags { assert_eq!(bag_as_ids(&bag_20), vec![62, 61]); // and when the node is re-fetched all the info is correct assert_eq!( - Node::::get(20, &61).unwrap(), + Node::::get(&61).unwrap(), Node:: { id: 61, prev: Some(62), next: None, bag_upper: 20 } ); @@ -374,14 +379,14 @@ mod bags { // when inserting a node with both prev & next pointing at an account in an incorrect // bag. let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(42, Some(1), Some(1), 0)); + bag_1000.insert_node(node(42, Some(1), Some(1), 500)); // then the proper perv and next is set. assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 42]); // and when the node is re-fetched all the info is correct assert_eq!( - Node::::get(1_000, &42).unwrap(), + Node::::get(&42).unwrap(), node(42, Some(4), None, bag_1000.bag_upper) ); }); @@ -402,7 +407,7 @@ mod bags { // and the last accessible node has an **incorrect** prev pointer. // TODO: consider doing a check on insert, it is cheap. assert_eq!( - Node::::get(1_000, &3).unwrap(), + Node::::get(&3).unwrap(), node(3, Some(4), None, bag_1000.bag_upper) ); }); @@ -418,7 +423,7 @@ mod bags { // and the re-fetched node has bad pointers assert_eq!( - Node::::get(1_000, &2).unwrap(), + Node::::get(&2).unwrap(), node(2, Some(4), None, bag_1000.bag_upper) ); // ^^^ despite being the bags head, it has a prev @@ -469,7 +474,7 @@ mod bags { assert_eq!(bag_as_ids(&bag_2000), vec![15, 16, 17, 18, 19]); // when removing a node that is not pointing at the head or tail - let node_4 = Node::::get(bag_1000.bag_upper, &4).unwrap(); + let node_4 = Node::::get(&4).unwrap(); let node_4_pre_remove = node_4.clone(); bag_1000.remove_node(&node_4); @@ -480,7 +485,7 @@ mod bags { assert_eq!(node_4, node_4_pre_remove); // when removing a head that is not pointing at the tail - let node_2 = Node::::get(bag_1000.bag_upper, &2).unwrap(); + let node_2 = Node::::get(&2).unwrap(); bag_1000.remove_node(&node_2); // then @@ -488,7 +493,7 @@ mod bags { assert_ok!(bag_1000.sanity_check()); // when removing a tail that is not pointing at the head - let node_14 = Node::::get(bag_1000.bag_upper, &14).unwrap(); + let node_14 = Node::::get(&14).unwrap(); bag_1000.remove_node(&node_14); // then @@ -496,7 +501,7 @@ mod bags { assert_ok!(bag_1000.sanity_check()); // when removing a tail that is pointing at the head - let node_13 = Node::::get(bag_1000.bag_upper, &13).unwrap(); + let node_13 = Node::::get(&13).unwrap(); bag_1000.remove_node(&node_13); // then @@ -504,7 +509,7 @@ mod bags { assert_ok!(bag_1000.sanity_check()); // when removing a node that is both the head & tail - let node_3 = Node::::get(bag_1000.bag_upper, &3).unwrap(); + let node_3 = Node::::get(&3).unwrap(); bag_1000.remove_node(&node_3); bag_1000.put(); // put into storage so `get` returns the updated bag @@ -512,7 +517,7 @@ mod bags { assert_eq!(Bag::::get(1_000), None); // when removing a node that is pointing at both the head & tail - let node_11 = Node::::get(bag_10.bag_upper, &11).unwrap(); + let node_11 = Node::::get(&11).unwrap(); bag_10.remove_node(&node_11); // then @@ -520,7 +525,7 @@ mod bags { assert_ok!(bag_10.sanity_check()); // when removing a head that is pointing at the tail - let node_1 = Node::::get(bag_10.bag_upper, &1).unwrap(); + let node_1 = Node::::get(&1).unwrap(); bag_10.remove_node(&node_1); // then @@ -531,7 +536,7 @@ mod bags { bag_10.put(); // when removing a node that is pointing at the head but not the tail - let node_16 = Node::::get(bag_2000.bag_upper, &16).unwrap(); + let node_16 = Node::::get(&16).unwrap(); bag_2000.remove_node(&node_16); // then @@ -539,7 +544,7 @@ mod bags { assert_ok!(bag_2000.sanity_check()); // when removing a node that is pointing at tail, but not head - let node_18 = Node::::get(bag_2000.bag_upper, &18).unwrap(); + let node_18 = Node::::get(&18).unwrap(); bag_2000.remove_node(&node_18); // then @@ -580,7 +585,7 @@ mod bags { // Removing a node that is in another bag, will mess up the other bag. ExtBuilder::default().build_and_execute_no_post_check(|| { // given a tail node is in bag 1_000 - let node_4 = Node::::get(1_000, &4).unwrap(); + let node_4 = Node::::get(&4).unwrap(); // when we remove it from bag 10 let mut bag_10 = Bag::::get(10).unwrap(); @@ -607,7 +612,7 @@ mod node { #[test] fn is_misplaced_works() { ExtBuilder::default().build_and_execute(|| { - let node = Node::::get(10, &1).unwrap(); + let node = Node::::get(&1).unwrap(); // given assert_eq!(node.bag_upper, 10); diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 61c6797e8db0b..d8be9dcdf7459 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -238,11 +238,9 @@ mod sorted_list_provider { #[test] fn on_remove_works() { let ensure_left = |id, counter| { - assert!(!VoterBagFor::::contains_key(id)); assert!(!VoterNodes::::contains_key(id)); assert_eq!(>::count(), counter); assert_eq!(CounterForVoters::::get(), counter); - assert_eq!(VoterBagFor::::iter().count() as u32, counter); assert_eq!(VoterNodes::::iter().count() as u32, counter); }; From 238bdd676f40f3f6a152800fdcb9b86834636efa Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 14:07:20 +0200 Subject: [PATCH 119/241] commit --- Cargo.lock | 2 +- bin/node/runtime/Cargo.toml | 3 ++ bin/node/runtime/src/lib.rs | 15 ++++++- frame/bags-list/Cargo.toml | 4 +- frame/bags-list/src/benchmarks.rs | 20 ++------- frame/bags-list/src/lib.rs | 17 +++----- frame/bags-list/src/mock.rs | 6 ++- frame/election-provider-support/src/lib.rs | 3 ++ frame/staking/src/pallet/impls.rs | 50 ++++++++++++++++------ 9 files changed, 73 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d99723dce57d..7f15d2fe77b48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4408,6 +4408,7 @@ dependencies = [ "pallet-authority-discovery", "pallet-authorship", "pallet-babe", + "pallet-bags-list", "pallet-balances", "pallet-bounties", "pallet-collective", @@ -4888,7 +4889,6 @@ dependencies = [ "log", "pallet-balances", "parity-scale-codec", - "sp-arithmetic", "sp-core", "sp-io", "sp-runtime", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 78e46edbd64e4..16230620fa016 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -50,6 +50,7 @@ pallet-assets = { version = "4.0.0-dev", default-features = false, path = "../.. pallet-authority-discovery = { version = "4.0.0-dev", default-features = false, path = "../../../frame/authority-discovery" } pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../../../frame/authorship" } pallet-babe = { version = "4.0.0-dev", default-features = false, path = "../../../frame/babe" } +pallet-bags-list = { version = "4.0.0-dev", default-features = false, path = "../../../frame/bags-list" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../../frame/balances" } pallet-bounties = { version = "4.0.0-dev", default-features = false, path = "../../../frame/bounties" } pallet-collective = { version = "4.0.0-dev", default-features = false, path = "../../../frame/collective" } @@ -106,6 +107,7 @@ std = [ "pallet-authorship/std", "sp-consensus-babe/std", "pallet-babe/std", + "pallet-bags-list/std", "pallet-balances/std", "pallet-bounties/std", "sp-block-builder/std", @@ -173,6 +175,7 @@ runtime-benchmarks = [ "sp-runtime/runtime-benchmarks", "pallet-assets/runtime-benchmarks", "pallet-babe/runtime-benchmarks", + "pallet-bags-list/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-bounties/runtime-benchmarks", "pallet-collective/runtime-benchmarks", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index a90e55848a979..c73d6bdc13792 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -486,7 +486,6 @@ parameter_types! { pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; pub OffchainRepeat: BlockNumber = 5; - pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; } use frame_election_provider_support::onchain; @@ -517,7 +516,6 @@ impl pallet_staking::Config for Runtime { pallet_election_provider_multi_phase::OnChainConfig, >; type WeightInfo = pallet_staking::weights::SubstrateWeight; - type VoterBagThresholds = VoterBagThresholds; } parameter_types! { @@ -607,6 +605,18 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type VoterSnapshotPerBlock = VoterSnapshotPerBlock; } +// TODO: where' the file that generated this?? +parameter_types! { + pub const BagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; +} + +impl pallet_bags_list::Config for Runtime { + type Event = Event; + type VoteWeightProvider = Staking; + type WeightInfo = (); + type BagThresholds = BagThresholds; +} + parameter_types! { pub const LaunchPeriod: BlockNumber = 28 * 24 * 60 * MINUTES; pub const VotingPeriod: BlockNumber = 28 * 24 * 60 * MINUTES; @@ -1231,6 +1241,7 @@ construct_runtime!( Gilt: pallet_gilt::{Pallet, Call, Storage, Event, Config}, Uniques: pallet_uniques::{Pallet, Call, Storage, Event}, TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Config, Event}, + BagsList: pallet_bags_list::{Pallet, Call, Storage, Config, Event}, } ); diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index afff9980aa531..7d8f2c0c1522f 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -17,8 +17,6 @@ codec = { package = "parity-scale-codec", version = "2.0.0", default-features = sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../primitives/arithmetic" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -31,8 +29,10 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = " [dev-dependencies] sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} +sp-io = { version = "4.0.0-dev", path = "../../primitives/io"} sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } +frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } [features] default = ["std"] diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 95abd48abec8c..c94f7890dc0e7 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -28,8 +28,6 @@ frame_benchmarking::benchmarks! { // - The node to be rebagged should exist as a non-terminal node in a bag with at least // 2 other nodes so both its prev and next are nodes that will need be updated // when it is removed. - // TODO once we optimize for not writing the bag being removed we will have the case - // of removing head/tail from a bag with at least 2 nodes // - The destination bag is not empty, because then we need to update the `next` pointer // of the previous node in addition to the work we do otherwise. @@ -62,7 +60,10 @@ frame_benchmarking::benchmarks! { // and the ags are in the expected state after insertions. assert_eq!( get_bags::(), - vec![(origin_bag_thresh, vec![origin_head.clone(), origin_middle.clone(), origin_tail.clone()]), (dest_bag_thresh, vec![dest_head.clone()])] + vec![ + (origin_bag_thresh, vec![origin_head.clone(), origin_middle.clone(), origin_tail.clone()]), + (dest_bag_thresh, vec![dest_head.clone()]) + ] ); }: { // TODO need to figure out how to mock a return value for VoteWeightProvider::vote_weight @@ -81,19 +82,6 @@ frame_benchmarking::benchmarks! { vec![(origin_bag_thresh, vec![origin_head, origin_tail]), (dest_bag_thresh, vec![dest_head, origin_middle])] ); } - - // rebag_head { TODO once we have optimized so we only write the origin bag if head tail was updated - // An expensive case for rebagging - // - // - the node being rebagged is a head node, so origin bag must be updated with a new head - // and second node in the bag must have its prev pointer updated. - // - The destination bag is not empty, because then we need to update the `next` pointer - // of the previous node in addition to the work we do otherwise. - - // } _(Origin::signed(white_listed_caller()), origin_head) - // verify { - - // } } use frame_benchmarking::impl_benchmark_test_suite; diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 8b52897e620f5..9a00a9d8c305b 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -56,11 +56,10 @@ use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; use frame_system::ensure_signed; -#[cfg(feature = "runtime-benchmarks")] +#[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; mod list; -// #[cfg(any(feature = "runtime-benchmarks", test))] -#[cfg(test)] +#[cfg(any(feature = "runtime-benchmarks", test))] mod mock; #[cfg(test)] mod tests; @@ -204,14 +203,10 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn integrity_test() { - sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| { - assert!( - T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "Voter bag thresholds must strictly increase", - ); - }); - } + assert!( + T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "Voter bag thresholds must strictly increase", + ); } } } diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index a18c74fa47e58..17c6850fe8dae 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -14,6 +14,11 @@ impl frame_election_provider_support::VoteWeightProvider for StakingM fn vote_weight(_: &AccountId) -> VoteWeight { NextVoteWeight::get() } + #[cfg(feature = "runtime-benchmarks")] + fn set_vote_weight_of(_: &AccountId, weight: VoteWeight) { + // we don't really keep a mapping, just set weight for everyone. + NextVoteWeight::set(weight) + } } impl frame_system::Config for Runtime { @@ -135,7 +140,6 @@ pub(crate) mod test_utils { bag.iter().map(|n| *n.id()).collect::>() } - // TODO change this to get_list_as_ids pub(crate) fn get_list_as_ids() -> Vec { List::::iter().map(|n| *n.id()).collect::>() } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index c87edbfb3a174..19e8f9763303b 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -325,4 +325,7 @@ pub trait SortedListProvider { /// `SortedListProvider`. pub trait VoteWeightProvider { fn vote_weight(who: &AccountId) -> VoteWeight; + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn set_vote_weight_of(_: &AccountId, _: VoteWeight) {} } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 25c5aa7177892..0d7ca8069fc5a 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -18,7 +18,8 @@ //! Implementations for the Staking FRAME Pallet. use frame_election_provider_support::{ - data_provider, ElectionDataProvider, ElectionProvider, SortedListProvider, Supports, VoteWeight, + data_provider, ElectionDataProvider, ElectionProvider, SortedListProvider, Supports, + VoteWeight, VoteWeightProvider, }; use frame_support::{ pallet_prelude::*, @@ -76,6 +77,12 @@ impl Pallet { }) } + /// Same as `weight_of_fn`, but made for one time use. + pub fn weight_of(who: &T::AccountId) -> VoteWeight { + let issuance = T::Currency::total_issuance(); + Self::slashable_balance_of_vote_weight(who, issuance) + } + pub(super) fn do_payout_stakers( validator_stash: T::AccountId, era: EraIndex, @@ -136,7 +143,7 @@ impl Pallet { // Nothing to do if they have no reward points. if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()); + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) } // This is the fraction of the total reward that the validator and the @@ -227,9 +234,8 @@ impl Pallet { Self::update_ledger(&controller, &l); r }), - RewardDestination::Account(dest_account) => { - Some(T::Currency::deposit_creating(&dest_account, amount)) - } + RewardDestination::Account(dest_account) => + Some(T::Currency::deposit_creating(&dest_account, amount)), RewardDestination::None => None, } } @@ -257,14 +263,14 @@ impl Pallet { _ => { // Either `Forcing::ForceNone`, // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None; - } + return None + }, } // New era. let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() - && matches!(ForceEra::::get(), Forcing::ForceNew) + if maybe_new_era_validators.is_some() && + matches!(ForceEra::::get(), Forcing::ForceNew) { ForceEra::::put(Forcing::NotForcing); } @@ -445,12 +451,12 @@ impl Pallet { // TODO: this should be simplified #8911 CurrentEra::::put(0); ErasStartSessionIndex::::insert(&0, &start_session_index); - } + }, _ => (), } Self::deposit_event(Event::StakingElectionFailed); - return None; + return None } Self::deposit_event(Event::StakersElected); @@ -813,7 +819,7 @@ impl ElectionDataProvider> for Pallet let target_count = CounterForValidators::::get() as usize; if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big"); + return Err("Target snapshot too big") } let weight = ::DbWeight::get().reads(target_count as u64); @@ -1077,7 +1083,7 @@ where add_db_reads_writes(1, 0); if active_era.is_none() { // This offence need not be re-submitted. - return consumed_weight; + return consumed_weight } active_era.expect("value checked not to be `None`; qed").index }; @@ -1123,7 +1129,7 @@ where // Skip if the validator is invulnerable. if invulnerables.contains(stash) { - continue; + continue } let unapplied = slashing::compute_slash::(slashing::SlashParams { @@ -1173,6 +1179,22 @@ where } } +impl VoteWeightProvider for Pallet { + fn vote_weight(who: &T::AccountId) -> VoteWeight { + Self::weight_of(who) + } + + #[cfg(feature = "runtime-benchmarks")] + fn set_vote_weight_of(who: &T::AccountId, weight: VoteWeight) { + // this will clearly results in an inconsistent state, but it should not matter for a + // benchmark. + let active: BalanceOf = weight.try_into().unwrap(); + let mut ledger = Self::ledger(who).unwrap_or_default(); + ledger.active = active; + >::insert(who, ledger); + } +} + /// A simple voter list implementation that does not require any additional pallets. pub struct UseNominatorsMap(sp_std::marker::PhantomData); impl SortedListProvider for UseNominatorsMap { From 2f4e62c04adb2e54c688a1c3ac8d337bbb15c491 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 14:07:55 +0200 Subject: [PATCH 120/241] bring in stashed changes --- frame/bags-list/src/lib.rs | 25 +++++++++++-------------- frame/bags-list/src/list/mod.rs | 11 ++++++++--- frame/bags-list/src/tests.rs | 20 +++++++++++++++++++- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 9a00a9d8c305b..94a19afba8248 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -183,19 +183,19 @@ pub mod pallet { #[pallet::call] impl Pallet { - /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its - /// stake that it should properly fall into a different bag than its current position. + /// Declare that some `reposition` account has, through rewards or penalties, sufficiently + /// changed its weight that it should properly fall into a different bag than its current + /// one. /// - /// This will adjust its position into the appropriate bag. This will affect its position - /// among the nominator/validator set once the snapshot is prepared for the election. + /// Anyone can call this function about any target account. /// - /// Anyone can call this function about any stash. - // #[pallet::weight(T::WeightInfo::rebag())] + /// Will never return an error; if `reposition` does not exist or doesn't need a rebag, then + /// it is a noop and only fees are collected from `origin`. #[pallet::weight(T::WeightInfo::rebag_middle())] - pub fn rebag(origin: OriginFor, account: T::AccountId) -> DispatchResult { + pub fn rebag(origin: OriginFor, reposition: T::AccountId) -> DispatchResult { ensure_signed(origin)?; - let weight = T::VoteWeightProvider::vote_weight(&account); - Pallet::::do_rebag(&account, weight); + let current_weight = T::VoteWeightProvider::vote_weight(&reposition); + Pallet::::do_rebag(&reposition, current_weight); Ok(()) } } @@ -203,9 +203,10 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn integrity_test() { + // ensure they are strictly increasing, this also implies that duplicates are detected. assert!( T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "Voter bag thresholds must strictly increase", + "Voter bag thresholds must strictly increase, and have no duplicates", ); } } @@ -239,10 +240,6 @@ impl SortedListProvider for Pallet { CounterForVoters::::get() } - /// # ⚠️ WARNING ⚠️ - /// - /// Do not insert an id that already exists in the list; doing so can result in catastrophic - /// failure of your blockchain, including entering into an infinite loop during block execution. fn on_insert(voter: T::AccountId, weight: VoteWeight) { List::::insert(voter, weight); } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 4553397d04ad7..214cd1873bc6c 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -312,6 +312,7 @@ impl List { // clear the old bag head/tail pointers as necessary if !node.is_terminal() { // this node is not a head or a tail, so we can just cut it out of the list. + // update and put the prev and next of this node, we do `node.put` later. node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { // this is a head or tail, so the bag must be updated. @@ -326,10 +327,14 @@ impl List { debug_assert!(false, "every node must have an extant bag associated with it"); } - // put the voter into the appropriate new bag - let new_idx = notional_bag_for::(new_weight); - node.bag_upper = new_idx; + // TODO: go through all APIs, and make a standard out of when things will put and when + // they don't. + + // put the voter into the appropriate new bag. + let new_bag_upper = notional_bag_for::(new_weight); let mut bag = Bag::::get_or_make(node.bag_upper); + // prev, next, and bag_upper of the node are updated inside `insert_node`, also + // `node.put` is in there. bag.insert_node(node); bag.put(); diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index d8be9dcdf7459..8876eebfef1ac 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -5,7 +5,7 @@ use frame_election_provider_support::SortedListProvider; use list::Bag; use mock::{ext_builder::*, test_utils::*, *}; -mod extrinsics { +mod pallet { use super::*; #[test] @@ -107,6 +107,24 @@ mod extrinsics { assert_eq!(Bag::::get(1_000), None); }); } + + #[test] + fn wrong_rebag_is_noop() { + // if rebag is wrong, + // or the target is non-existent, then it is a noop + todo!() + } + + #[test] + fn duplicate_in_bags_threshold_panics() { + todo!() + // probably needs some UI test + } + + #[test] + fn decreasing_in_bags_threshold_panics() { + todo!() + } } mod sorted_list_provider { From 30423d495cbe17f2d374b9d31951f787e0804212 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 05:21:25 -0700 Subject: [PATCH 121/241] save --- frame/bags-list/src/list/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 6e5bc25df4ae9..1615d18ea9dde 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -316,9 +316,8 @@ impl List { node.is_misplaced(new_weight).then(move || { let old_idx = node.bag_upper; - // clear the old bag head/tail pointers as necessary if !node.is_terminal() { - // this node is not a head or a tail, so we can just cut it out of the list. + // this node is not a head or a tail, so we can just cut it out of the bag. node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { // this is a head or tail, so the bag must be updated. From 794fce0fecfe5f0ef39f2e119f4334822da47b06 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 14:49:40 +0200 Subject: [PATCH 122/241] bench pipeline works now, but I can't run stuff --- bin/node/runtime/src/lib.rs | 7 +++++-- frame/bags-list/Cargo.toml | 10 +++++++++- frame/bags-list/src/lib.rs | 15 ++++++++++++++- frame/bags-list/src/list/mod.rs | 16 ++++++++-------- frame/bags-list/src/weights.rs | 2 +- frame/election-provider-support/src/lib.rs | 7 +++++++ frame/staking/Cargo.toml | 1 + frame/staking/src/lib.rs | 1 + frame/staking/src/pallet/impls.rs | 10 +++++++++- 9 files changed, 55 insertions(+), 14 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index c73d6bdc13792..009972e2d35a8 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -515,6 +515,9 @@ impl pallet_staking::Config for Runtime { type GenesisElectionProvider = onchain::OnChainSequentialPhragmen< pallet_election_provider_multi_phase::OnChainConfig, >; + // Alternatively, use pallet_staking::UseNominatorsMap to just use the nominators map. + // Note that this method does not scale to a very large number of nominators. + type SortedListProvider = BagsList; type WeightInfo = pallet_staking::weights::SubstrateWeight; } @@ -605,7 +608,6 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type VoterSnapshotPerBlock = VoterSnapshotPerBlock; } -// TODO: where' the file that generated this?? parameter_types! { pub const BagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; } @@ -1241,7 +1243,7 @@ construct_runtime!( Gilt: pallet_gilt::{Pallet, Call, Storage, Event, Config}, Uniques: pallet_uniques::{Pallet, Call, Storage, Event}, TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Config, Event}, - BagsList: pallet_bags_list::{Pallet, Call, Storage, Config, Event}, + BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, } ); @@ -1575,6 +1577,7 @@ impl_runtime_apis! { list_benchmark!(list, extra, pallet_assets, Assets); list_benchmark!(list, extra, pallet_babe, Babe); + list_benchmark!(list, extra, pallet_bags_list, BagsList); list_benchmark!(list, extra, pallet_balances, Balances); list_benchmark!(list, extra, pallet_bounties, Bounties); list_benchmark!(list, extra, pallet_collective, Council); diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 7d8f2c0c1522f..6d439390cd42e 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -25,7 +25,11 @@ frame-election-provider-support = { version = "4.0.0-dev", default-features = fa log = { version = "0.4.14", default-features = false } # Optional imports for benchmarking -frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking", optional = true, default-features = false } +sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = true, default-features = false } +sp-io = { version = "4.0.0-dev", path = "../../primitives/io", optional = true, default-features = false } +sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true, default-features = false } +pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true, default-features = false } [dev-dependencies] sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} @@ -47,4 +51,8 @@ std = [ ] runtime-benchmarks = [ "frame-benchmarking", + "sp-core", + "sp-io", + "pallet-balances", + "sp-tracing", ] diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 94a19afba8248..d1613b49d7c00 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -23,6 +23,9 @@ //! of accounts to another pallet. It needs some other pallet to give it some information about the //! weights of accounts via [`sp_election_provider_support::VoteWeightProvider`]. //! +//! This pallet is not configurable at genesis. Whoever uses it should call appropriate functions of +//! the `SortedListProvider` (i.e. `on_insert`) at their genesis. +//! //! # ⚠️ WARNING ⚠️ //! //! Do not insert an id that already exists in the list; doing so can result in catastrophic failure @@ -53,13 +56,16 @@ //! *new terminology*: fully decouple this pallet from voting names, use account id instead of //! voter and priority instead of weight. +#![cfg_attr(not(feature = "std"), no_std)] + use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; use frame_system::ensure_signed; +use sp_std::prelude::*; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; mod list; -#[cfg(any(feature = "runtime-benchmarks", test))] +#[cfg(test)] mod mock; #[cfg(test)] mod tests; @@ -252,6 +258,13 @@ impl SortedListProvider for Pallet { List::::remove(voter) } + fn regenerate( + all: impl IntoIterator, + weight_of: Box VoteWeight>, + ) -> u32 { + List::::regenerate(all, weight_of) + } + fn sanity_check() -> Result<(), &'static str> { List::::sanity_check() } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 214cd1873bc6c..0346f1feb4a12 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -76,13 +76,13 @@ impl List { /// consensus. /// /// Returns the number of voters migrated. - #[allow(dead_code)] pub fn regenerate( all: impl IntoIterator, weight_of: Box VoteWeight>, - ) { + ) -> u32 { Self::clear(); Self::insert_many(all, weight_of); + 0 // TODO } /// Migrate the voter list from one set of thresholds to another. @@ -125,7 +125,7 @@ impl List { if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's // no point iterating through bag 10 twice. - continue; + continue } if let Some(bag) = Bag::::get(affected_bag) { @@ -136,7 +136,7 @@ impl List { // a removed bag means that all members of that bag must be rebagged for removed_bag in old_set.difference(&new_set).copied() { if !affected_old_bags.insert(removed_bag) { - continue; + continue } if let Some(bag) = Bag::::get(removed_bag) { @@ -303,11 +303,11 @@ impl List { /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient /// to call [`self.remove_many`] followed by [`self.insert_many`]. pub(crate) fn update_position_for( - mut node: Node, + node: Node, new_weight: VoteWeight, ) -> Option<(VoteWeight, VoteWeight)> { node.is_misplaced(new_weight).then(move || { - let old_idx = node.bag_upper; + let old_bag_upper = node.bag_upper; // clear the old bag head/tail pointers as necessary if !node.is_terminal() { @@ -338,7 +338,7 @@ impl List { bag.insert_node(node); bag.put(); - (old_idx, new_idx) + (old_bag_upper, new_bag_upper) }) } @@ -483,7 +483,7 @@ impl Bag { // this should never happen, but this check prevents a worst case infinite loop debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); - return; + return }; } diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 284fa27afaa70..656883b8a08be 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -4,7 +4,7 @@ pub trait WeightInfo { fn rebag_middle() -> Weight; } -pub struct SubstrateWeight(std::marker::PhantomData); +pub struct SubstrateWeight(sp_std::marker::PhantomData); impl WeightInfo for SubstrateWeight { fn rebag_middle() -> Weight { // FAKE numbers diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 19e8f9763303b..26d0ba9590afa 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -318,6 +318,13 @@ pub trait SortedListProvider { fn on_remove(voter: &AccountId); /// Sanity check internal state of list. Only meant for debug compilation. fn sanity_check() -> Result<(), &'static str>; + /// Regenerate this list from scratch. Returns the count of items inserted. + /// + /// This should typically only be used to enable this flag at a runtime upgrade. + fn regenerate( + all: impl IntoIterator, + weight_of: Box VoteWeight>, + ) -> u32; } /// Something that can provide the `VoteWeight` of an account. Similar to [`ElectionProvider`] and diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index ef569f561b68e..73078d0076057 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -85,6 +85,7 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] +# TODO: clean this flag. make-bags = [ "chrono", "git2", diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 3ddfa4d4d5e7e..a43072c00d7c7 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -427,6 +427,7 @@ pub struct UnlockChunk { /// The ledger of a (bonded) stash. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] +#[cfg_attr(feature = "runtime-benchmarks", derive(Default))] pub struct StakingLedger { /// The stash account whose balance is actually locked and at stake. pub stash: AccountId, diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 0d7ca8069fc5a..aa57b998d6cf7 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1188,7 +1188,8 @@ impl VoteWeightProvider for Pallet { fn set_vote_weight_of(who: &T::AccountId, weight: VoteWeight) { // this will clearly results in an inconsistent state, but it should not matter for a // benchmark. - let active: BalanceOf = weight.try_into().unwrap(); + use sp_std::convert::TryInto; + let active: BalanceOf = weight.try_into().map_err(|_| ()).unwrap(); let mut ledger = Self::ledger(who).unwrap_or_default(); ledger.active = active; >::insert(who, ledger); @@ -1214,6 +1215,13 @@ impl SortedListProvider for UseNominatorsMap { fn on_remove(_voter: &T::AccountId) { // nothing to do on update. } + fn regenerate( + _: impl IntoIterator, + _: Box VoteWeight>, + ) -> u32 { + // nothing to do upon regenerate. + 0 + } fn sanity_check() -> Result<(), &'static str> { Ok(()) } From f5d9dc8532ffb76b1d7d3a176c8de9970ee43a67 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 05:51:12 -0700 Subject: [PATCH 123/241] sabe --- frame/bags-list/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 8b52897e620f5..50b74c66077ee 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -41,7 +41,7 @@ //! - items are kept in bags, which are delineated by their range of weight (See [`BagThresholds`]). //! - for iteration, bags are chained together from highest to lowest and elements within the bag //! are iterated from head to tail. -//! - thus, items within a bag are iterated in order of insertion. Thus removing an item and +//! - items within a bag are iterated in order of insertion. Thus removing an item and //! re-inserting it will worsen its position in list iteration; this reduces incentives for some //! types of spam that involve consistently removing and inserting for better position. Further, //! ordering granularity is thus dictated by range between each bag threshold. From c22e3fad4125e4fbc2f073957682f942b6a29729 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 15:56:21 +0200 Subject: [PATCH 124/241] benchmarks now run, but we have a failure --- bin/node/runtime/src/lib.rs | 1 + frame/bags-list/src/benchmarks.rs | 30 ++++++++--------------------- frame/bags-list/src/lib.rs | 2 +- frame/bags-list/src/list/mod.rs | 32 +++++++++++++------------------ frame/bags-list/src/weights.rs | 16 +++++----------- 5 files changed, 28 insertions(+), 53 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 009972e2d35a8..825f6ad77ac34 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1650,6 +1650,7 @@ impl_runtime_apis! { add_benchmark!(params, batches, pallet_assets, Assets); add_benchmark!(params, batches, pallet_babe, Babe); add_benchmark!(params, batches, pallet_balances, Balances); + add_benchmark!(params, batches, pallet_bags_list, BagsList); add_benchmark!(params, batches, pallet_bounties, Bounties); add_benchmark!(params, batches, pallet_collective, Council); add_benchmark!(params, batches, pallet_contracts, Contracts); diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index c94f7890dc0e7..bdec2d2917e75 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -3,8 +3,9 @@ use super::*; use crate::list::{Bag, List}; // use frame_benchmarking::{account, whitelisted_caller}; -use frame_benchmarking::account; +use frame_benchmarking::{account, whitelisted_caller}; use frame_support::traits::Get; +use frame_system::RawOrigin as SystemOrigin; // use frame_system::RawOrigin; fn get_bags() -> Vec<(VoteWeight, Vec)> { @@ -17,12 +18,8 @@ fn get_bags() -> Vec<(VoteWeight, Vec)> { .collect::>() } -fn get_list_as_ids() -> Vec { - List::::iter().map(|n| n.id().clone()).collect::>() -} - frame_benchmarking::benchmarks! { - rebag_middle { + rebag { // An expensive case for rebag-ing: // // - The node to be rebagged should exist as a non-terminal node in a bag with at least @@ -52,11 +49,6 @@ frame_benchmarking::benchmarks! { let dest_head: T::AccountId = account("dest_head", 0, 0); List::::insert(dest_head.clone(), dest_bag_thresh); - // check that the list - assert_eq!( - get_list_as_ids::(), - vec![dest_head.clone(), origin_head.clone(), origin_middle.clone(), origin_tail.clone()] - ); // and the ags are in the expected state after insertions. assert_eq!( get_bags::(), @@ -65,18 +57,12 @@ frame_benchmarking::benchmarks! { (dest_bag_thresh, vec![dest_head.clone()]) ] ); - }: { - // TODO need to figure out how to mock a return value for VoteWeightProvider::vote_weight - // and then call the real rebag extrinsic - Pallet::::do_rebag(&origin_middle, dest_bag_thresh) - } + + let caller = whitelisted_caller(); + T::VoteWeightProvider::set_vote_weight_of(&origin_middle, dest_bag_thresh); + }: _(SystemOrigin::Signed(caller), origin_middle.clone()) verify { - // check that the list - assert_eq!( - List::::iter().map(|n| n.id().clone()).collect::>(), - vec![dest_head.clone(), origin_middle.clone(), origin_head.clone(), origin_tail.clone()] - ); - // and the bags have updated as expected. + // check the bags have updated as expected. assert_eq!( get_bags::(), vec![(origin_bag_thresh, vec![origin_head, origin_tail]), (dest_bag_thresh, vec![dest_head, origin_middle])] diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index d1613b49d7c00..44df69b92d5f3 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -197,7 +197,7 @@ pub mod pallet { /// /// Will never return an error; if `reposition` does not exist or doesn't need a rebag, then /// it is a noop and only fees are collected from `origin`. - #[pallet::weight(T::WeightInfo::rebag_middle())] + #[pallet::weight(T::WeightInfo::rebag())] pub fn rebag(origin: OriginFor, reposition: T::AccountId) -> DispatchResult { ensure_signed(origin)?; let current_weight = T::VoteWeightProvider::vote_weight(&reposition); diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 0346f1feb4a12..d23c0f48b89f9 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -36,9 +36,9 @@ mod tests; /// Bags are identified by their upper threshold; the value returned by this function is guaranteed /// to be a member of `T::BagThresholds`. /// -/// This is used instead of a simpler scheme, such as the index within `T::BagThresholds`, -/// because in the event that bags are inserted or deleted, the number of affected voters which need -/// to be migrated is smaller. +/// This is used instead of a simpler scheme, such as the index within `T::BagThresholds`, because +/// in the event that bags are inserted or deleted, the number of affected voters which need to be +/// migrated is smaller. /// /// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this /// function behaves as if it does. @@ -54,12 +54,12 @@ fn notional_bag_for(weight: VoteWeight) -> VoteWeight { /// arbitrary and unbounded length, all having a vote weight within a particular constant range. /// This structure means that voters can be added and removed in `O(1)` time. /// -/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. -/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall -/// stake decreases as successive bags are reached. This means that it is valid to truncate -/// iteration at any desired point; only those voters in the lowest bag (who are known to have -/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire -/// for fairness and the requirement for efficiency. +/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While +/// the users within any particular bag are sorted in an entirely arbitrary order, the overall stake +/// decreases as successive bags are reached. This means that it is valid to truncate iteration at +/// any desired point; only those voters in the lowest bag (who are known to have relatively little +/// power to affect the outcome) can be excluded. This satisfies both the desire for fairness and +/// the requirement for efficiency. pub struct List(PhantomData); impl List { @@ -362,13 +362,7 @@ impl List { let iter_count = Self::iter().collect::>().len() as u32; let stored_count = crate::CounterForVoters::::get(); - ensure!( - iter_count == stored_count, - // TODO @kian how strongly do you feel about this String? - // afaict its non-trivial to get this work with compile flags etc. - // format!("iter_count {} != stored_count {}", iter_count, stored_count) - "iter_count != stored_count", - ); + ensure!(iter_count == stored_count, "iter_count != stored_count",); let _ = T::BagThresholds::get() .into_iter() @@ -382,9 +376,9 @@ impl List { /// A Bag is a doubly-linked list of voters. /// -/// Note that we maintain both head and tail pointers. While it would be possible to get away -/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's -/// more desirable to ensure that there is some element of first-come, first-serve to the list's +/// Note that we maintain both head and tail pointers. While it would be possible to get away with +/// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more +/// desirable to ensure that there is some element of first-come, first-serve to the list's /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. #[derive(DefaultNoBound, Encode, Decode)] diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 656883b8a08be..e3db1b1e57881 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -1,24 +1,18 @@ use frame_support::{pallet_prelude::Weight, weights::constants::RocksDbWeight}; pub trait WeightInfo { - fn rebag_middle() -> Weight; + fn rebag() -> Weight; } pub struct SubstrateWeight(sp_std::marker::PhantomData); impl WeightInfo for SubstrateWeight { - fn rebag_middle() -> Weight { - // FAKE numbers - (84_501_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + fn rebag() -> Weight { + 0 } } impl WeightInfo for () { - fn rebag_middle() -> Weight { - // FAKE numbers - (84_501_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + fn rebag() -> Weight { + 0 } } From da3e0d52daaee4570ac23649fa146222cb294bba Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 06:57:16 -0700 Subject: [PATCH 125/241] WIP: Wire up make_bags --- Cargo.lock | 7 +- Cargo.toml | 2 +- .../{voter-bags => bag-thresholds}/Cargo.toml | 4 +- bin/node/runtime/bag-thresholds/src/main.rs | 49 ++++ bin/node/runtime/voter-bags/src/main.rs | 38 --- frame/bags-list/Cargo.toml | 11 + frame/bags-list/src/lib.rs | 4 +- frame/bags-list/src/list/mod.rs | 23 +- frame/bags-list/src/make_bags.rs | 227 +++++++++++++++++- frame/staking/src/pallet/mod.rs | 12 +- frame/staking/src/tests.rs | 24 +- 11 files changed, 319 insertions(+), 82 deletions(-) rename bin/node/runtime/{voter-bags => bag-thresholds}/Cargo.toml (76%) create mode 100644 bin/node/runtime/bag-thresholds/src/main.rs delete mode 100644 bin/node/runtime/voter-bags/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 7f15d2fe77b48..b3c689fba026b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4470,11 +4470,11 @@ dependencies = [ ] [[package]] -name = "node-runtime-voter-bags" +name = "node-runtime-bag-thresholds" version = "3.0.0" dependencies = [ "node-runtime", - "pallet-staking", + "pallet-bags-list", "sp-io", "structopt", ] @@ -4882,11 +4882,14 @@ dependencies = [ name = "pallet-bags-list" version = "4.0.0-dev" dependencies = [ + "chrono", "frame-benchmarking", "frame-election-provider-support", "frame-support", "frame-system", + "git2", "log", + "num-format", "pallet-balances", "parity-scale-codec", "sp-core", diff --git a/Cargo.toml b/Cargo.toml index 5d695a0cd7cf5..e0d1eab7b38d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ "bin/node/rpc", "bin/node/rpc-client", "bin/node/runtime", - "bin/node/runtime/voter-bags", + "bin/node/runtime/bag-thresholds", "bin/node/testing", "bin/utils/chain-spec-builder", "bin/utils/subkey", diff --git a/bin/node/runtime/voter-bags/Cargo.toml b/bin/node/runtime/bag-thresholds/Cargo.toml similarity index 76% rename from bin/node/runtime/voter-bags/Cargo.toml rename to bin/node/runtime/bag-thresholds/Cargo.toml index 55b9dca859db8..e95320877c38d 100644 --- a/bin/node/runtime/voter-bags/Cargo.toml +++ b/bin/node/runtime/bag-thresholds/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "node-runtime-voter-bags" +name = "node-runtime-bag-thresholds" version = "3.0.0" authors = ["Parity Technologies "] edition = "2018" @@ -11,6 +11,6 @@ readme = "README.md" [dependencies] node-runtime = { version = "3.0.0-dev", path = ".." } -pallet-staking = { version = "4.0.0-dev", path = "../../../../frame/staking", features = ["make-bags"] } +pallet-bags-list = { version = "4.0.0-dev", path = "../../../../frame/bags-list", features = ["make-bags"] } sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } structopt = "0.3.21" diff --git a/bin/node/runtime/bag-thresholds/src/main.rs b/bin/node/runtime/bag-thresholds/src/main.rs new file mode 100644 index 0000000000000..f278d6617cbf5 --- /dev/null +++ b/bin/node/runtime/bag-thresholds/src/main.rs @@ -0,0 +1,49 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Make the set of voting bag thresholds to be used in `voter_bags.rs`. + +use pallet_voter_bags::make_bags::generate_thresholds_module; +use std::path::PathBuf; +use structopt::StructOpt; +use node_runtime::Runtime; +use frame_support::traits::{Currency, CurrencyToVote}; + +#[derive(Debug, StructOpt)] +struct Opt { + /// How many bags to generate. + #[structopt(long, default_value = "200")] + n_bags: usize, + + /// Where to write the output. + output: PathBuf, +} + +fn main() -> Result<(), std::io::Error> { + let Opt { n_bags, output } = Opt::from_args(); + let mut ext = sp_io::TestExternalities::new_empty(); + + + + ext.execute_with(|| + let existential_deposit = >>::minimum_balance(); + let issuance = >>::total_issuance(); + let existential_weight = Runtime::CurrencyToVote::to_vote(existential_deposit, issuance); + + generate_thresholds_module::(n_bags, existential_weight, &output); + ) +} diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs deleted file mode 100644 index ca4454839547c..0000000000000 --- a/bin/node/runtime/voter-bags/src/main.rs +++ /dev/null @@ -1,38 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Make the set of voting bag thresholds to be used in `voter_bags.rs`. - -// use pallet_staking::voter_bags::make_bags::generate_thresholds_module; -// use std::path::PathBuf; -// use structopt::StructOpt; - -// #[derive(Debug, StructOpt)] -// struct Opt { -// /// How many bags to generate. -// #[structopt(long, default_value = "200")] -// n_bags: usize, - -// /// Where to write the output. -// output: PathBuf, -// } - -// fn main() -> Result<(), std::io::Error> { -// let Opt { n_bags, output } = Opt::from_args(); -// let mut ext = sp_io::TestExternalities::new_empty(); -// ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) -// } diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 6d439390cd42e..50b24af9b68ca 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -31,6 +31,11 @@ sp-io = { version = "4.0.0-dev", path = "../../primitives/io", optional = true, sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true, default-features = false } pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true, default-features = false } +# Optional imports for making voter bags lists +chrono = { version = "0.4.19", optional = true } +git2 = { version = "0.13.20", default-features = false, optional = true } +num-format = { version = "0.4.0", optional = true } + [dev-dependencies] sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} sp-io = { version = "4.0.0-dev", path = "../../primitives/io"} @@ -56,3 +61,9 @@ runtime-benchmarks = [ "pallet-balances", "sp-tracing", ] +make-bags = [ + "chrono", + "git2", + "num-format", + "std", +] diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index a3887a78a111e..28234ca203436 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -65,6 +65,8 @@ use sp_std::prelude::*; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; mod list; +#[cfg(feature = "make-bags")] +pub mod make_bags; #[cfg(test)] mod mock; #[cfg(test)] @@ -107,7 +109,7 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: weights::WeightInfo; - // Something that provides the weights of voters. + /// Something that provides the weights of voters. type VoteWeightProvider: VoteWeightProvider; /// The list of thresholds separating the various voter bags. diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index fc2086ec93536..358e1be835b7e 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -81,8 +81,7 @@ impl List { weight_of: Box VoteWeight>, ) -> u32 { Self::clear(); - Self::insert_many(all, weight_of); - 0 // TODO + Self::insert_many(all, weight_of) } /// Migrate the voter list from one set of thresholds to another. @@ -148,7 +147,7 @@ impl List { let weight_of = T::VoteWeightProvider::vote_weight; Self::remove_many(affected_accounts.iter().map(|voter| voter)); let num_affected = affected_accounts.len() as u32; - Self::insert_many(affected_accounts.into_iter(), weight_of); + let _ = Self::insert_many(affected_accounts.into_iter(), weight_of); // we couldn't previously remove the old bags because both insertion and removal assume that // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid @@ -210,11 +209,15 @@ impl List { fn insert_many( voters: impl IntoIterator, weight_of: impl Fn(&T::AccountId) -> VoteWeight, - ) { + ) -> u32 { + let mut count = 0; voters.into_iter().for_each(|v| { let weight = weight_of(&v); Self::insert(v, weight); + count += 1; }); + + count // TODO check if this is really necessary } /// Insert a new voter into the appropriate bag in the voter list. Does not check for duplicates. @@ -266,10 +269,8 @@ impl List { }; count += 1; - // check if node.is_terminal - if !node.is_terminal() { - // this node is not a head or a tail and thus the bag does not need to be updated. + // this node is not a head or a tail and thus the bag does not need to be updated node.excise() } else { // this node is a head or tail, so the bag needs to be updated @@ -361,13 +362,7 @@ impl List { let iter_count = Self::iter().collect::>().len() as u32; let stored_count = crate::CounterForVoters::::get(); - ensure!( - iter_count == stored_count, - // TODO @kian how strongly do you feel about this String? - // afaict its non-trivial to get this work with compile flags etc. - // format!("iter_count {} != stored_count {}", iter_count, stored_count) - "iter_count != stored_count", - ); + ensure!(iter_count == stored_count, "iter_count != stored_count",); let _ = T::BagThresholds::get() .into_iter() diff --git a/frame/bags-list/src/make_bags.rs b/frame/bags-list/src/make_bags.rs index a5e14c3f0c13b..40dc25d86338c 100644 --- a/frame/bags-list/src/make_bags.rs +++ b/frame/bags-list/src/make_bags.rs @@ -1,13 +1,228 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Support code to ease the process of generating voter bags. +//! +//! The process of adding voter bags to a runtime requires only four steps. +//! +//! 1. Update the runtime definition. +//! +//! ```ignore +//! parameter_types!{ +//! pub const VoterBagThresholds: &'static [u64] = &[]; +//! } +//! +//! impl pallet_staking::Config for Runtime { +//! // +//! type VoterBagThresholds = VoterBagThresholds; +//! } +//! ``` +//! +//! 2. Write a little program to generate the definitions. This can be a near-identical copy of +//! `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime +//! definitions with the various calculations here. +//! +//! 3. Run that program: +//! +//! ```sh,notrust +//! $ cargo run -p node-runtime-voter-bags -- bin/node/runtime/src/voter_bags.rs +//! ``` +//! +//! 4. Update the runtime definition. +//! +//! ```diff,notrust +//! + mod voter_bags; +//! - pub const VoterBagThresholds: &'static [u64] = &[]; +//! + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; +//! ``` + +use crate::Config; +use frame_election_provider_support::VoteWeight; +use frame_support::traits::Get; +use std::{ + io::Write, + path::{Path, PathBuf}, +}; + /// Compute the existential weight for the specified configuration. /// /// Note that this value depends on the current issuance, a quantity known to change over time. /// This makes the project of computing a static value suitable for inclusion in a static, /// generated file _excitingly unstable_. -#[cfg(any(feature = "std", feature = "make-bags"))] -pub fn existential_weight() -> VoteWeight { - // use frame_support::traits::{Currency, CurrencyToVote}; +// TODO we can't access currency or currency to weight from this module +// #[cfg(any(feature = "std", feature = "make-bags"))] +// fn existential_weight() -> VoteWeight { +// use frame_support::traits::{Currency, CurrencyToVote}; + +// let existential_deposit = >::minimum_balance(); +// let issuance = >::total_issuance(); +// T::CurrencyToVote::to_vote(existential_deposit, issuance) +// } + +/// Return the path to a header file used in this repository if is exists. +/// +/// Just searches the git working directory root for files matching certain patterns; it's +/// pretty naive. +fn path_to_header_file() -> Option { + let repo = git2::Repository::open_from_env().ok()?; + let workdir = repo.workdir()?; + for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { + let path = workdir.join(file_name); + if path.exists() { + return Some(path) + } + } + None +} + +/// Create an underscore formatter: a formatter which inserts `_` every 3 digits of a number. +fn underscore_formatter() -> num_format::CustomFormat { + num_format::CustomFormat::builder() + .grouping(num_format::Grouping::Standard) + .separator("_") + .build() + .expect("format described here meets all constraints") +} + +/// Compute the constant ratio for the thresholds. +/// +/// This ratio ensures that each bag, with the possible exceptions of certain small ones and the +/// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` +/// space. +pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { + ((VoteWeight::MAX as f64 / existential_weight as f64).ln() / ((n_bags - 1) as f64)).exp() +} + +/// Compute the list of bag thresholds. +/// +/// Returns a list of exactly `n_bags` elements, except in the case of overflow. +/// The first element is always `existential_weight`. +/// The last element is always `VoteWeight::MAX`. +/// +/// All other elements are computed from the previous according to the formula +/// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1); +pub fn thresholds( + existential_weight: VoteWeight, + constant_ratio: f64, + n_bags: usize, +) -> Vec { + const WEIGHT_LIMIT: f64 = VoteWeight::MAX as f64; + + let mut thresholds = Vec::with_capacity(n_bags); + + if n_bags > 1 { + thresholds.push(existential_weight); + } + + while n_bags > 0 && thresholds.len() < n_bags - 1 { + let last = thresholds.last().copied().unwrap_or(existential_weight); + let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); + if successor < WEIGHT_LIMIT { + thresholds.push(successor as VoteWeight); + } else { + eprintln!("unexpectedly exceeded weight limit; breaking threshold generation loop"); + break + } + } + + thresholds.push(VoteWeight::MAX); + + debug_assert_eq!(thresholds.len(), n_bags); + debug_assert!(n_bags == 0 || thresholds[0] == existential_weight); + debug_assert!(n_bags == 0 || thresholds[thresholds.len() - 1] == VoteWeight::MAX); + + thresholds +} + +/// Write a thresholds module to the path specified. +/// +/// The `output` path should terminate with a Rust module name, i.e. `foo/bar/thresholds.rs`. +/// +/// This generated module contains, in order: +/// +/// - The contents of the header file in this repository's root, if found. +/// - Module documentation noting that this is autogenerated and when. +/// - Some associated constants. +/// - The constant array of thresholds. +pub fn generate_thresholds_module( + n_bags: usize, + existential_weight: VoteWeight, + output: &Path, +) -> Result<(), std::io::Error> { + // ensure the file is accessable + if let Some(parent) = output.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)?; + } + } + + // copy the header file + if let Some(header_path) = path_to_header_file() { + std::fs::copy(header_path, output)?; + } + + // open an append buffer + let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; + let mut buf = std::io::BufWriter::new(file); + + // create underscore formatter and format buffer + let mut num_buf = num_format::Buffer::new(); + let format = underscore_formatter(); + + // module docs + let now = chrono::Utc::now(); + writeln!(buf)?; + writeln!(buf, "//! Autogenerated voter bag thresholds.")?; + writeln!(buf, "//!")?; + writeln!(buf, "//! Generated on {}", now.to_rfc3339())?; + writeln!( + buf, + "//! for the {} runtime.", + ::Version::get().spec_name, + )?; + + // existential weight TODO can't get existential weight since we don't have currency module. + // let existential_weight = existential_weight::(); + num_buf.write_formatted(&existential_weight, &format); + writeln!(buf)?; + writeln!(buf, "/// Existential weight for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", num_buf.as_str())?; + + // constant ratio + let constant_ratio = constant_ratio(existential_weight, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Constant ratio between bags for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const CONSTANT_RATIO: f64 = {:.16};", constant_ratio)?; + + // thresholds + let thresholds = thresholds(existential_weight, constant_ratio, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; + writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; + for threshold in thresholds { + num_buf.write_formatted(&threshold, &format); + // u64::MAX, with spacers every 3 digits, is 26 characters wide + writeln!(buf, " {:>26},", num_buf.as_str())?; + } + writeln!(buf, "];")?; - let existential_deposit = >::minimum_balance(); - let issuance = >::total_issuance(); - T::CurrencyToVote::to_vote(existential_deposit, issuance) + Ok(()) } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 31c33fc5824a7..10a5f3da0da8f 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -518,7 +518,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue; + continue } match status { StakerStatus::Validator => { @@ -528,7 +528,7 @@ pub mod pallet { ) { log!(warn, "failed to validate staker at genesis: {:?}.", why); } - } + }, StakerStatus::Nominator(votes) => { if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), @@ -536,7 +536,7 @@ pub mod pallet { ) { log!(warn, "failed to nominate staker at genesis: {:?}.", why); } - } + }, _ => (), }; } @@ -1343,9 +1343,9 @@ pub mod pallet { Self::update_ledger(&controller, &ledger); T::SortedListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. Ok(Some( - 35 * WEIGHT_PER_MICROS - + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) - + T::DbWeight::get().reads_writes(3, 2), + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), ) .into()) } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index fd165f52ac2e8..85193c2a54046 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -276,9 +276,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * total_payout_0 * 2 / 3 - + part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * total_payout_0 * 2 / 3 + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -314,9 +314,9 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); assert_eq_error_rate!( Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) - + part_for_100_from_20 * total_payout_0 * 1 / 3, + init_balance_100 + + part_for_100_from_10 * (total_payout_0 * 2 / 3 + total_payout_1) + + part_for_100_from_20 * total_payout_0 * 1 / 3, 2 ); assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); @@ -3834,8 +3834,8 @@ mod election_data_provider { #[test] fn targets_2sec_block() { let mut validators = 1000; - while ::WeightInfo::get_npos_targets(validators) - < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_targets(validators) < + 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { validators += 1; } @@ -3852,8 +3852,8 @@ mod election_data_provider { let slashing_spans = validators; let mut nominators = 1000; - while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) - < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) < + 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { nominators += 1; } @@ -3927,8 +3927,8 @@ mod election_data_provider { ExtBuilder::default().validator_pool(true).build_and_execute(|| { // sum of all nominators who'd be voters, plus the self votes. assert_eq!( - ::SortedListProvider::count() - + >::iter().count() as u32, + ::SortedListProvider::count() + + >::iter().count() as u32, 5 ); From 49828f49778d9322c316b369d954372315494e41 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 07:18:59 -0700 Subject: [PATCH 126/241] bags-thresholds compiles --- Cargo.lock | 2 ++ bin/node/runtime/bag-thresholds/Cargo.toml | 10 +++++++++- bin/node/runtime/bag-thresholds/src/main.rs | 17 ++++++++++------- frame/bags-list/Cargo.toml | 6 +++++- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3c689fba026b..9b6d0790cd86c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4473,8 +4473,10 @@ dependencies = [ name = "node-runtime-bag-thresholds" version = "3.0.0" dependencies = [ + "frame-support", "node-runtime", "pallet-bags-list", + "pallet-staking", "sp-io", "structopt", ] diff --git a/bin/node/runtime/bag-thresholds/Cargo.toml b/bin/node/runtime/bag-thresholds/Cargo.toml index e95320877c38d..1551134db72b4 100644 --- a/bin/node/runtime/bag-thresholds/Cargo.toml +++ b/bin/node/runtime/bag-thresholds/Cargo.toml @@ -6,11 +6,19 @@ edition = "2018" license = "Apache-2.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -description = "Voter Bag generation script for pallet-staking and node-runtime." +description = "Bag threshold generation script for pallet-bag-list and node-runtime." readme = "README.md" [dependencies] node-runtime = { version = "3.0.0-dev", path = ".." } + +# FRAME +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/support" } pallet-bags-list = { version = "4.0.0-dev", path = "../../../../frame/bags-list", features = ["make-bags"] } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/staking" } + +# primitives sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } + +# third-party structopt = "0.3.21" diff --git a/bin/node/runtime/bag-thresholds/src/main.rs b/bin/node/runtime/bag-thresholds/src/main.rs index f278d6617cbf5..da4df88d43ae7 100644 --- a/bin/node/runtime/bag-thresholds/src/main.rs +++ b/bin/node/runtime/bag-thresholds/src/main.rs @@ -15,9 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Make the set of voting bag thresholds to be used in `voter_bags.rs`. +//! Make the set of bag thresholds to be used in pallet-bags-list. -use pallet_voter_bags::make_bags::generate_thresholds_module; +use pallet_bags_list::make_bags::generate_thresholds_module; use std::path::PathBuf; use structopt::StructOpt; use node_runtime::Runtime; @@ -39,11 +39,14 @@ fn main() -> Result<(), std::io::Error> { - ext.execute_with(|| - let existential_deposit = >>::minimum_balance(); - let issuance = >>::total_issuance(); - let existential_weight = Runtime::CurrencyToVote::to_vote(existential_deposit, issuance); + let res = ext.execute_with(||{ + // let existential_deposit = >::minimum_balance(); + let existential_deposit = ::Currency::minimum_balance(); + let issuance = ::Currency::total_issuance(); + let existential_weight = ::CurrencyToVote::to_vote(existential_deposit, issuance); generate_thresholds_module::(n_bags, existential_weight, &output); - ) + }); + + Ok(res) } diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 50b24af9b68ca..0df9561849d2c 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -13,23 +13,27 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +# Parity codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +# Primitives sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +# FRAME frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } +# Third party log = { version = "0.4.14", default-features = false } # Optional imports for benchmarking frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking", optional = true, default-features = false } +pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true, default-features = false } sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = true, default-features = false } sp-io = { version = "4.0.0-dev", path = "../../primitives/io", optional = true, default-features = false } sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true, default-features = false } -pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true, default-features = false } # Optional imports for making voter bags lists chrono = { version = "0.4.19", optional = true } From 5b1f237d2f62ddd61d99d9e914ce1ed801b983f3 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 19:34:18 +0200 Subject: [PATCH 127/241] Fix most build issues --- frame/bags-list/Cargo.toml | 1 + frame/bags-list/src/benchmarks.rs | 2 +- frame/bags-list/src/lib.rs | 4 ++ frame/bags-list/src/list/mod.rs | 14 +++---- frame/bags-list/src/mock.rs | 2 +- frame/election-provider-support/src/lib.rs | 2 + frame/staking/src/pallet/impls.rs | 49 ++++++++++++++++++++-- frame/staking/src/pallet/mod.rs | 29 +++++++------ 8 files changed, 75 insertions(+), 28 deletions(-) diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 6d439390cd42e..79c7222e9cb30 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -36,6 +36,7 @@ sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} sp-io = { version = "4.0.0-dev", path = "../../primitives/io"} sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } +frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support", features = ["runtime-benchmarks"] } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } [features] diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index bdec2d2917e75..69264ea98eee3 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -4,9 +4,9 @@ use super::*; use crate::list::{Bag, List}; // use frame_benchmarking::{account, whitelisted_caller}; use frame_benchmarking::{account, whitelisted_caller}; +use frame_election_provider_support::VoteWeightProvider; use frame_support::traits::Get; use frame_system::RawOrigin as SystemOrigin; -// use frame_system::RawOrigin; fn get_bags() -> Vec<(VoteWeight, Vec)> { T::BagThresholds::get() diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 44df69b92d5f3..5edb280e38fa0 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -246,6 +246,10 @@ impl SortedListProvider for Pallet { CounterForVoters::::get() } + fn contains(voter: &T::AccountId) -> bool { + VoterNodes::::contains_key(voter) + } + fn on_insert(voter: T::AccountId, weight: VoteWeight) { List::::insert(voter, weight); } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index d23c0f48b89f9..882196ecd977c 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -15,7 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Basic implementation of a doubly-linked list +//! Basic implementation of a doubly-linked list. + +// TODO: further testings: fuzzer for the outer API, dev chain with a lot of nominators. use crate::Config; use codec::{Decode, Encode}; @@ -226,7 +228,6 @@ impl List { pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { // TODO: can check if this voter exists as a node by checking if `voter` exists in the nodes // map and return early if it does. - // OR create a can_insert let bag_weight = notional_bag_for::(weight); crate::log!( @@ -259,6 +260,8 @@ impl List { let mut bags = BTreeMap::new(); let mut count = 0; + // TODO: assert that it is noop with bad data. + for voter_id in voters.into_iter() { let node = match Node::::get(voter_id) { Some(node) => node, @@ -311,8 +314,8 @@ impl List { // clear the old bag head/tail pointers as necessary if !node.is_terminal() { - // this node is not a head or a tail, so we can just cut it out of the list. - // update and put the prev and next of this node, we do `node.put` later. + // this node is not a head or a tail, so we can just cut it out of the list. update + // and put the prev and next of this node, we do `node.put` inside `insert_note`. node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { // this is a head or tail, so the bag must be updated. @@ -327,9 +330,6 @@ impl List { debug_assert!(false, "every node must have an extant bag associated with it"); } - // TODO: go through all APIs, and make a standard out of when things will put and when - // they don't. - // put the voter into the appropriate new bag. let new_bag_upper = notional_bag_for::(new_weight); let mut bag = Bag::::get_or_make(node.bag_upper); diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 17c6850fe8dae..4e019495cf7f5 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -14,7 +14,7 @@ impl frame_election_provider_support::VoteWeightProvider for StakingM fn vote_weight(_: &AccountId) -> VoteWeight { NextVoteWeight::get() } - #[cfg(feature = "runtime-benchmarks")] + #[cfg(any(feature = "runtime-benchmarks", test))] fn set_vote_weight_of(_: &AccountId, weight: VoteWeight) { // we don't really keep a mapping, just set weight for everyone. NextVoteWeight::set(weight) diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 26d0ba9590afa..d30079b6ba113 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -310,6 +310,8 @@ pub trait SortedListProvider { fn iter() -> Box>; /// get the current count of voters. fn count() -> u32; + /// Return true if the insertion can happen. + fn contains(voter: &AccountId) -> bool; // Hook for inserting a voter. fn on_insert(voter: AccountId, weight: VoteWeight); /// Hook for updating a single voter. diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index aa57b998d6cf7..a13ff896949b0 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -723,11 +723,38 @@ impl Pallet { /// wrong. pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { - CounterForNominators::::mutate(|x| x.saturating_inc()) + CounterForNominators::::mutate(|x| x.saturating_inc()); + if !T::SortedListProvider::contains(who) { + T::SortedListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + } else { + // TODO: emit a lot of warnings. + } } + // TODO: test: re-nominate should not increase the counter and T::SortedListProvider::count() + // consider these cases as well: + // chilled + // - validate + // - nominate + // - chill + // - bond-extra + // - unbond + // - rebond + // nominator + // - validate + // - nominate + // - chill + // - bond-extra + // - unbond + // - rebond + // validators + // - validate + // - nominate + // - chill + // - bond-extra + // - unbond + // - rebond Nominators::::insert(who, nominations); - T::SortedListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } /// This function will remove a nominator from the `Nominators` storage map, @@ -1181,7 +1208,12 @@ where impl VoteWeightProvider for Pallet { fn vote_weight(who: &T::AccountId) -> VoteWeight { - Self::weight_of(who) + // TODO: def. remove this. + if cfg!(feature = "runtime-benchmarks") { + Self::slashable_balance_of_vote_weight(who, sp_runtime::traits::Bounded::max_value()) + } else { + Self::weight_of(who) + } } #[cfg(feature = "runtime-benchmarks")] @@ -1193,6 +1225,12 @@ impl VoteWeightProvider for Pallet { let mut ledger = Self::ledger(who).unwrap_or_default(); ledger.active = active; >::insert(who, ledger); + + // also, we play a trick to make sure that a issuance based-`CurrencyToVote` behaves well: + // This will make sure that total issuance is zero, thus the currency to vote will be a 1-1 + // conversion. + // TODO + // let imbalance = T::Currency::burn(T::Currency::total_issuance()); } } @@ -1206,6 +1244,9 @@ impl SortedListProvider for UseNominatorsMap { fn count() -> u32 { CounterForNominators::::get() } + fn contains(voter: &T::AccountId) -> bool { + Nominators::::contains_key(voter) + } fn on_insert(_voter: T::AccountId, _weight: VoteWeight) { // nothing to do on update. } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 31c33fc5824a7..3905853cbd47f 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -518,7 +518,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue; + continue } match status { StakerStatus::Validator => { @@ -528,7 +528,7 @@ pub mod pallet { ) { log!(warn, "failed to validate staker at genesis: {:?}.", why); } - } + }, StakerStatus::Nominator(votes) => { if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), @@ -536,7 +536,7 @@ pub mod pallet { ) { log!(warn, "failed to nominate staker at genesis: {:?}.", why); } - } + }, _ => (), }; } @@ -783,7 +783,9 @@ pub mod pallet { Error::::InsufficientBond ); - T::SortedListProvider::on_update(&stash, Self::weight_of_fn()(&ledger.stash)); + if T::SortedListProvider::contains(&stash) { + T::SortedListProvider::on_update(&stash, Self::weight_of_fn()(&ledger.stash)); + } Self::deposit_event(Event::::Bonded(stash.clone(), extra)); Self::update_ledger(&controller, &ledger); } @@ -845,10 +847,9 @@ pub mod pallet { let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); Self::update_ledger(&controller, &ledger); - T::SortedListProvider::on_update( - &ledger.stash, - Self::weight_of_fn()(&ledger.stash), - ); + if T::SortedListProvider::contains(&ledger.stash) { + T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); + } Self::deposit_event(Event::::Unbonded(ledger.stash, value)); } Ok(()) @@ -1341,13 +1342,11 @@ pub mod pallet { Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); Self::update_ledger(&controller, &ledger); - T::SortedListProvider::on_update(&ledger.stash, Self::weight_of_fn()(&ledger.stash)); // TODO we already have the ledger here. - Ok(Some( - 35 * WEIGHT_PER_MICROS - + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) - + T::DbWeight::get().reads_writes(3, 2), - ) - .into()) + if T::SortedListProvider::contains(&ledger.stash) { + T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); + } + // TODO: fix this in master earlier. + Ok(Some(T::WeightInfo::rebond(ledger.unlocking.len() as u32)).into()) } /// Set `HistoryDepth` value. This function will delete any history information From 8fa8e6df751314125171019960c5d663f1a84fa2 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 5 Aug 2021 19:40:16 +0200 Subject: [PATCH 128/241] This will fix all the tests --- frame/bags-list/src/list/mod.rs | 2 +- frame/bags-list/src/list/tests.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index e5aa028acc812..cdf6df0c41f63 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -332,7 +332,7 @@ impl List { // put the voter into the appropriate new bag. let new_bag_upper = notional_bag_for::(new_weight); - let mut bag = Bag::::get_or_make(node.bag_upper); + let mut bag = Bag::::get_or_make(new_bag_upper); // prev, next, and bag_upper of the node are updated inside `insert_node`, also // `node.put` is in there. bag.insert_node(node); diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 7de8f50be4341..5a545f66e5748 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -224,14 +224,14 @@ mod voter_list { #[test] fn update_position_for_works() { ExtBuilder::default().build_and_execute(|| { - // given a correctly placed account 1 + // given a correctly placed account 1 at bag 10. let node = Node::::get(&1).unwrap(); assert!(!node.is_misplaced(10)); // .. it is invalid with weight 20 assert!(node.is_misplaced(20)); - // then updating position moves it to the correct bag + // move it to bag 20. assert_eq!(List::::update_position_for(node, 20), Some((10, 20))); assert_eq!(get_bags(), vec![(20, vec![1]), (1_000, vec![2, 3, 4])]); From 54c250ad7c7a47a4b81853cbc5d3e966ae6e8089 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 13:45:49 -0700 Subject: [PATCH 129/241] move bag thresholds to bags-list --- Cargo.toml | 2 +- frame/bags-list/Cargo.toml | 8 ++++---- .../bags-list}/bag-thresholds/Cargo.toml | 12 ++++++------ .../bags-list}/bag-thresholds/src/main.rs | 0 4 files changed, 11 insertions(+), 11 deletions(-) rename {bin/node/runtime => frame/bags-list}/bag-thresholds/Cargo.toml (62%) rename {bin/node/runtime => frame/bags-list}/bag-thresholds/src/main.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index e0d1eab7b38d4..85ec674e3e213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ members = [ "bin/node/rpc", "bin/node/rpc-client", "bin/node/runtime", - "bin/node/runtime/bag-thresholds", "bin/node/testing", "bin/utils/chain-spec-builder", "bin/utils/subkey", @@ -131,6 +130,7 @@ members = [ "frame/utility", "frame/vesting", "frame/bags-list", + "frame/bags-list/bag-thresholds", "primitives/api", "primitives/api/proc-macro", "primitives/api/test", diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 0df9561849d2c..cf19fb2533c70 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -13,10 +13,10 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -# Parity +# parity codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -# Primitives +# primitives sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } @@ -25,7 +25,7 @@ frame-support = { version = "4.0.0-dev", default-features = false, path = "../su frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } -# Third party +# third party log = { version = "0.4.14", default-features = false } # Optional imports for benchmarking @@ -35,7 +35,7 @@ sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = tr sp-io = { version = "4.0.0-dev", path = "../../primitives/io", optional = true, default-features = false } sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true, default-features = false } -# Optional imports for making voter bags lists +# optional imports for making voter bags lists chrono = { version = "0.4.19", optional = true } git2 = { version = "0.13.20", default-features = false, optional = true } num-format = { version = "0.4.0", optional = true } diff --git a/bin/node/runtime/bag-thresholds/Cargo.toml b/frame/bags-list/bag-thresholds/Cargo.toml similarity index 62% rename from bin/node/runtime/bag-thresholds/Cargo.toml rename to frame/bags-list/bag-thresholds/Cargo.toml index 1551134db72b4..64782c910346c 100644 --- a/bin/node/runtime/bag-thresholds/Cargo.toml +++ b/frame/bags-list/bag-thresholds/Cargo.toml @@ -10,15 +10,15 @@ description = "Bag threshold generation script for pallet-bag-list and node-runt readme = "README.md" [dependencies] -node-runtime = { version = "3.0.0-dev", path = ".." } +node-runtime = { version = "3.0.0-dev", path = "../../../bin/node/runtime" } # FRAME -frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/support" } -pallet-bags-list = { version = "4.0.0-dev", path = "../../../../frame/bags-list", features = ["make-bags"] } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/staking" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } +pallet-bags-list = { version = "4.0.0-dev", path = "../../bags-list", features = ["make-bags"] } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../staking" } # primitives -sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } +sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } -# third-party +# third-party structopt = "0.3.21" diff --git a/bin/node/runtime/bag-thresholds/src/main.rs b/frame/bags-list/bag-thresholds/src/main.rs similarity index 100% rename from bin/node/runtime/bag-thresholds/src/main.rs rename to frame/bags-list/bag-thresholds/src/main.rs From 6d7bf29ec2df13ad474004d95e1bfb4066144810 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 14:16:06 -0700 Subject: [PATCH 130/241] Move bag-thresholds bin to within pallet-bags --- Cargo.lock | 1 + frame/bags-list/Cargo.toml | 2 ++ frame/bags-list/bag-thresholds/src/main.rs | 16 +------------ frame/bags-list/src/make_bags.rs | 26 +++++++++++----------- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b6d0790cd86c..bf85b5935c16f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4893,6 +4893,7 @@ dependencies = [ "log", "num-format", "pallet-balances", + "pallet-staking", "parity-scale-codec", "sp-core", "sp-io", diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index f9175a16f2ce1..7c78087624c50 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -39,6 +39,7 @@ sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optiona chrono = { version = "0.4.19", optional = true } git2 = { version = "0.13.20", default-features = false, optional = true } num-format = { version = "0.4.0", optional = true } +pallet-staking = { version = "4.0.0-dev", path = "../staking", optional = true } [dev-dependencies] sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} @@ -71,4 +72,5 @@ make-bags = [ "git2", "num-format", "std", + "pallet-staking" ] diff --git a/frame/bags-list/bag-thresholds/src/main.rs b/frame/bags-list/bag-thresholds/src/main.rs index da4df88d43ae7..b573d962d0c4e 100644 --- a/frame/bags-list/bag-thresholds/src/main.rs +++ b/frame/bags-list/bag-thresholds/src/main.rs @@ -20,9 +20,6 @@ use pallet_bags_list::make_bags::generate_thresholds_module; use std::path::PathBuf; use structopt::StructOpt; -use node_runtime::Runtime; -use frame_support::traits::{Currency, CurrencyToVote}; - #[derive(Debug, StructOpt)] struct Opt { /// How many bags to generate. @@ -37,16 +34,5 @@ fn main() -> Result<(), std::io::Error> { let Opt { n_bags, output } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - - - let res = ext.execute_with(||{ - // let existential_deposit = >::minimum_balance(); - let existential_deposit = ::Currency::minimum_balance(); - let issuance = ::Currency::total_issuance(); - let existential_weight = ::CurrencyToVote::to_vote(existential_deposit, issuance); - - generate_thresholds_module::(n_bags, existential_weight, &output); - }); - - Ok(res) + ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) } diff --git a/frame/bags-list/src/make_bags.rs b/frame/bags-list/src/make_bags.rs index 40dc25d86338c..037447d7445b3 100644 --- a/frame/bags-list/src/make_bags.rs +++ b/frame/bags-list/src/make_bags.rs @@ -17,6 +17,10 @@ //! Support code to ease the process of generating voter bags. //! +//! NOTE: this assume the runtime implements [`pallet_staking::Config`], as it requires an +//! implementation of the traits [`frame_support::traits::Currency`] and +//! [`frame_support::traits::Currency`]. +//! //! The process of adding voter bags to a runtime requires only four steps. //! //! 1. Update the runtime definition. @@ -50,7 +54,6 @@ //! + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; //! ``` -use crate::Config; use frame_election_provider_support::VoteWeight; use frame_support::traits::Get; use std::{ @@ -63,15 +66,14 @@ use std::{ /// Note that this value depends on the current issuance, a quantity known to change over time. /// This makes the project of computing a static value suitable for inclusion in a static, /// generated file _excitingly unstable_. -// TODO we can't access currency or currency to weight from this module -// #[cfg(any(feature = "std", feature = "make-bags"))] -// fn existential_weight() -> VoteWeight { -// use frame_support::traits::{Currency, CurrencyToVote}; +#[cfg(any(feature = "std", feature = "make-bags"))] +fn existential_weight() -> VoteWeight { + use frame_support::traits::{Currency, CurrencyToVote}; -// let existential_deposit = >::minimum_balance(); -// let issuance = >::total_issuance(); -// T::CurrencyToVote::to_vote(existential_deposit, issuance) -// } + let existential_deposit = >::minimum_balance(); + let issuance = >::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) +} /// Return the path to a header file used in this repository if is exists. /// @@ -158,9 +160,8 @@ pub fn thresholds( /// - Module documentation noting that this is autogenerated and when. /// - Some associated constants. /// - The constant array of thresholds. -pub fn generate_thresholds_module( +pub fn generate_thresholds_module( n_bags: usize, - existential_weight: VoteWeight, output: &Path, ) -> Result<(), std::io::Error> { // ensure the file is accessable @@ -195,8 +196,7 @@ pub fn generate_thresholds_module( ::Version::get().spec_name, )?; - // existential weight TODO can't get existential weight since we don't have currency module. - // let existential_weight = existential_weight::(); + let existential_weight = existential_weight::(); num_buf.write_formatted(&existential_weight, &format); writeln!(buf)?; writeln!(buf, "/// Existential weight for this runtime.")?; From fb79356ebed64dede93e0b9b652c28e1ec172053 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 14:45:29 -0700 Subject: [PATCH 131/241] Remove some unnescary TODOs --- frame/bags-list/src/list/mod.rs | 2 +- frame/bags-list/src/list/tests.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index cdf6df0c41f63..4302e4ace8658 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -219,7 +219,7 @@ impl List { count += 1; }); - count // TODO check if this is really necessary + count } /// Insert a new voter into the appropriate bag in the voter list. Does not check for duplicates. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 5a545f66e5748..e90ac1f0d0c01 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -238,7 +238,6 @@ mod voter_list { // get the new updated node; try and update the position with no change in weight. let node = Node::::get(&1).unwrap(); - // TODO: we can pass a ref to node to this function as well. assert_storage_noop!(assert_eq!( List::::update_position_for(node.clone(), 20), None From dc2b5932b676d23e6410cf23ff019d68f4b25371 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 15:52:22 -0700 Subject: [PATCH 132/241] Impl tets wrong_rebag_is_noop --- frame/bags-list/src/lib.rs | 2 +- frame/bags-list/src/list/mod.rs | 2 +- frame/bags-list/src/tests.rs | 20 ++++++++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 9a51ff11e7eb0..ddacc4adc60fc 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -203,7 +203,7 @@ pub mod pallet { pub fn rebag(origin: OriginFor, reposition: T::AccountId) -> DispatchResult { ensure_signed(origin)?; let current_weight = T::VoteWeightProvider::vote_weight(&reposition); - Pallet::::do_rebag(&reposition, current_weight); + let _ = Pallet::::do_rebag(&reposition, current_weight); Ok(()) } } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 4302e4ace8658..f84bdc8459522 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -616,7 +616,7 @@ impl Node { } /// `true` when this voter is in the wrong bag. - fn is_misplaced(&self, current_weight: VoteWeight) -> bool { + pub(crate) fn is_misplaced(&self, current_weight: VoteWeight) -> bool { notional_bag_for::(current_weight) != self.bag_upper } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 8876eebfef1ac..25ae162ba4174 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,4 +1,4 @@ -use frame_support::assert_ok; +use frame_support::{assert_ok, assert_storage_noop}; use super::*; use frame_election_provider_support::SortedListProvider; @@ -110,9 +110,21 @@ mod pallet { #[test] fn wrong_rebag_is_noop() { - // if rebag is wrong, - // or the target is non-existent, then it is a noop - todo!() + ExtBuilder::default().build_and_execute(|| { + let node_3 = list::Node::::get(&3).unwrap(); + // when account 3 is _not_ misplaced with weight 500 + NextVoteWeight::set(500); + assert!(!node_3.is_misplaced(500)); + + // then calling rebag on account 3 with weight 500 is a noop + assert_storage_noop!(assert_eq!(BagsList::rebag(Origin::signed(0), 3), Ok(()))); + + // when account 42 is not in the list + assert!(!BagsList::contains(&42)); + + // then rebag-ing account 42 is a noop + assert_storage_noop!(assert_eq!(BagsList::rebag(Origin::signed(0), 42), Ok(()))); + }); } #[test] From e68a9f24d549451e246df69405214191d25bc74b Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 16:25:27 -0700 Subject: [PATCH 133/241] assert remove is a noop with bad data --- frame/bags-list/src/list/mod.rs | 2 -- frame/bags-list/src/list/tests.rs | 23 ++++++++++++++++------- frame/bags-list/src/tests.rs | 7 ++----- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index f84bdc8459522..6c7e4e4c42cf0 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -263,8 +263,6 @@ impl List { let mut bags = BTreeMap::new(); let mut count = 0; - // TODO: assert that it is noop with bad data. - for voter_id in voters.into_iter() { let node = match Node::::get(voter_id) { Some(node) => node, diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index e90ac1f0d0c01..01a5926ff0c5d 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -3,6 +3,7 @@ use crate::{ mock::{ext_builder::*, test_utils::*, *}, CounterForVoters, VoterBags, VoterNodes, }; +use frame_election_provider_support::SortedListProvider; use frame_support::{assert_ok, assert_storage_noop}; #[test] @@ -180,14 +181,9 @@ mod voter_list { }; ExtBuilder::default().build_and_execute(|| { - // when removing a non-existent voter + // removing a non-existent voter is a noop assert!(!VoterNodes::::contains_key(42)); - List::::remove(&42); - - // then nothing changes - assert_eq!(get_list_as_ids(), vec![2, 3, 4, 1]); - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); - assert_eq!(CounterForVoters::::get(), 4); + assert_storage_noop!(List::::remove(&42)); // when removing a node from a bag with multiple nodes List::::remove(&2); @@ -221,6 +217,19 @@ mod voter_list { }); } + #[test] + fn remove_many_is_noop_with_non_existent_ids() { + ExtBuilder::default().build_and_execute(|| { + let non_existent_ids = vec![&42, &666, &13]; + + // when account ids don' exist in the list + assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); + + // then removing them is a noop + assert_storage_noop!(List::::remove_many(non_existent_ids)); + }); + } + #[test] fn update_position_for_works() { ExtBuilder::default().build_and_execute(|| { diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 25ae162ba4174..5ff85c9e27985 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -275,12 +275,9 @@ mod sorted_list_provider { }; ExtBuilder::default().build_and_execute(|| { - // when removing a non-existent voter + // it is a noop removing a non-existent voter assert!(!VoterNodes::::contains_key(42)); - >::on_remove(&42); - - // then nothing changes - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + assert_storage_noop!(BagsList::on_remove(&42)); // when removing a node from a bag with multiple nodes >::on_remove(&2); From 16ff8ba686fd6114d758228eb3b3fc8b3d0b529e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 17:19:42 -0700 Subject: [PATCH 134/241] Assert integrity test panics --- frame/bags-list/src/mock.rs | 6 +++++- frame/bags-list/src/tests.rs | 13 +++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 4e019495cf7f5..242e820ef06ad 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -51,7 +51,7 @@ impl frame_system::Config for Runtime { const THRESHOLDS: [VoteWeight; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; parameter_types! { - pub const BagThresholds: &'static [VoteWeight] = &THRESHOLDS; + pub static BagThresholds: &'static [VoteWeight] = &THRESHOLDS; } impl crate::Config for Runtime { @@ -143,4 +143,8 @@ pub(crate) mod test_utils { pub(crate) fn get_list_as_ids() -> Vec { List::::iter().map(|n| *n.id()).collect::>() } + + pub(crate) fn set_bag_thresholds(thresholds: &'static [VoteWeight]) { + BagThresholds::set(thresholds); + } } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 5ff85c9e27985..a309ac98e9981 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,4 +1,4 @@ -use frame_support::{assert_ok, assert_storage_noop}; +use frame_support::{assert_ok, assert_storage_noop, traits::IntegrityTest}; use super::*; use frame_election_provider_support::SortedListProvider; @@ -128,14 +128,19 @@ mod pallet { } #[test] + #[should_panic = "Voter bag thresholds must strictly increase, and have no duplicates"] fn duplicate_in_bags_threshold_panics() { - todo!() - // probably needs some UI test + const DUPE_THRESH: &[VoteWeight; 4] = &[10, 20, 30, 30]; + set_bag_thresholds(DUPE_THRESH); + BagsList::integrity_test(); } #[test] + #[should_panic = "Voter bag thresholds must strictly increase, and have no duplicates"] fn decreasing_in_bags_threshold_panics() { - todo!() + const DECREASING_THRESH: &[VoteWeight; 4] = &[10, 30, 20, 40]; + set_bag_thresholds(DECREASING_THRESH); + BagsList::integrity_test(); } } From 4ff45514a620cd3e9270165814c828c7b824fe32 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 18:14:12 -0700 Subject: [PATCH 135/241] Return an error when inserting duplicates --- frame/bags-list/src/benchmarks.rs | 10 +-- frame/bags-list/src/lib.rs | 15 +++-- frame/bags-list/src/list/mod.rs | 33 ++++----- frame/bags-list/src/list/tests.rs | 30 +++++++-- frame/bags-list/src/mock.rs | 4 +- frame/bags-list/src/tests.rs | 78 ++++++++-------------- frame/bags-list/src/weights.rs | 2 +- frame/election-provider-support/src/lib.rs | 2 +- frame/staking/src/pallet/impls.rs | 7 +- 9 files changed, 94 insertions(+), 87 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 69264ea98eee3..095f45a9f7529 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -5,7 +5,7 @@ use crate::list::{Bag, List}; // use frame_benchmarking::{account, whitelisted_caller}; use frame_benchmarking::{account, whitelisted_caller}; use frame_election_provider_support::VoteWeightProvider; -use frame_support::traits::Get; +use frame_support::{assert_ok, traits::Get}; use frame_system::RawOrigin as SystemOrigin; fn get_bags() -> Vec<(VoteWeight, Vec)> { @@ -37,17 +37,17 @@ frame_benchmarking::benchmarks! { // seed items in the origin bag. let origin_head: T::AccountId = account("origin_head", 0, 0); - List::::insert(origin_head.clone(), origin_bag_thresh); + assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); let origin_middle: T::AccountId = account("origin_middle", 0, 0); - List::::insert(origin_middle.clone(), origin_bag_thresh); + assert_ok!(List::::insert(origin_middle.clone(), origin_bag_thresh)); let origin_tail: T::AccountId = account("origin_tail", 0, 0); - List::::insert(origin_tail.clone(), origin_bag_thresh); + assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); // seed items in the destination bag. let dest_head: T::AccountId = account("dest_head", 0, 0); - List::::insert(dest_head.clone(), dest_bag_thresh); + assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); // and the ags are in the expected state after insertions. assert_eq!( diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index ddacc4adc60fc..e94b2f5888293 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -29,7 +29,9 @@ //! # ⚠️ WARNING ⚠️ //! //! Do not insert an id that already exists in the list; doing so can result in catastrophic failure -//! of your blockchain, including entering into an infinite loop during block execution. +//! of your blockchain, including entering into an infinite loop during block execution. This +//! situation can be avoided by strictly using [`Pallet::::on_insert`] for insertions because +//! it will exit early and return an error if the id is a duplicate. //! //! # Goals //! @@ -48,8 +50,9 @@ //! re-inserting it will worsen its position in list iteration; this reduces incentives for some //! types of spam that involve consistently removing and inserting for better position. Further, //! ordering granularity is thus dictated by range between each bag threshold. -//! - if an items weight changes to a value no longer within the range of its current bag its -//! position will need to be updated by an external actor with rebag or removal & insertion. +//! - if an items weight changes to a value no longer within the range of its current bag the item's +//! position will need to be updated by an external actor with rebag (update), or removal and +//! insertion. // //! ### Further Plans //! @@ -249,11 +252,11 @@ impl SortedListProvider for Pallet { } fn contains(voter: &T::AccountId) -> bool { - VoterNodes::::contains_key(voter) + List::::contains(voter) } - fn on_insert(voter: T::AccountId, weight: VoteWeight) { - List::::insert(voter, weight); + fn on_insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), ()> { + List::::insert(voter, weight) } fn on_update(voter: &T::AccountId, new_weight: VoteWeight) { diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 6c7e4e4c42cf0..9d29f9fd35337 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -176,6 +176,11 @@ impl List { num_affected } + /// Returns true if the list contains `id`, otherwise returns `false`. + pub(crate) fn contains(voter: &T::AccountId) -> bool { + crate::VoterNodes::::contains_key(voter) + } + /// Iterate over all nodes in all bags in the voter list. /// /// Full iteration can be expensive; it's recommended to limit the number of items with @@ -200,14 +205,9 @@ impl List { } /// Insert several voters into the appropriate bags in the voter list. Does not check for - /// duplicates. + /// duplicates; instead continues with insertions if duplicates are detected. /// /// This is more efficient than repeated calls to `Self::insert`. - /// - /// # ⚠️ WARNING ⚠️ - /// - /// Do not insert an id that already exists in the list; doing so can result in catastrophic - /// failure of your blockchain, including entering into an infinite loop during block execution. fn insert_many( voters: impl IntoIterator, weight_of: impl Fn(&T::AccountId) -> VoteWeight, @@ -215,22 +215,21 @@ impl List { let mut count = 0; voters.into_iter().for_each(|v| { let weight = weight_of(&v); - Self::insert(v, weight); - count += 1; + if Self::insert(v, weight).is_ok() { + count += 1; + } }); count } - /// Insert a new voter into the appropriate bag in the voter list. Does not check for duplicates. - /// - /// # ⚠️ WARNING ⚠️ + /// Insert a new voter into the appropriate bag in the voter list. /// - /// Do not insert an id that already exists in the list; doing so can result in catastrophic - /// failure of your blockchain, including entering into an infinite loop during block execution. - pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) { - // TODO: can check if this voter exists as a node by checking if `voter` exists in the nodes - // map and return early if it does. + /// Returns an error if the list already contains `voter`. + pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), ()> { + if Self::contains(&voter) { + return Err(()); + } let bag_weight = notional_bag_for::(weight); crate::log!( @@ -249,6 +248,8 @@ impl List { crate::CounterForVoters::::mutate(|prev_count| { *prev_count = prev_count.saturating_add(1) }); + + Ok(()) } /// Remove a voter (by id) from the voter list. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 01a5926ff0c5d..bb921a739ab1e 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -75,13 +75,13 @@ fn remove_last_voter_in_bags_cleans_bag() { // bump 1 to a bigger bag List::::remove(&1); - List::::insert(1, 10_000); + assert_ok!(List::::insert(1, 10_000)); // then the bag with bound 10 is wiped from storage. assert_eq!(get_bags(), vec![(1_000, vec![2, 3, 4]), (10_000, vec![1])]); // and can be recreated again as needed. - List::::insert(77, 10); + assert_ok!(List::::insert(77, 10)); assert_eq!(get_bags(), vec![(10, vec![77]), (1_000, vec![2, 3, 4]), (10_000, vec![1])]); }); } @@ -111,7 +111,7 @@ mod voter_list { ); // when adding a voter that has a higher weight than pre-existing voters in the bag - List::::insert(7, 10); + assert_ok!(List::::insert(7, 10)); // then assert_eq!( @@ -156,14 +156,14 @@ mod voter_list { fn insert_works() { ExtBuilder::default().build_and_execute(|| { // when inserting into an existing bag - List::::insert(5, 1_000); + assert_ok!(List::::insert(5, 1_000)); // then assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5])]); assert_eq!(get_list_as_ids(), vec![2, 3, 4, 5, 1]); // when inserting into a non-existent bag - List::::insert(6, 1_001); + assert_ok!(List::::insert(6, 1_001)); // then assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5]), (2000, vec![6])]); @@ -268,6 +268,26 @@ mod voter_list { )); }); } + + #[test] + fn sanity_check_works() { + ExtBuilder::default().build_and_execute_no_post_check(|| { + assert_ok!(List::::sanity_check()); + }); + + // make sure there are no duplicates. + ExtBuilder::default().build_and_execute_no_post_check(|| { + Bag::::get(10).unwrap().insert(2); + assert_eq!(List::::sanity_check(), Err("duplicate identified")); + }); + + // ensure count is in sync with `CounterForVoters`. + ExtBuilder::default().build_and_execute_no_post_check(|| { + crate::CounterForVoters::::mutate(|counter| *counter += 1); + assert_eq!(crate::CounterForVoters::::get(), 5); + assert_eq!(List::::sanity_check(), Err("iter_count != stored_count")); + }); + } } mod bags { diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 242e820ef06ad..370fa10c336d9 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -100,7 +100,9 @@ pub(crate) mod ext_builder { let mut ext = sp_io::TestExternalities::from(storage); ext.execute_with(|| { for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { - List::::insert(*id, *weight); + frame_support::assert_ok!( + List::::insert(*id, *weight) + ); } }); diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index a309ac98e9981..6ac61b77aad8d 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -151,7 +151,7 @@ mod sorted_list_provider { fn iter_works() { ExtBuilder::default().build_and_execute(|| { let expected = vec![2, 3, 4, 1]; - for (i, id) in >::iter().enumerate() { + for (i, id) in BagsList::iter().enumerate() { assert_eq!(id, expected[i]) } }); @@ -161,22 +161,22 @@ mod sorted_list_provider { fn count_works() { ExtBuilder::default().build_and_execute(|| { // given - assert_eq!(>::count(), 4); + assert_eq!(BagsList::count(), 4); // when inserting - >::on_insert(201, 0); + assert_ok!(BagsList::on_insert(201, 0)); // then the count goes up - assert_eq!(>::count(), 5); + assert_eq!(BagsList::count(), 5); // when removing - >::on_remove(&201); + BagsList::on_remove(&201); // then the count goes down - assert_eq!(>::count(), 4); + assert_eq!(BagsList::count(), 4); // when updating - >::on_update(&201, VoteWeight::MAX); + BagsList::on_update(&201, VoteWeight::MAX); // then the count stays the same - assert_eq!(>::count(), 4); + assert_eq!(BagsList::count(), 4); }); } @@ -184,30 +184,30 @@ mod sorted_list_provider { fn on_insert_works() { ExtBuilder::default().build_and_execute(|| { // when - >::on_insert(6, 1_000); + assert_ok!(BagsList::on_insert(6, 1_000)); // then the bags assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6])]); // and list correctly include the new id, assert_eq!( - >::iter().collect::>(), + BagsList::iter().collect::>(), vec![2, 3, 4, 6, 1] ); // and the count is incremented. - assert_eq!(>::count(), 5); + assert_eq!(BagsList::count(), 5); // when - List::::insert(7, 1_001); + assert_ok!(BagsList::on_insert(7, 1_001)); // then the bags assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2000, vec![7])]); // and list correctly include the new id, assert_eq!( - >::iter().collect::>(), + BagsList::iter().collect::>(), vec![7, 2, 3, 4, 6, 1] ); // and the count is incremented. - assert_eq!(>::count(), 6); + assert_eq!(BagsList::count(), 6); }) } @@ -216,32 +216,32 @@ mod sorted_list_provider { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])]); - assert_eq!(>::count(), 5); + assert_eq!(BagsList::count(), 5); // when increasing weight to the level of non-existent bag - >::on_update(&42, 2_000); + BagsList::on_update(&42, 2_000); // then the bag is created with the voter in it, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); // and the id position is updated in the list. assert_eq!( - >::iter().collect::>(), + BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1] ); // when decreasing weight within the range of the current bag - >::on_update(&42, 1_001); + BagsList::on_update(&42, 1_001); // then the voter does not change bags, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); // or change position in the list. assert_eq!( - >::iter().collect::>(), + BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1] ); // when increasing weight to the level of a non-existent bag with the max threshold - >::on_update(&42, VoteWeight::MAX); + BagsList::on_update(&42, VoteWeight::MAX); // the the new bag is created with the voter in it, assert_eq!( @@ -250,23 +250,23 @@ mod sorted_list_provider { ); // and the id position is updated in the list. assert_eq!( - >::iter().collect::>(), + BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1] ); // when decreasing the weight to a pre-existing bag - >::on_update(&42, 1_000); + BagsList::on_update(&42, 1_000); // then voter is moved to the correct bag (as the last member), assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); // and the id position is updated in the list. assert_eq!( - >::iter().collect::>(), + BagsList::iter().collect::>(), vec![2, 3, 4, 42, 1] ); // since we have only called on_update, the `count` has not changed. - assert_eq!(>::count(), 5); + assert_eq!(BagsList::count(), 5); }); } @@ -274,7 +274,7 @@ mod sorted_list_provider { fn on_remove_works() { let ensure_left = |id, counter| { assert!(!VoterNodes::::contains_key(id)); - assert_eq!(>::count(), counter); + assert_eq!(BagsList::count(), counter); assert_eq!(CounterForVoters::::get(), counter); assert_eq!(VoterNodes::::iter().count() as u32, counter); }; @@ -285,7 +285,7 @@ mod sorted_list_provider { assert_storage_noop!(BagsList::on_remove(&42)); // when removing a node from a bag with multiple nodes - >::on_remove(&2); + BagsList::on_remove(&2); // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); @@ -293,7 +293,7 @@ mod sorted_list_provider { ensure_left(2, 3); // when removing a node from a bag with only one node - >::on_remove(&1); + BagsList::on_remove(&1); // then assert_eq!(get_list_as_ids(), vec![3, 4]); @@ -301,34 +301,14 @@ mod sorted_list_provider { ensure_left(1, 2); // when removing all remaining voters - >::on_remove(&4); + BagsList::on_remove(&4); assert_eq!(get_list_as_ids(), vec![3]); ensure_left(4, 1); - >::on_remove(&3); + BagsList::on_remove(&3); // then the storage is completely cleaned up assert_eq!(get_list_as_ids(), Vec::::new()); ensure_left(3, 0); }); } - - #[test] - fn sanity_check_works() { - ExtBuilder::default().build_and_execute_no_post_check(|| { - assert_ok!(List::::sanity_check()); - }); - - // make sure there are no duplicates. - ExtBuilder::default().build_and_execute_no_post_check(|| { - >::on_insert(2, 10); - assert_eq!(List::::sanity_check(), Err("duplicate identified")); - }); - - // ensure count is in sync with `CounterForVoters`. - ExtBuilder::default().build_and_execute_no_post_check(|| { - crate::CounterForVoters::::mutate(|counter| *counter += 1); - assert_eq!(crate::CounterForVoters::::get(), 5); - assert_eq!(List::::sanity_check(), Err("iter_count != stored_count")); - }); - } } diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index e3db1b1e57881..cd58fd247ff53 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -1,4 +1,4 @@ -use frame_support::{pallet_prelude::Weight, weights::constants::RocksDbWeight}; +use frame_support::{pallet_prelude::Weight}; pub trait WeightInfo { fn rebag() -> Weight; diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index d30079b6ba113..2236f23526f88 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -313,7 +313,7 @@ pub trait SortedListProvider { /// Return true if the insertion can happen. fn contains(voter: &AccountId) -> bool; // Hook for inserting a voter. - fn on_insert(voter: AccountId, weight: VoteWeight); + fn on_insert(voter: AccountId, weight: VoteWeight) -> Result<(), ()>; /// Hook for updating a single voter. fn on_update(voter: &AccountId, weight: VoteWeight); /// Hook for removing a voter from the list. diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index a13ff896949b0..59c7024dfcaad 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1247,14 +1247,15 @@ impl SortedListProvider for UseNominatorsMap { fn contains(voter: &T::AccountId) -> bool { Nominators::::contains_key(voter) } - fn on_insert(_voter: T::AccountId, _weight: VoteWeight) { - // nothing to do on update. + fn on_insert(_voter: T::AccountId, _weight: VoteWeight) -> Result<(), ()>{ + // nothing to do on insert. + Ok(()) } fn on_update(_voter: &T::AccountId, _weight: VoteWeight) { // nothing to do on update. } fn on_remove(_voter: &T::AccountId) { - // nothing to do on update. + // nothing to do on remove. } fn regenerate( _: impl IntoIterator, From 89bdefbca679d374c9a14997455a0ea0bbeccff3 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 18:58:25 -0700 Subject: [PATCH 136/241] Update to handle error in staking pallet when inserting to list --- frame/bags-list/src/list/mod.rs | 2 +- frame/bags-list/src/mock.rs | 4 +--- frame/bags-list/src/tests.rs | 30 ++++++-------------------- frame/bags-list/src/weights.rs | 2 +- frame/staking/src/pallet/impls.rs | 36 +++++++++++++++++++++++++++---- frame/staking/src/pallet/mod.rs | 5 +---- 6 files changed, 42 insertions(+), 37 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 9d29f9fd35337..b586a75254f71 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -228,7 +228,7 @@ impl List { /// Returns an error if the list already contains `voter`. pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), ()> { if Self::contains(&voter) { - return Err(()); + return Err(()) } let bag_weight = notional_bag_for::(weight); diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 370fa10c336d9..0ebedd18b4d2e 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -100,9 +100,7 @@ pub(crate) mod ext_builder { let mut ext = sp_io::TestExternalities::from(storage); ext.execute_with(|| { for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { - frame_support::assert_ok!( - List::::insert(*id, *weight) - ); + frame_support::assert_ok!(List::::insert(*id, *weight)); } }); diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 6ac61b77aad8d..8d48bd44999b1 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -189,10 +189,7 @@ mod sorted_list_provider { // then the bags assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6])]); // and list correctly include the new id, - assert_eq!( - BagsList::iter().collect::>(), - vec![2, 3, 4, 6, 1] - ); + assert_eq!(BagsList::iter().collect::>(), vec![2, 3, 4, 6, 1]); // and the count is incremented. assert_eq!(BagsList::count(), 5); @@ -202,10 +199,7 @@ mod sorted_list_provider { // then the bags assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2000, vec![7])]); // and list correctly include the new id, - assert_eq!( - BagsList::iter().collect::>(), - vec![7, 2, 3, 4, 6, 1] - ); + assert_eq!(BagsList::iter().collect::>(), vec![7, 2, 3, 4, 6, 1]); // and the count is incremented. assert_eq!(BagsList::count(), 6); }) @@ -224,10 +218,7 @@ mod sorted_list_provider { // then the bag is created with the voter in it, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); // and the id position is updated in the list. - assert_eq!( - BagsList::iter().collect::>(), - vec![42, 2, 3, 4, 1] - ); + assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); // when decreasing weight within the range of the current bag BagsList::on_update(&42, 1_001); @@ -235,10 +226,7 @@ mod sorted_list_provider { // then the voter does not change bags, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); // or change position in the list. - assert_eq!( - BagsList::iter().collect::>(), - vec![42, 2, 3, 4, 1] - ); + assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); // when increasing weight to the level of a non-existent bag with the max threshold BagsList::on_update(&42, VoteWeight::MAX); @@ -249,10 +237,7 @@ mod sorted_list_provider { vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (VoteWeight::MAX, vec![42])] ); // and the id position is updated in the list. - assert_eq!( - BagsList::iter().collect::>(), - vec![42, 2, 3, 4, 1] - ); + assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); // when decreasing the weight to a pre-existing bag BagsList::on_update(&42, 1_000); @@ -260,10 +245,7 @@ mod sorted_list_provider { // then voter is moved to the correct bag (as the last member), assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); // and the id position is updated in the list. - assert_eq!( - BagsList::iter().collect::>(), - vec![2, 3, 4, 42, 1] - ); + assert_eq!(BagsList::iter().collect::>(), vec![2, 3, 4, 42, 1]); // since we have only called on_update, the `count` has not changed. assert_eq!(BagsList::count(), 5); diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index cd58fd247ff53..0beabb2d49e31 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -1,4 +1,4 @@ -use frame_support::{pallet_prelude::Weight}; +use frame_support::pallet_prelude::Weight; pub trait WeightInfo { fn rebag() -> Weight; diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 59c7024dfcaad..ccbb9bb1049f7 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -724,12 +724,38 @@ impl Pallet { pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { CounterForNominators::::mutate(|x| x.saturating_inc()); + + // TODO: should we only mutate the counter if they are in not Nominators && SortedListProvider? + // To preserve current behavior not stopping adding them if SortedListProvider has issues + if !T::SortedListProvider::contains(who) { - T::SortedListProvider::on_insert(who.clone(), Self::weight_of_fn()(who)); - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { + // insertion only fails if the voter is "contained" in `SortedListProvider`. + // Since we check if the voter is in the list above getting here is + // impossible. + log!( + warn, + "attempt to insert duplicate nominator ({:#?}) despite `List::contain` return false", + who + ); + debug_assert!(false, "attempt to insert duplicate nominator despite `List::contain` return false"); + }; } else { - // TODO: emit a lot of warnings. + // we are in a very bad situation + log!( + warn, + "Nominators and SortedListProvider out of sync: tried to insert into list a duplicate nominator ({:#?})", + who + ); + debug_assert!(false, "Nominators and SortedListProvider out of sync: tried to insert into list a duplicate nominator"); } + + debug_assert_eq!( + CounterForNominators::::get(), + T::SortedListProvider::count(), + "voter_count must be accurate", + ); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } // TODO: test: re-nominate should not increase the counter and T::SortedListProvider::count() // consider these cases as well: @@ -754,6 +780,7 @@ impl Pallet { // - bond-extra // - unbond // - rebond + Nominators::::insert(who, nominations); } @@ -771,6 +798,7 @@ impl Pallet { CounterForNominators::::mutate(|x| x.saturating_dec()); T::SortedListProvider::on_remove(who); debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + debug_assert_eq!(CounterForNominators::::get(), T::SortedListProvider::count()); true } else { false @@ -1247,7 +1275,7 @@ impl SortedListProvider for UseNominatorsMap { fn contains(voter: &T::AccountId) -> bool { Nominators::::contains_key(voter) } - fn on_insert(_voter: T::AccountId, _weight: VoteWeight) -> Result<(), ()>{ + fn on_insert(_voter: T::AccountId, _weight: VoteWeight) -> Result<(), ()> { // nothing to do on insert. Ok(()) } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 3905853cbd47f..1dd17caea8ca5 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -24,10 +24,7 @@ use frame_support::{ Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, LockIdentifier, LockableCurrency, OnUnbalanced, UnixTime, }, - weights::{ - constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}, - Weight, - }, + weights::Weight, }; use frame_system::{ensure_root, ensure_signed, offchain::SendTransactionTypes, pallet_prelude::*}; use sp_runtime::{ From 168295e4f11796d07c915fd34a2a05b5a632a7dd Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 19:21:18 -0700 Subject: [PATCH 137/241] Test contains and on_insert error --- frame/bags-list/src/list/tests.rs | 23 ++++++++++++++++++++++- frame/bags-list/src/mock.rs | 2 +- frame/bags-list/src/tests.rs | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index bb921a739ab1e..ff507e6cd678d 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -171,6 +171,17 @@ mod voter_list { }); } + #[test] + fn insert_errors_with_duplicate_id() { + ExtBuilder::default().build_and_execute(|| { + // given + assert!(get_list_as_ids().contains(&3)); + + // then + assert_storage_noop!(assert_eq!(List::::insert(3, 20), Err(()))); + }); + } + #[test] fn remove_works() { use crate::{CounterForVoters, VoterBags, VoterNodes}; @@ -288,6 +299,17 @@ mod voter_list { assert_eq!(List::::sanity_check(), Err("iter_count != stored_count")); }); } + + #[test] + fn contains_works() { + ExtBuilder::default().build_and_execute(|| { + assert!(ext_builder::GENESIS_IDS.iter().all(|(id, _)| List::::contains(id))); + + let non_existent_ids = vec![&42, &666, &13]; + assert!(non_existent_ids.iter().all(|id| !List::::contains(id))); + }) + + } } mod bags { @@ -433,7 +455,6 @@ mod bags { assert_eq!(get_list_as_ids(), vec![2, 3, 1]); // and the last accessible node has an **incorrect** prev pointer. - // TODO: consider doing a check on insert, it is cheap. assert_eq!( Node::::get(&3).unwrap(), node(3, Some(4), None, bag_1000.bag_upper) diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 0ebedd18b4d2e..cc2c8806f88ce 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -78,7 +78,7 @@ pub(crate) mod ext_builder { use super::*; /// Default AccountIds and their weights. - const GENESIS_IDS: [(AccountId, VoteWeight); 4] = [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; + pub const GENESIS_IDS: [(AccountId, VoteWeight); 4] = [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; #[derive(Default)] pub(crate) struct ExtBuilder { diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 8d48bd44999b1..d97d1621e4715 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -205,6 +205,17 @@ mod sorted_list_provider { }) } + #[test] + fn on_insert_errors_with_duplicate_id() { + ExtBuilder::default().build_and_execute(|| { + // given + assert!(get_list_as_ids().contains(&3)); + + // then + assert_storage_noop!(assert_eq!(BagsList::on_insert(3, 20), Err(()))); + }); + } + #[test] fn on_update_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { @@ -293,4 +304,15 @@ mod sorted_list_provider { ensure_left(3, 0); }); } + + #[test] + fn contains_works() { + ExtBuilder::default().build_and_execute(|| { + assert!(ext_builder::GENESIS_IDS.iter().all(|(id, _)| BagsList::contains(id))); + + let non_existent_ids = vec![&42, &666, &13]; + assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); + }) + + } } From 4c3d6895f7c5e659879b84e4038c5743bc11ac33 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 5 Aug 2021 20:06:40 -0700 Subject: [PATCH 138/241] Test re-nominate does not mess up list or count --- frame/bags-list/src/list/tests.rs | 1 - frame/bags-list/src/mock.rs | 3 ++- frame/bags-list/src/tests.rs | 1 - frame/staking/src/pallet/impls.rs | 8 +------- frame/staking/src/tests.rs | 20 ++++++++++++++++++++ 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index ff507e6cd678d..484ea23ee60ed 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -308,7 +308,6 @@ mod voter_list { let non_existent_ids = vec![&42, &666, &13]; assert!(non_existent_ids.iter().all(|id| !List::::contains(id))); }) - } } diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index cc2c8806f88ce..a6df7c07dece8 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -78,7 +78,8 @@ pub(crate) mod ext_builder { use super::*; /// Default AccountIds and their weights. - pub const GENESIS_IDS: [(AccountId, VoteWeight); 4] = [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; + pub(crate) const GENESIS_IDS: [(AccountId, VoteWeight); 4] = + [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; #[derive(Default)] pub(crate) struct ExtBuilder { diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index d97d1621e4715..4d541b0530c9e 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -313,6 +313,5 @@ mod sorted_list_provider { let non_existent_ids = vec![&42, &666, &13]; assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); }) - } } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index ccbb9bb1049f7..b73450420eedb 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -750,15 +750,9 @@ impl Pallet { debug_assert!(false, "Nominators and SortedListProvider out of sync: tried to insert into list a duplicate nominator"); } - debug_assert_eq!( - CounterForNominators::::get(), - T::SortedListProvider::count(), - "voter_count must be accurate", - ); debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } - // TODO: test: re-nominate should not increase the counter and T::SortedListProvider::count() - // consider these cases as well: + // TODO: // chilled // - validate // - nominate diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 85193c2a54046..73948178bc4b9 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -4318,3 +4318,23 @@ mod election_data_provider { }) } } + +#[test] +fn re_nominate_does_not_change_counters_or_list() { + ExtBuilder::default().nominate(true).build_and_execute(|| { + // given + let pre_insert_nominator_count = Nominators::::iter().count() as u32; + assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); + assert!(Nominators::::contains_key(101)); + assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + + // when account 101 renominates + assert_ok!(Staking::nominate(Origin::signed(100), vec![41])); + + // then counts don't change + assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); + assert_eq!(Nominators::::iter().count() as u32, pre_insert_nominator_count); + // and the list is the same + assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + }); +} From 216718c263ebdb248e6144f4c65f7a9c5b60b461 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 6 Aug 2021 11:47:26 +0200 Subject: [PATCH 139/241] Everything builds and works, only the benchmark... --- frame/bags-list/src/lib.rs | 5 ++- frame/bags-list/src/list/mod.rs | 16 ++++++--- frame/bags-list/src/list/tests.rs | 5 ++- frame/bags-list/src/make_bags.rs | 2 +- frame/bags-list/src/tests.rs | 5 ++- frame/election-provider-support/src/lib.rs | 4 ++- frame/staking/src/pallet/impls.rs | 33 +++++++---------- frame/staking/src/tests.rs | 41 ++++++++++++---------- 8 files changed, 62 insertions(+), 49 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index e94b2f5888293..6bc4857a133e3 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -79,6 +79,7 @@ pub mod weights; pub use pallet::*; pub use weights::WeightInfo; +pub use list::Error; use list::List; pub(crate) const LOG_TARGET: &'static str = "runtime::bags_list"; @@ -243,6 +244,8 @@ impl Pallet { } impl SortedListProvider for Pallet { + type Error = Error; + fn iter() -> Box> { Box::new(List::::iter().map(|n| n.id().clone())) } @@ -255,7 +258,7 @@ impl SortedListProvider for Pallet { List::::contains(voter) } - fn on_insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), ()> { + fn on_insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), Error> { List::::insert(voter, weight) } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index b586a75254f71..a5a87245bb871 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -30,6 +30,12 @@ use sp_std::{ marker::PhantomData, }; +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + /// A duplicate voter has been detected. + Duplicate, +} + #[cfg(test)] mod tests; @@ -204,10 +210,10 @@ impl List { iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } - /// Insert several voters into the appropriate bags in the voter list. Does not check for - /// duplicates; instead continues with insertions if duplicates are detected. + /// Insert several voters into the appropriate bags in the voter Continues with insertions if + /// duplicates are detected. /// - /// This is more efficient than repeated calls to `Self::insert`. + /// Return the final count of number of voters inserted. fn insert_many( voters: impl IntoIterator, weight_of: impl Fn(&T::AccountId) -> VoteWeight, @@ -226,9 +232,9 @@ impl List { /// Insert a new voter into the appropriate bag in the voter list. /// /// Returns an error if the list already contains `voter`. - pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), ()> { + pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), Error> { if Self::contains(&voter) { - return Err(()) + return Err(Error::Duplicate) } let bag_weight = notional_bag_for::(weight); diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 484ea23ee60ed..4ce61aaeb8638 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -178,7 +178,10 @@ mod voter_list { assert!(get_list_as_ids().contains(&3)); // then - assert_storage_noop!(assert_eq!(List::::insert(3, 20), Err(()))); + assert_storage_noop!(assert_eq!( + List::::insert(3, 20).unwrap_err(), + Error::Duplicate + )); }); } diff --git a/frame/bags-list/src/make_bags.rs b/frame/bags-list/src/make_bags.rs index 037447d7445b3..0988b33eac75f 100644 --- a/frame/bags-list/src/make_bags.rs +++ b/frame/bags-list/src/make_bags.rs @@ -19,7 +19,7 @@ //! //! NOTE: this assume the runtime implements [`pallet_staking::Config`], as it requires an //! implementation of the traits [`frame_support::traits::Currency`] and -//! [`frame_support::traits::Currency`]. +//! [`frame_support::traits::CurrencyToVote`]. //! //! The process of adding voter bags to a runtime requires only four steps. //! diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 4d541b0530c9e..c22299a0b956c 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -212,7 +212,10 @@ mod sorted_list_provider { assert!(get_list_as_ids().contains(&3)); // then - assert_storage_noop!(assert_eq!(BagsList::on_insert(3, 20), Err(()))); + assert_storage_noop!(assert_eq!( + BagsList::on_insert(3, 20).unwrap_err(), + Error::Duplicate + )); }); } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 2236f23526f88..0100bc554a4a2 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -306,6 +306,8 @@ impl ElectionProvider for () { /// Something that implements this trait will do a best-effort sort over voters, and thus can be /// used on the implementing side of `ElectionDataProvider`. pub trait SortedListProvider { + type Error; + /// Returns iterator over voter list, which can have `take` called on it. fn iter() -> Box>; /// get the current count of voters. @@ -313,7 +315,7 @@ pub trait SortedListProvider { /// Return true if the insertion can happen. fn contains(voter: &AccountId) -> bool; // Hook for inserting a voter. - fn on_insert(voter: AccountId, weight: VoteWeight) -> Result<(), ()>; + fn on_insert(voter: AccountId, weight: VoteWeight) -> Result<(), Self::Error>; /// Hook for updating a single voter. fn on_update(voter: &AccountId, weight: VoteWeight); /// Hook for removing a voter from the list. diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index b73450420eedb..f7f3134a24699 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -724,31 +724,20 @@ impl Pallet { pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { CounterForNominators::::mutate(|x| x.saturating_inc()); - - // TODO: should we only mutate the counter if they are in not Nominators && SortedListProvider? - // To preserve current behavior not stopping adding them if SortedListProvider has issues - - if !T::SortedListProvider::contains(who) { - if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { - // insertion only fails if the voter is "contained" in `SortedListProvider`. - // Since we check if the voter is in the list above getting here is - // impossible. - log!( + if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { + // insertion only fails if the voter is "contained" in `SortedListProvider`. + // Since we check if the voter is in the list above getting here is + // impossible. + log!( warn, "attempt to insert duplicate nominator ({:#?}) despite `List::contain` return false", who ); - debug_assert!(false, "attempt to insert duplicate nominator despite `List::contain` return false"); - }; - } else { - // we are in a very bad situation - log!( - warn, - "Nominators and SortedListProvider out of sync: tried to insert into list a duplicate nominator ({:#?})", - who + debug_assert!( + false, + "attempt to insert duplicate nominator despite `List::contain` return false" ); - debug_assert!(false, "Nominators and SortedListProvider out of sync: tried to insert into list a duplicate nominator"); - } + }; debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } @@ -1259,6 +1248,8 @@ impl VoteWeightProvider for Pallet { /// A simple voter list implementation that does not require any additional pallets. pub struct UseNominatorsMap(sp_std::marker::PhantomData); impl SortedListProvider for UseNominatorsMap { + type Error = (); + /// Returns iterator over voter list, which can have `take` called on it. fn iter() -> Box> { Box::new(Nominators::::iter().map(|(n, _)| n)) @@ -1269,7 +1260,7 @@ impl SortedListProvider for UseNominatorsMap { fn contains(voter: &T::AccountId) -> bool { Nominators::::contains_key(voter) } - fn on_insert(_voter: T::AccountId, _weight: VoteWeight) -> Result<(), ()> { + fn on_insert(_voter: T::AccountId, _weight: VoteWeight) -> Result<(), Self::Error> { // nothing to do on insert. Ok(()) } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 73948178bc4b9..21c363e4eefc1 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -4319,22 +4319,27 @@ mod election_data_provider { } } -#[test] -fn re_nominate_does_not_change_counters_or_list() { - ExtBuilder::default().nominate(true).build_and_execute(|| { - // given - let pre_insert_nominator_count = Nominators::::iter().count() as u32; - assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); - assert!(Nominators::::contains_key(101)); - assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); - - // when account 101 renominates - assert_ok!(Staking::nominate(Origin::signed(100), vec![41])); - - // then counts don't change - assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); - assert_eq!(Nominators::::iter().count() as u32, pre_insert_nominator_count); - // and the list is the same - assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); - }); +mod sorted_list_provider { + use super::*; + use frame_election_provider_support::SortedListProvider; + + #[test] + fn re_nominate_does_not_change_counters_or_list() { + ExtBuilder::default().nominate(true).build_and_execute(|| { + // given + let pre_insert_nominator_count = Nominators::::iter().count() as u32; + assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); + assert!(Nominators::::contains_key(101)); + assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + + // when account 101 renominates + assert_ok!(Staking::nominate(Origin::signed(100), vec![41])); + + // then counts don't change + assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); + assert_eq!(Nominators::::iter().count() as u32, pre_insert_nominator_count); + // and the list is the same + assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + }); + } } From b3424edf5420f818a9ca9682ab6b70a8c2714a39 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 6 Aug 2021 12:55:33 +0200 Subject: [PATCH 140/241] fuck yeah benchmarks --- frame/bags-list/src/benchmarks.rs | 11 ++++++++++- frame/staking/src/pallet/impls.rs | 12 ++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 095f45a9f7529..924a608b7a530 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -65,7 +65,16 @@ frame_benchmarking::benchmarks! { // check the bags have updated as expected. assert_eq!( get_bags::(), - vec![(origin_bag_thresh, vec![origin_head, origin_tail]), (dest_bag_thresh, vec![dest_head, origin_middle])] + vec![ + ( + origin_bag_thresh, + vec![origin_head, origin_tail], + ), + ( + dest_bag_thresh, + vec![dest_head, origin_middle], + ) + ] ); } } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index f7f3134a24699..f3523b816f3dd 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1219,29 +1219,25 @@ where impl VoteWeightProvider for Pallet { fn vote_weight(who: &T::AccountId) -> VoteWeight { - // TODO: def. remove this. - if cfg!(feature = "runtime-benchmarks") { - Self::slashable_balance_of_vote_weight(who, sp_runtime::traits::Bounded::max_value()) - } else { - Self::weight_of(who) - } + Self::weight_of(who) } #[cfg(feature = "runtime-benchmarks")] fn set_vote_weight_of(who: &T::AccountId, weight: VoteWeight) { // this will clearly results in an inconsistent state, but it should not matter for a // benchmark. - use sp_std::convert::TryInto; let active: BalanceOf = weight.try_into().map_err(|_| ()).unwrap(); let mut ledger = Self::ledger(who).unwrap_or_default(); ledger.active = active; >::insert(who, ledger); + >::insert(who, who); // also, we play a trick to make sure that a issuance based-`CurrencyToVote` behaves well: // This will make sure that total issuance is zero, thus the currency to vote will be a 1-1 // conversion. // TODO - // let imbalance = T::Currency::burn(T::Currency::total_issuance()); + let imbalance = T::Currency::burn(T::Currency::total_issuance()); + sp_std::mem::forget(imbalance); } } From 09e01068adcf07e594403498936d63328fa53062 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 6 Aug 2021 18:23:00 +0200 Subject: [PATCH 141/241] more cleanup, more hardening. --- Cargo.lock | 1 + frame/bags-list/src/lib.rs | 4 ++ frame/bags-list/src/list/mod.rs | 20 +++--- frame/bags-list/src/list/tests.rs | 78 ---------------------- frame/election-provider-support/src/lib.rs | 2 + frame/staking/Cargo.toml | 4 +- frame/staking/src/benchmarking.rs | 11 +-- frame/staking/src/mock.rs | 28 ++++++++ frame/staking/src/pallet/impls.rs | 45 ++++--------- frame/staking/src/pallet/mod.rs | 10 ++- frame/staking/src/testing_utils.rs | 4 ++ frame/staking/src/tests.rs | 34 +++++----- 12 files changed, 94 insertions(+), 147 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf85b5935c16f..2af479009cc4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5579,6 +5579,7 @@ dependencies = [ "log", "num-format", "pallet-authorship", + "pallet-bags-list", "pallet-balances", "pallet-session", "pallet-staking-reward-curve", diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 6bc4857a133e3..aa236c6dd0fa0 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -280,4 +280,8 @@ impl SortedListProvider for Pallet { fn sanity_check() -> Result<(), &'static str> { List::::sanity_check() } + + fn clear() { + List::::clear() + } } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index a5a87245bb871..7f04a8f87af88 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -17,8 +17,6 @@ //! Basic implementation of a doubly-linked list. -// TODO: further testings: fuzzer for the outer API, dev chain with a lot of nominators. - use crate::Config; use codec::{Decode, Encode}; use frame_election_provider_support::{VoteWeight, VoteWeightProvider}; @@ -238,15 +236,8 @@ impl List { } let bag_weight = notional_bag_for::(weight); - crate::log!( - debug, - "inserting {:?} with weight {} into bag {:?}", - voter, - weight, - bag_weight - ); let mut bag = Bag::::get_or_make(bag_weight); - bag.insert(voter); + bag.insert(voter.clone()); // new inserts are always the tail, so we must write the bag. bag.put(); @@ -255,6 +246,15 @@ impl List { *prev_count = prev_count.saturating_add(1) }); + crate::log!( + debug, + "inserted {:?} with weight {} into bag {:?}, new count is {}", + voter, + weight, + bag_weight, + crate::CounterForVoters::::get(), + ); + Ok(()) } diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 4ce61aaeb8638..a5ef455e36e62 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -677,82 +677,4 @@ mod node { assert!(node.is_misplaced(11)); }); } - - // TODO a test similiar to this should exist in the staking pallet - // #[test] - // fn voting_data_works() { - // ExtBuilder::default().build_and_execute_without_check_count(|| { - // let weight_of = Staking::weight_of_fn(); - - // // add nominator with no targets - // bond_nominator(42, 43, 1_000, vec![11]); - - // // given - // assert_eq!( - // get_voter_list_as_voters(), - // vec![ - // Voter::validator(11), - // Voter::validator(21), - // Voter::nominator(101), - // Voter::nominator(42), - // Voter::validator(31), - // ] - // ); - // assert_eq!(active_era(), 0); - - // let slashing_spans = - // ::SlashingSpans::iter().collect::>(); - // assert_eq!(slashing_spans.keys().len(), 0); // no pre-existing slashing spans - - // let node_11 = Node::::get(10, &11).unwrap(); - // assert_eq!( - // node_11.voting_data(&weight_of, &slashing_spans).unwrap(), - // (11, 1_000, vec![11]) - // ); - - // // getting data for a nominators with 0 slashed targets - // let node_101 = Node::::get(1_000, &101).unwrap(); - // assert_eq!( - // node_101.voting_data(&weight_of, &slashing_spans).unwrap(), - // (101, 500, vec![11, 21]) - // ); - // let node_42 = Node::::get(10, &42).unwrap(); - // assert_eq!( - // node_42.voting_data(&weight_of, &slashing_spans).unwrap(), - // (42, 1_000, vec![11]) - // ); - - // // roll ahead an era so any slashes will be after the previous nominations - // start_active_era(1); - - // // when a validator gets a slash, - // add_slash(&11); - // let slashing_spans = - // ::SlashingSpans::iter().collect::>(); - - // assert_eq!(slashing_spans.keys().cloned().collect::>(), vec![11, 42, 101]); - // // then its node no longer exists - // assert_eq!( - // get_voter_list_as_voters(), - // vec![ - // Voter::validator(21), - // Voter::nominator(101), - // Voter::nominator(42), - // Voter::validator(31), - // ] - // ); - // // and its nominators no longer have it as a target - // let node_101 = Node::::get(10, &101).unwrap(); - // assert_eq!( - // node_101.voting_data(&weight_of, &slashing_spans), - // Some((101, 475, vec![21])), - // ); - - // let node_42 = Node::::get(10, &42).unwrap(); - // assert_eq!( - // node_42.voting_data(&weight_of, &slashing_spans), - // None, // no voting data since its 1 target has been slashed since nominating - // ); - // }); - // } } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 0100bc554a4a2..c72b593172d19 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -329,6 +329,8 @@ pub trait SortedListProvider { all: impl IntoIterator, weight_of: Box VoteWeight>, ) -> u32; + /// Remove everything + fn clear(); } /// Something that can provide the `VoteWeight` of an account. Similar to [`ElectionProvider`] and diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 73078d0076057..2914294faf4b3 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -51,9 +51,7 @@ sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elect pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } -# TODO: all staking tests should work with this AND the staking stub. Similar to balances pallet we -# can create two different mocks with a macro helper (low priority). -# pallet-bags-list = { version = "4.0.0-dev", path = "../bags-list" } +pallet-bags-list = { version = "4.0.0-dev", path = "../bags-list" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } frame-election-provider-support = { version = "4.0.0-dev", features = ["runtime-benchmarks"], path = "../election-provider-support" } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 53f0f7c8431c5..d8eef256fd782 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -662,7 +662,7 @@ mod tests { #[test] fn create_validators_with_nominators_for_era_works() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let v = 10; let n = 100; @@ -678,6 +678,9 @@ mod tests { let count_validators = Validators::::iter().count(); let count_nominators = Nominators::::iter().count(); + assert_eq!(count_validators, CounterForValidators::::get() as usize); + assert_eq!(count_nominators, CounterForNominators::::get() as usize); + assert_eq!(count_validators, v as usize); assert_eq!(count_nominators, n as usize); }); @@ -685,7 +688,7 @@ mod tests { #[test] fn create_validator_with_nominators_works() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let n = 10; let (validator_stash, nominators) = create_validator_with_nominators::( @@ -710,7 +713,7 @@ mod tests { #[test] fn add_slashing_spans_works() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let n = 10; let (validator_stash, _nominators) = create_validator_with_nominators::( @@ -742,7 +745,7 @@ mod tests { #[test] fn test_payout_all() { - ExtBuilder::default().has_stakers(true).build_and_execute(|| { + ExtBuilder::default().build_and_execute(|| { let v = 10; let n = 100; diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 996d3a334a305..f2dcdd993e519 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -105,6 +105,7 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, + BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, } ); @@ -241,6 +242,20 @@ impl OnUnbalanced> for RewardRemainderMock { } } +const THRESHOLDS: [sp_npos_elections::VoteWeight; 9] = + [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; + +parameter_types! { + pub static BagThresholds: &'static [sp_npos_elections::VoteWeight] = &THRESHOLDS; +} + +impl pallet_bags_list::Config for Test { + type Event = Event; + type WeightInfo = (); + type VoteWeightProvider = Staking; + type BagThresholds = BagThresholds; +} + impl onchain::Config for Test { type AccountId = AccountId; type BlockNumber = BlockNumber; @@ -269,6 +284,7 @@ impl crate::pallet::pallet::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + // NOTE: consider a macro and use `UseNominatorsMap` as well. type SortedListProvider = UseNominatorsMap; } @@ -819,3 +835,15 @@ pub(crate) fn staking_events() -> Vec> { pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { (Balances::free_balance(who), Balances::reserved_balance(who)) } + +/// ensure that the given staking ledger has `total`, `own`, and is being only backed by `others`. +pub(crate) fn assert_eq_exposure( + exposure: Exposure, + total: Balance, + own: Balance, + others: Vec>, +) { + assert_eq!(exposure.total, total); + assert_eq!(exposure.own, own); + substrate_test_utils::assert_eq_uvec!(exposure.others, others); +} diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index f3523b816f3dd..43471412a67a4 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -655,7 +655,6 @@ impl Pallet { pub fn get_npos_voters( maybe_max_len: Option, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { - // TODO: weight this function and see how many can fit in a block. let weight_of = Self::weight_of_fn(); let nominator_count = CounterForNominators::::get() as usize; @@ -723,16 +722,16 @@ impl Pallet { /// wrong. pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { + // maybe update the counter. CounterForNominators::::mutate(|x| x.saturating_inc()); + + // maybe update sorted list. Defensive-only: this should never fail. if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { - // insertion only fails if the voter is "contained" in `SortedListProvider`. - // Since we check if the voter is in the list above getting here is - // impossible. log!( - warn, - "attempt to insert duplicate nominator ({:#?}) despite `List::contain` return false", - who - ); + warn, + "attempt to insert duplicate nominator ({:#?}) despite `List::contain` return false", + who + ); debug_assert!( false, "attempt to insert duplicate nominator despite `List::contain` return false" @@ -741,28 +740,6 @@ impl Pallet { debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } - // TODO: - // chilled - // - validate - // - nominate - // - chill - // - bond-extra - // - unbond - // - rebond - // nominator - // - validate - // - nominate - // - chill - // - bond-extra - // - unbond - // - rebond - // validators - // - validate - // - nominate - // - chill - // - bond-extra - // - unbond - // - rebond Nominators::::insert(who, nominations); } @@ -1235,8 +1212,9 @@ impl VoteWeightProvider for Pallet { // also, we play a trick to make sure that a issuance based-`CurrencyToVote` behaves well: // This will make sure that total issuance is zero, thus the currency to vote will be a 1-1 // conversion. - // TODO let imbalance = T::Currency::burn(T::Currency::total_issuance()); + // kinda ugly, but gets the job done. The fact that this works here is a HUGE exception. + // Don't try this pattern in other places. sp_std::mem::forget(imbalance); } } @@ -1276,4 +1254,9 @@ impl SortedListProvider for UseNominatorsMap { fn sanity_check() -> Result<(), &'static str> { Ok(()) } + + fn clear() { + Nominators::::remove_all(None); + CounterForNominators::::kill(); + } } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 1dd17caea8ca5..9c81657990f6c 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -780,9 +780,11 @@ pub mod pallet { Error::::InsufficientBond ); + // update this staker in the sorted list, if they exist in it. if T::SortedListProvider::contains(&stash) { T::SortedListProvider::on_update(&stash, Self::weight_of_fn()(&ledger.stash)); } + Self::deposit_event(Event::::Bonded(stash.clone(), extra)); Self::update_ledger(&controller, &ledger); } @@ -844,9 +846,12 @@ pub mod pallet { let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); Self::update_ledger(&controller, &ledger); + + // update this staker in the sorted list, if they exist in it. if T::SortedListProvider::contains(&ledger.stash) { T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); } + Self::deposit_event(Event::::Unbonded(ledger.stash, value)); } Ok(()) @@ -1339,10 +1344,11 @@ pub mod pallet { Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); Self::update_ledger(&controller, &ledger); + if T::SortedListProvider::contains(&ledger.stash) { T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); } - // TODO: fix this in master earlier. + Ok(Some(T::WeightInfo::rebond(ledger.unlocking.len() as u32)).into()) } @@ -1509,8 +1515,6 @@ pub mod pallet { /// /// This can be helpful if bond requirements are updated, and we need to remove old users /// who do not satisfy these requirements. - // TODO: Maybe we can deprecate `chill` in the future. - // https://github.com/paritytech/substrate/issues/9111 #[pallet::weight(T::WeightInfo::chill_other())] pub fn chill_other(origin: OriginFor, controller: T::AccountId) -> DispatchResult { // Anyone can call this function. diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 795c066d09bb3..10348761268ae 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -27,6 +27,7 @@ use rand_chacha::{ }; use sp_io::hashing::blake2_256; +use frame_election_provider_support::SortedListProvider; use frame_support::{pallet_prelude::*, traits::Currency}; use sp_runtime::{traits::StaticLookup, Perbill}; use sp_std::prelude::*; @@ -37,8 +38,11 @@ const SEED: u32 = 0; pub fn clear_validators_and_nominators() { Validators::::remove_all(None); CounterForValidators::::kill(); + + // whenever we touch nominators counter we should update `T::SortedListProvider` as well. Nominators::::remove_all(None); CounterForNominators::::kill(); + T::SortedListProvider::clear(); } /// Grab a funded user. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 21c363e4eefc1..e3a193abf9fda 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -544,27 +544,25 @@ fn nominating_and_rewards_should_work() { .count(), 2 ); - assert_eq!( + + assert_eq_exposure( Staking::eras_stakers(Staking::active_era().unwrap().index, 11), - Exposure { - total: 1000 + 800, - own: 1000, - others: vec![ - IndividualExposure { who: 3, value: 400 }, - IndividualExposure { who: 1, value: 400 }, - ] - }, + 1000 + 800, + 1000, + vec![ + IndividualExposure { who: 3, value: 400 }, + IndividualExposure { who: 1, value: 400 }, + ], ); - assert_eq!( + + assert_eq_exposure( Staking::eras_stakers(Staking::active_era().unwrap().index, 21), - Exposure { - total: 1000 + 1200, - own: 1000, - others: vec![ - IndividualExposure { who: 3, value: 600 }, - IndividualExposure { who: 1, value: 600 }, - ] - }, + 1000 + 1200, + 1000, + vec![ + IndividualExposure { who: 3, value: 600 }, + IndividualExposure { who: 1, value: 600 }, + ], ); // the total reward for era 1 From e3f5f91fcf3c1bb8bf85910f21ad471e8db4e704 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 6 Aug 2021 18:23:22 +0200 Subject: [PATCH 142/241] use the bags list again --- frame/staking/src/mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index f2dcdd993e519..23304ef61cc54 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -285,7 +285,7 @@ impl crate::pallet::pallet::Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); // NOTE: consider a macro and use `UseNominatorsMap` as well. - type SortedListProvider = UseNominatorsMap; + type SortedListProvider = BagsList; } impl frame_system::offchain::SendTransactionTypes for Test From 7d78aafe816f0007440e9aebaf2b9d4607c3906e Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 6 Aug 2021 18:35:22 +0200 Subject: [PATCH 143/241] fix benhc --- frame/staking/src/pallet/impls.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 43471412a67a4..cb1313cd80cc8 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1203,6 +1203,7 @@ impl VoteWeightProvider for Pallet { fn set_vote_weight_of(who: &T::AccountId, weight: VoteWeight) { // this will clearly results in an inconsistent state, but it should not matter for a // benchmark. + use sp_std::convert::TryInto; let active: BalanceOf = weight.try_into().map_err(|_| ()).unwrap(); let mut ledger = Self::ledger(who).unwrap_or_default(); ledger.active = active; From ceb7066d1b696cb9a5d25ed8b77900cb8508d586 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 6 Aug 2021 12:35:20 -0700 Subject: [PATCH 144/241] Some questions and changs for List::migration --- frame/bags-list/src/list/mod.rs | 22 +++++++++++++++++----- frame/election-provider-support/src/lib.rs | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 7f04a8f87af88..1bbe54da444fd 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -15,7 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Basic implementation of a doubly-linked list. +//! Implementation of a "bags list": a semi-sorted list where ordering granularity is dictated +//! by configurable thresholds that delineate the boundaries of bags. Within the bounds of each bag +//! ordering is dictated by insertion order. use crate::Config; use codec::{Decode, Encode}; @@ -90,6 +92,7 @@ impl List { Self::insert_many(all, weight_of) } + // TODO test this /// Migrate the voter list from one set of thresholds to another. /// /// This should only be called as part of an intentional migration; it's fairly expensive. @@ -120,12 +123,20 @@ impl List { let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); let new_set: BTreeSet<_> = T::BagThresholds::get().iter().copied().collect(); + // accounts that need to be rebaged let mut affected_accounts = BTreeSet::new(); + // track affected old bags to make sure we only iterate them once let mut affected_old_bags = BTreeSet::new(); + let new_bags = new_set.difference(&old_set).copied(); // a new bag means that all accounts previously using the old bag's threshold must now // be rebagged - for inserted_bag in new_set.difference(&old_set).copied() { + for inserted_bag in new_bags { + // TODO: why use notional_bag_for here? This affected_bag is a new bag since + // new_set.difference(&old_set) is just the new bags; notional_bag_for is using the + // T::BagThresholds, which are the new threhsholds .. so plugging in a threhold to + // notional_bag_for will just give back that threshold. I think we want to instead + // use notional_bag_for logic with old_thresholds? let affected_bag = notional_bag_for::(inserted_bag); if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's @@ -138,8 +149,9 @@ impl List { } } + let removed_bags = old_set.difference(&new_set).copied(); // a removed bag means that all members of that bag must be rebagged - for removed_bag in old_set.difference(&new_set).copied() { + for removed_bag in removed_bags { if !affected_old_bags.insert(removed_bag) { continue } @@ -149,7 +161,7 @@ impl List { } } - // migrate the + // migrate the voters whose bag has changed let weight_of = T::VoteWeightProvider::vote_weight; Self::remove_many(affected_accounts.iter().map(|voter| voter)); let num_affected = affected_accounts.len() as u32; @@ -161,7 +173,7 @@ impl List { // // it's pretty cheap to iterate this again, because both sets are in-memory and require no // lookups. - for removed_bag in old_set.difference(&new_set).copied() { + for removed_bag in removed_bags { debug_assert!( !crate::VoterNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), "no voter should be present in a removed bag", diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index c72b593172d19..a78fb7f9e3f71 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -329,7 +329,7 @@ pub trait SortedListProvider { all: impl IntoIterator, weight_of: Box VoteWeight>, ) -> u32; - /// Remove everything + /// Remove everything. fn clear(); } From b41564f29bc6b02861bb48d1ba8938362b55df87 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 6 Aug 2021 13:33:43 -0700 Subject: [PATCH 145/241] Fix migration removed_bags and new_bags usage --- frame/bags-list/src/list/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 1bbe54da444fd..70e182d7a7d4f 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -151,7 +151,7 @@ impl List { let removed_bags = old_set.difference(&new_set).copied(); // a removed bag means that all members of that bag must be rebagged - for removed_bag in removed_bags { + for removed_bag in removed_bags.clone() { if !affected_old_bags.insert(removed_bag) { continue } From 93b4455a414dc54f3baef099139fdd88b9f7dd31 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 6 Aug 2021 13:57:01 -0700 Subject: [PATCH 146/241] Some trivial aesthetic changes --- frame/bags-list/src/list/mod.rs | 37 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 70e182d7a7d4f..925c52bbad25c 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -17,7 +17,7 @@ //! Implementation of a "bags list": a semi-sorted list where ordering granularity is dictated //! by configurable thresholds that delineate the boundaries of bags. Within the bounds of each bag -//! ordering is dictated by insertion order. +//! ordering is dictated by insertion order. Each bag is a doubly linked-list of items (voters). use crate::Config; use codec::{Decode, Encode}; @@ -362,13 +362,12 @@ impl List { /// Sanity check the voter list. /// /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) - /// is being used, after all other staking data (such as counter) has been updated. It checks - /// that: + /// is being used, after all other staking data (such as counter) has been updated. It checks: /// - /// * Iterate all voters in list and make sure there are no duplicates. - /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. - /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are - /// checked per *any* update to `List`. + /// * there are no duplicate ids, + /// * length of this list are in sync with `CounterForVoters`. + /// * and sanity-checks all bags. This will cascade down all the checks and makes sure all bags + /// are checked per *any* update to `List`. pub(crate) fn sanity_check() -> Result<(), &'static str> { use frame_support::ensure; let mut seen_in_list = BTreeSet::new(); @@ -377,7 +376,7 @@ impl List { "duplicate identified", ); - let iter_count = Self::iter().collect::>().len() as u32; + let iter_count = Self::iter().count() as u32; let stored_count = crate::CounterForVoters::::get(); ensure!(iter_count == stored_count, "iter_count != stored_count",); @@ -419,6 +418,11 @@ impl Bag { Self { head, tail, bag_upper } } + /// Iterate over the nodes in this bag. + pub(crate) fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + /// Get a bag by its upper vote weight. pub(crate) fn get(bag_upper: VoteWeight) -> Option> { debug_assert!( @@ -464,11 +468,6 @@ impl Bag { self.tail.as_ref().and_then(|id| Node::get(id)) } - /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) - } - /// Insert a new voter into this bag. /// /// This is private on purpose because it's naive: it doesn't check whether this is the @@ -491,7 +490,7 @@ impl Bag { fn insert_node(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { - // this should never happen, but this check prevents a worst case infinite loop + // this should never happen, but this check prevents a worst case infinite loop. debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); return @@ -502,13 +501,13 @@ impl Bag { // to be `self.bag_upper`. node.bag_upper = self.bag_upper; - // update this node now, treating as the new tail. let id = node.id.clone(); + // update this node now, treating it as the new tail. node.prev = self.tail.clone(); node.next = None; node.put(); - // update the previous tail + // update the previous tail. if let Some(mut old_tail) = self.tail() { old_tail.next = Some(id.clone()); old_tail.put(); @@ -524,18 +523,18 @@ impl Bag { } } - /// Remove a voter node from this bag. Returns true iff the bag's head or tail is updated. + /// Remove a node from this bag. /// /// This is private on purpose because it doesn't check whether this bag contains the voter in /// the first place. Generally, use [`List::remove`] instead. /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()` and `VoterNodes::remove(voter_id)` to update storage for the bag and `node`. + /// `self.put()` and `VoterNodes::remove(id)` to update storage for this bag and `node`. fn remove_node(&mut self, node: &Node) { // reassign neighboring nodes. node.excise(); - // clear the bag head/tail pointers as necessary + // clear the bag head/tail pointers as necessary. if self.tail.as_ref() == Some(&node.id) { self.tail = node.prev.clone(); } From c0867215d2c477fceac59ddc8311301d01668f20 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 6 Aug 2021 14:01:40 -0700 Subject: [PATCH 147/241] Some more trivial changes --- frame/bags-list/src/list/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 925c52bbad25c..c459c60d09f4f 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -583,7 +583,7 @@ impl Bag { } } -/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. +/// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] #[cfg_attr(test, derive(PartialEq, Clone))] @@ -595,7 +595,7 @@ pub(crate) struct Node { } impl Node { - /// Get a node by bag idx and account id. + /// Get a node by account id. pub(crate) fn get(account_id: &T::AccountId) -> Option> { crate::VoterNodes::::try_get(account_id).ok() } From fa879068f79cedf254aef11f3a7d538c9c77d4e0 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 6 Aug 2021 20:30:47 -0700 Subject: [PATCH 148/241] tiny changes/ --- frame/bags-list/src/benchmarks.rs | 3 +-- frame/bags-list/src/lib.rs | 2 +- frame/staking/src/pallet/impls.rs | 27 ++++++++++----------------- frame/staking/src/pallet/mod.rs | 2 +- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 924a608b7a530..83cb94d268c4b 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -2,7 +2,6 @@ use super::*; use crate::list::{Bag, List}; -// use frame_benchmarking::{account, whitelisted_caller}; use frame_benchmarking::{account, whitelisted_caller}; use frame_election_provider_support::VoteWeightProvider; use frame_support::{assert_ok, traits::Get}; @@ -49,7 +48,7 @@ frame_benchmarking::benchmarks! { let dest_head: T::AccountId = account("dest_head", 0, 0); assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); - // and the ags are in the expected state after insertions. + // and the bags are in the expected state after insertions. assert_eq!( get_bags::(), vec![ diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index aa236c6dd0fa0..31ee509fffdc2 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -143,7 +143,7 @@ pub mod pallet { /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * /// constant_ratio).max(threshold[k] + 1)` for all `k`. /// - /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use + /// The helpers in the `crate::make_bags` module can simplify this calculation. To use /// them, the `make-bags` feature must be enabled. /// /// # Examples diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index cb1313cd80cc8..875bed9cea1b3 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -646,7 +646,7 @@ impl Pallet { /// `voter_count` imposes a cap on the number of voters returned; care should be taken to ensure /// that it is accurate. /// - /// This will use all on-chain nominators, and all the validators will inject a self vote. + /// This will use on-chain nominators, and all the validators will inject a self vote. /// /// ### Slashing /// @@ -655,8 +655,6 @@ impl Pallet { pub fn get_npos_voters( maybe_max_len: Option, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { - let weight_of = Self::weight_of_fn(); - let nominator_count = CounterForNominators::::get() as usize; let validator_count = CounterForValidators::::get() as usize; let all_voter_count = validator_count.saturating_add(nominator_count); @@ -669,7 +667,8 @@ impl Pallet { // first, grab all validators, capped by the maximum allowed length. for (validator, _) in >::iter().take(max_allowed_len) { // Append self vote. - let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]); + let self_vote = + (validator.clone(), Self::weight_of(&validator), vec![validator.clone()]); all_voters.push(self_vote); } @@ -686,7 +685,7 @@ impl Pallet { .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) }); if !targets.len().is_zero() { - all_voters.push((nominator.clone(), weight_of(&nominator), targets)) + all_voters.push((nominator.clone(), Self::weight_of(&nominator), targets)) } } else { log!(error, "invalid item in `SortedListProvider`: {:?}", nominator) @@ -706,7 +705,8 @@ impl Pallet { all_voters } - /// This is a very expensive function and result should be cached versus being called multiple times. + /// This is a very expensive function and result should be cached versus being called multiple + /// times. pub fn get_npos_targets() -> Vec { // all current validators to be included Validators::::iter().map(|(v, _)| v).collect::>() @@ -717,8 +717,8 @@ impl Pallet { /// /// If the nominator already exists, their nominations will be updated. /// - /// NOTE: you must ALWAYS use this function to add a nominator to the system. Any access to - /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// NOTE: you must ALWAYS use this function to add nominator or update their targets. Any access + /// to `Nominators`, its counter, or `VoterList` outside of this function is almost certainly /// wrong. pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { @@ -727,15 +727,8 @@ impl Pallet { // maybe update sorted list. Defensive-only: this should never fail. if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { - log!( - warn, - "attempt to insert duplicate nominator ({:#?}) despite `List::contain` return false", - who - ); - debug_assert!( - false, - "attempt to insert duplicate nominator despite `List::contain` return false" - ); + log!(warn, "attempt to insert duplicate nominator ({:#?})", who); + debug_assert!(false, "attempt to insert duplicate nominator"); }; debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 9c81657990f6c..f95bd6ab041f6 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -782,7 +782,7 @@ pub mod pallet { // update this staker in the sorted list, if they exist in it. if T::SortedListProvider::contains(&stash) { - T::SortedListProvider::on_update(&stash, Self::weight_of_fn()(&ledger.stash)); + T::SortedListProvider::on_update(&stash, Self::weight_of_fn(&ledger.stash)); } Self::deposit_event(Event::::Bonded(stash.clone(), extra)); From 928fa952a0300a860bf1e0e7b90a8449a1de5db0 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Sat, 7 Aug 2021 13:31:29 +0200 Subject: [PATCH 149/241] mega rename --- Cargo.lock | 2 +- Cargo.toml | 2 +- frame/bags-list/Cargo.toml | 2 +- .../Cargo.toml | 4 +- .../src/main.rs | 2 +- .../src/{make_bags.rs => generate_bags.rs} | 22 +- frame/bags-list/src/lib.rs | 117 +++++----- frame/bags-list/src/list/mod.rs | 204 +++++++++--------- frame/bags-list/src/list/tests.rs | 102 ++++----- frame/bags-list/src/mock.rs | 9 +- frame/bags-list/src/tests.rs | 60 ++++-- frame/election-provider-support/src/lib.rs | 30 +-- frame/staking/Cargo.toml | 3 +- frame/staking/src/pallet/impls.rs | 10 +- 14 files changed, 288 insertions(+), 281 deletions(-) rename frame/bags-list/{bag-thresholds => generate-bags}/Cargo.toml (91%) rename frame/bags-list/{bag-thresholds => generate-bags}/src/main.rs (94%) rename frame/bags-list/src/{make_bags.rs => generate_bags.rs} (90%) diff --git a/Cargo.lock b/Cargo.lock index 2af479009cc4f..f0a2cdc60fc77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4470,7 +4470,7 @@ dependencies = [ ] [[package]] -name = "node-runtime-bag-thresholds" +name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ "frame-support", diff --git a/Cargo.toml b/Cargo.toml index 85ec674e3e213..2f31b19b80c1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,7 +130,7 @@ members = [ "frame/utility", "frame/vesting", "frame/bags-list", - "frame/bags-list/bag-thresholds", + "frame/bags-list/generate-bags", "primitives/api", "primitives/api/proc-macro", "primitives/api/test", diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 7c78087624c50..90695565e1be2 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -67,7 +67,7 @@ runtime-benchmarks = [ "pallet-balances", "sp-tracing", ] -make-bags = [ +generate-bags = [ "chrono", "git2", "num-format", diff --git a/frame/bags-list/bag-thresholds/Cargo.toml b/frame/bags-list/generate-bags/Cargo.toml similarity index 91% rename from frame/bags-list/bag-thresholds/Cargo.toml rename to frame/bags-list/generate-bags/Cargo.toml index 64782c910346c..7d86865f15adb 100644 --- a/frame/bags-list/bag-thresholds/Cargo.toml +++ b/frame/bags-list/generate-bags/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "node-runtime-bag-thresholds" +name = "node-runtime-generate-bags" version = "3.0.0" authors = ["Parity Technologies "] edition = "2018" @@ -14,7 +14,7 @@ node-runtime = { version = "3.0.0-dev", path = "../../../bin/node/runtime" } # FRAME frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } -pallet-bags-list = { version = "4.0.0-dev", path = "../../bags-list", features = ["make-bags"] } +pallet-bags-list = { version = "4.0.0-dev", path = "../../bags-list", features = ["generate-bags"] } pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../staking" } # primitives diff --git a/frame/bags-list/bag-thresholds/src/main.rs b/frame/bags-list/generate-bags/src/main.rs similarity index 94% rename from frame/bags-list/bag-thresholds/src/main.rs rename to frame/bags-list/generate-bags/src/main.rs index b573d962d0c4e..7f0bb5c7be74b 100644 --- a/frame/bags-list/bag-thresholds/src/main.rs +++ b/frame/bags-list/generate-bags/src/main.rs @@ -17,7 +17,7 @@ //! Make the set of bag thresholds to be used in pallet-bags-list. -use pallet_bags_list::make_bags::generate_thresholds_module; +use pallet_bags_list::generate_bags::generate_thresholds_module; use std::path::PathBuf; use structopt::StructOpt; #[derive(Debug, StructOpt)] diff --git a/frame/bags-list/src/make_bags.rs b/frame/bags-list/src/generate_bags.rs similarity index 90% rename from frame/bags-list/src/make_bags.rs rename to frame/bags-list/src/generate_bags.rs index 0988b33eac75f..b6d65cb2dda88 100644 --- a/frame/bags-list/src/make_bags.rs +++ b/frame/bags-list/src/generate_bags.rs @@ -15,43 +15,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Support code to ease the process of generating voter bags. +//! Support code to ease the process of generating bag thresholds. //! //! NOTE: this assume the runtime implements [`pallet_staking::Config`], as it requires an //! implementation of the traits [`frame_support::traits::Currency`] and //! [`frame_support::traits::CurrencyToVote`]. //! -//! The process of adding voter bags to a runtime requires only four steps. +//! The process of adding bags to a runtime requires only four steps. //! //! 1. Update the runtime definition. //! //! ```ignore //! parameter_types!{ -//! pub const VoterBagThresholds: &'static [u64] = &[]; +//! pub const BagThresholds: &'static [u64] = &[]; //! } //! //! impl pallet_staking::Config for Runtime { //! // -//! type VoterBagThresholds = VoterBagThresholds; +//! type BagThresholds = BagThresholds; //! } //! ``` //! //! 2. Write a little program to generate the definitions. This can be a near-identical copy of -//! `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime +//! `substrate/frame/bags-list/generate-bags`. This program exists only to hook together the runtime //! definitions with the various calculations here. //! //! 3. Run that program: //! //! ```sh,notrust -//! $ cargo run -p node-runtime-voter-bags -- bin/node/runtime/src/voter_bags.rs +//! $ cargo run -p node-runtime-generate-bags -- output.rs //! ``` //! //! 4. Update the runtime definition. //! //! ```diff,notrust -//! + mod voter_bags; -//! - pub const VoterBagThresholds: &'static [u64] = &[]; -//! + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; +//! + mod output; +//! - pub const BagThresholds: &'static [u64] = &[]; +//! + pub const BagThresholds: &'static [u64] = &output::THRESHOLDS; //! ``` use frame_election_provider_support::VoteWeight; @@ -66,7 +66,7 @@ use std::{ /// Note that this value depends on the current issuance, a quantity known to change over time. /// This makes the project of computing a static value suitable for inclusion in a static, /// generated file _excitingly unstable_. -#[cfg(any(feature = "std", feature = "make-bags"))] +#[cfg(any(feature = "std", feature = "generate-bags"))] fn existential_weight() -> VoteWeight { use frame_support::traits::{Currency, CurrencyToVote}; @@ -187,7 +187,7 @@ pub fn generate_thresholds_module( // module docs let now = chrono::Utc::now(); writeln!(buf)?; - writeln!(buf, "//! Autogenerated voter bag thresholds.")?; + writeln!(buf, "//! Autogenerated bag thresholds.")?; writeln!(buf, "//!")?; writeln!(buf, "//! Generated on {}", now.to_rfc3339())?; writeln!( diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index aa236c6dd0fa0..6f67e8017f2b1 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -16,8 +16,8 @@ // limitations under the License. //! A semi-sorted list, where items hold an `AccountId` based on some `VoteWeight`. The `AccountId` -//! can be synonym to a `Voter` and `VoteWeight` signifies the chance of each voter being included -//! in the final [`VoteWeightProvider::iter`]. +//! (`id` for short) might be synonym to a `voter` or `nominator` in some context, and `VoteWeight` +//! signifies the chance of each id being included in the final [`VoteWeightProvider::iter`]. //! //! It implements [`sp_election_provider_support::SortedListProvider`] to provide a semi-sorted list //! of accounts to another pallet. It needs some other pallet to give it some information about the @@ -26,13 +26,6 @@ //! This pallet is not configurable at genesis. Whoever uses it should call appropriate functions of //! the `SortedListProvider` (i.e. `on_insert`) at their genesis. //! -//! # ⚠️ WARNING ⚠️ -//! -//! Do not insert an id that already exists in the list; doing so can result in catastrophic failure -//! of your blockchain, including entering into an infinite loop during block execution. This -//! situation can be avoided by strictly using [`Pallet::::on_insert`] for insertions because -//! it will exit early and return an error if the id is a duplicate. -//! //! # Goals //! //! The data structure exposed by this pallet aims to be optimized for: @@ -46,18 +39,13 @@ //! - items are kept in bags, which are delineated by their range of weight (See [`BagThresholds`]). //! - for iteration, bags are chained together from highest to lowest and elements within the bag //! are iterated from head to tail. -//! - items within a bag are iterated in order of insertion. Thus removing an item and -//! re-inserting it will worsen its position in list iteration; this reduces incentives for some -//! types of spam that involve consistently removing and inserting for better position. Further, -//! ordering granularity is thus dictated by range between each bag threshold. +//! - items within a bag are iterated in order of insertion. Thus removing an item and re-inserting +//! it will worsen its position in list iteration; this reduces incentives for some types of spam +//! that involve consistently removing and inserting for better position. Further, ordering +//! granularity is thus dictated by range between each bag threshold. //! - if an items weight changes to a value no longer within the range of its current bag the item's //! position will need to be updated by an external actor with rebag (update), or removal and //! insertion. -// -//! ### Further Plans -//! -//! *new terminology*: fully decouple this pallet from voting names, use account id instead of -//! voter and priority instead of weight. #![cfg_attr(not(feature = "std"), no_std)] @@ -67,9 +55,10 @@ use sp_std::prelude::*; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; + +#[cfg(feature = "generate-bags")] +pub mod generate_bags; mod list; -#[cfg(feature = "make-bags")] -pub mod make_bags; #[cfg(test)] mod mock; #[cfg(test)] @@ -113,19 +102,18 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: weights::WeightInfo; - /// Something that provides the weights of voters. + /// Something that provides the weights of ids. type VoteWeightProvider: VoteWeightProvider; - /// The list of thresholds separating the various voter bags. + /// The list of thresholds separating the various bags. /// - /// Voters are separated into unsorted bags according to their vote weight. This specifies - /// the thresholds separating the bags. A voter's bag is the largest bag for which the - /// voter's weight is less than or equal to its upper threshold. + /// Ids are separated into unsorted bags according to their vote weight. This specifies the + /// thresholds separating the bags. An id's bag is the largest bag for which the id's weight + /// is less than or equal to its upper threshold. /// - /// When voters are iterated, higher bags are iterated completely before lower bags. This - /// means that iteration is _semi-sorted_: voters of higher weight tend to come before - /// voters of lower weight, but peer voters within a particular bag are sorted in insertion - /// order. + /// When ids are iterated, higher bags are iterated completely before lower bags. This means + /// that iteration is _semi-sorted_: ids of higher weight tend to come before ids of lower + /// weight, but peer ids within a particular bag are sorted in insertion order. /// /// # Expressing the constant /// @@ -143,47 +131,40 @@ pub mod pallet { /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * /// constant_ratio).max(threshold[k] + 1)` for all `k`. /// - /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use - /// them, the `make-bags` feature must be enabled. + /// The helpers in the `generate-bags` module can simplify this calculation. To use them, + /// the `generate-bags` feature must be enabled. /// /// # Examples /// - /// - If `BagThresholds::get().is_empty()`, then all voters are put into the same bag, - /// and iteration is strictly in insertion order. - /// - If `BagThresholds::get().len() == 64`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is equal to 2. - /// - If `BagThresholds::get().len() == 200`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is approximately equal - /// to 1.248. - /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will - /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. + /// - If `BagThresholds::get().is_empty()`, then all ids are put into the same bag, and + /// iteration is strictly in insertion order. + /// - If `BagThresholds::get().len() == 64`, and the thresholds are determined according to + /// the procedure given above, then the constant ratio is equal to 2. + /// - If `BagThresholds::get().len() == 200`, and the thresholds are determined according to + /// the procedure given above, then the constant ratio is approximately equal to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then an id with weight 0 or 1 will + /// fall into bag 0, an id with weight 2 will fall into bag 1, etc. /// /// # Migration /// /// In the event that this list ever changes, a copy of the old bags list must be retained. - /// With that `List::migrate` can be called, which will perform the appropriate - /// migration. + /// With that `List::migrate` can be called, which will perform the appropriate migration. #[pallet::constant] type BagThresholds: Get<&'static [VoteWeight]>; } - /// How many voters are registered. + /// How many ids are registered. #[pallet::storage] - pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; + pub(crate) type CounterForListNodes = StorageValue<_, u32, ValueQuery>; - /// Voter nodes store links forward and back within their respective bags, the stash id, and - /// whether the voter is a validator or nominator. - /// - /// There is nothing in this map directly identifying to which bag a particular node belongs. - /// However, the `Node` data structure has helpers which can provide that information. + /// Nodes store links forward and back within their respective bags, id. #[pallet::storage] - pub(crate) type VoterNodes = - StorageMap<_, Twox64Concat, T::AccountId, list::Node>; + pub(crate) type ListNodes = StorageMap<_, Twox64Concat, T::AccountId, list::Node>; /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] - pub(crate) type VoterBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; + pub(crate) type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] @@ -195,19 +176,19 @@ pub mod pallet { #[pallet::call] impl Pallet { - /// Declare that some `reposition` account has, through rewards or penalties, sufficiently + /// Declare that some `dislocated` account has, through rewards or penalties, sufficiently /// changed its weight that it should properly fall into a different bag than its current /// one. /// - /// Anyone can call this function about any target account. + /// Anyone can call this function about any potentially dislocated account. /// - /// Will never return an error; if `reposition` does not exist or doesn't need a rebag, then + /// Will never return an error; if `dislocated` does not exist or doesn't need a rebag, then /// it is a noop and only fees are collected from `origin`. #[pallet::weight(T::WeightInfo::rebag())] - pub fn rebag(origin: OriginFor, reposition: T::AccountId) -> DispatchResult { + pub fn rebag(origin: OriginFor, dislocated: T::AccountId) -> DispatchResult { ensure_signed(origin)?; - let current_weight = T::VoteWeightProvider::vote_weight(&reposition); - let _ = Pallet::::do_rebag(&reposition, current_weight); + let current_weight = T::VoteWeightProvider::vote_weight(&dislocated); + let _ = Pallet::::do_rebag(&dislocated, current_weight); Ok(()) } } @@ -218,7 +199,7 @@ pub mod pallet { // ensure they are strictly increasing, this also implies that duplicates are detected. assert!( T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "Voter bag thresholds must strictly increase, and have no duplicates", + "thresholds must strictly increase, and have no duplicates", ); } } @@ -251,23 +232,23 @@ impl SortedListProvider for Pallet { } fn count() -> u32 { - CounterForVoters::::get() + CounterForListNodes::::get() } - fn contains(voter: &T::AccountId) -> bool { - List::::contains(voter) + fn contains(id: &T::AccountId) -> bool { + List::::contains(id) } - fn on_insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), Error> { - List::::insert(voter, weight) + fn on_insert(id: T::AccountId, weight: VoteWeight) -> Result<(), Error> { + List::::insert(id, weight) } - fn on_update(voter: &T::AccountId, new_weight: VoteWeight) { - Pallet::::do_rebag(voter, new_weight); + fn on_update(id: &T::AccountId, new_weight: VoteWeight) { + Pallet::::do_rebag(id, new_weight); } - fn on_remove(voter: &T::AccountId) { - List::::remove(voter) + fn on_remove(id: &T::AccountId) { + List::::remove(id) } fn regenerate( diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 7f04a8f87af88..d9cbd5c06313f 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -15,7 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Basic implementation of a doubly-linked list. +//! Basic implementation of a doubly-linked list. It uses a pattern of composite data structures, +//! where multiple storage items are masked by one outer API. See [`ListNodes`], +//! [`CounterForListNodes`] and [`ListBags`] for more information. +//! +//! +//! The outer API of this module is the [`List`] struct. It wraps all acceptable operations on top +//! of the aggregate linked list. All operations with the linked list should happen through this +//! interface. use crate::Config; use codec::{Decode, Encode}; @@ -30,22 +37,18 @@ use sp_std::{ #[derive(Debug, PartialEq, Eq)] pub enum Error { - /// A duplicate voter has been detected. + /// A duplicate id has been detected. Duplicate, } #[cfg(test)] mod tests; -/// Given a certain vote weight, which bag should contain this voter? +/// Given a certain vote weight, to which bag does it belong to? /// /// Bags are identified by their upper threshold; the value returned by this function is guaranteed /// to be a member of `T::BagThresholds`. /// -/// This is used instead of a simpler scheme, such as the index within `T::BagThresholds`, because -/// in the event that bags are inserted or deleted, the number of affected voters which need to be -/// migrated is smaller. -/// /// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this /// function behaves as if it does. fn notional_bag_for(weight: VoteWeight) -> VoteWeight { @@ -54,34 +57,36 @@ fn notional_bag_for(weight: VoteWeight) -> VoteWeight { thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } -/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. +/// Data structure providing efficient mostly-accurate selection of the top N id by `VoteWeight`. /// -/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of +/// It's implemented as a set of linked lists. Each linked list comprises a bag of ids of /// arbitrary and unbounded length, all having a vote weight within a particular constant range. -/// This structure means that voters can be added and removed in `O(1)` time. +/// This structure means that ids can be added and removed in `O(1)` time. /// /// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While -/// the users within any particular bag are sorted in an entirely arbitrary order, the overall stake -/// decreases as successive bags are reached. This means that it is valid to truncate iteration at -/// any desired point; only those voters in the lowest bag (who are known to have relatively little -/// power to affect the outcome) can be excluded. This satisfies both the desire for fairness and -/// the requirement for efficiency. +/// the users within any particular bag are sorted in an entirely arbitrary order, the overall vote +/// weight decreases as successive bags are reached. This means that it is valid to truncate +/// iteration at any desired point; only those ids in the lowest bag can be excluded. This +/// satisfies both the desire for fairness and the requirement for efficiency. pub struct List(PhantomData); impl List { - /// Remove all data associated with the voter list from storage. + /// Remove all data associated with the list from storage. pub(crate) fn clear() { - crate::CounterForVoters::::kill(); - crate::VoterBags::::remove_all(None); - crate::VoterNodes::::remove_all(None); + crate::CounterForListNodes::::kill(); + crate::ListBags::::remove_all(None); + crate::ListNodes::::remove_all(None); } - /// Regenerate voter data from the given ids. + /// Regenerate all of the data from the given ids. + /// + /// This is expensive and should only ever be performed during a migration a migration, or when + /// the data needs to be generated from scratch again. /// - /// This is expensive and should only ever be performed during a migration, never during - /// consensus. + /// This may or may not need to be called at genesis as well, based on the configuration of the + /// pallet using this `List`. /// - /// Returns the number of voters migrated. + /// Returns the number of ids migrated. pub fn regenerate( all: impl IntoIterator, weight_of: Box VoteWeight>, @@ -90,7 +95,7 @@ impl List { Self::insert_many(all, weight_of) } - /// Migrate the voter list from one set of thresholds to another. + /// Migrate the list from one set of thresholds to another. /// /// This should only be called as part of an intentional migration; it's fairly expensive. /// @@ -100,20 +105,20 @@ impl List { /// /// - `old_thresholds` is the previous list of thresholds. /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::BagThresholds` has already been updated. + /// - `T::BagThresholds` has already been updated and is the new set of thresholds. /// /// Postconditions: /// /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. - /// - No voter is changed unless required to by the difference between the old threshold list + /// - No id is changed unless required to by the difference between the old threshold list /// and the new. - /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the /// new threshold set. #[allow(dead_code)] pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { // we can't check all preconditions, but we can check one debug_assert!( - crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + crate::ListBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), "not all `bag_upper` currently in storage are members of `old_thresholds`", ); @@ -151,7 +156,7 @@ impl List { // migrate the let weight_of = T::VoteWeightProvider::vote_weight; - Self::remove_many(affected_accounts.iter().map(|voter| voter)); + Self::remove_many(affected_accounts.iter().map(|id| id)); let num_affected = affected_accounts.len() as u32; let _ = Self::insert_many(affected_accounts.into_iter(), weight_of); @@ -163,16 +168,16 @@ impl List { // lookups. for removed_bag in old_set.difference(&new_set).copied() { debug_assert!( - !crate::VoterNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), - "no voter should be present in a removed bag", + !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), + "no id should be present in a removed bag", ); - crate::VoterBags::::remove(removed_bag); + crate::ListBags::::remove(removed_bag); } debug_assert!( { let thresholds = T::BagThresholds::get(); - crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + crate::ListBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) }, "all `bag_upper` in storage must be members of the new thresholds", ); @@ -180,12 +185,12 @@ impl List { num_affected } - /// Returns true if the list contains `id`, otherwise returns `false`. - pub(crate) fn contains(voter: &T::AccountId) -> bool { - crate::VoterNodes::::contains_key(voter) + /// Returns `true` if the list contains `id`, otherwise returns `false`. + pub(crate) fn contains(id: &T::AccountId) -> bool { + crate::ListNodes::::contains_key(id) } - /// Iterate over all nodes in all bags in the voter list. + /// Iterate over all nodes in all bags in the list. /// /// Full iteration can be expensive; it's recommended to limit the number of items with /// `.take(n)`. @@ -205,19 +210,20 @@ impl List { // otherwise, insert it here. Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) }; + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } - /// Insert several voters into the appropriate bags in the voter Continues with insertions if - /// duplicates are detected. + /// Insert several ids into the appropriate bags in the list. Continues with insertions + /// if duplicates are detected. /// - /// Return the final count of number of voters inserted. + /// Return the final count of number of ids inserted. fn insert_many( - voters: impl IntoIterator, + ids: impl IntoIterator, weight_of: impl Fn(&T::AccountId) -> VoteWeight, ) -> u32 { let mut count = 0; - voters.into_iter().for_each(|v| { + ids.into_iter().for_each(|v| { let weight = weight_of(&v); if Self::insert(v, weight).is_ok() { count += 1; @@ -227,51 +233,52 @@ impl List { count } - /// Insert a new voter into the appropriate bag in the voter list. + /// Insert a new id into the appropriate bag in the list. /// - /// Returns an error if the list already contains `voter`. - pub(crate) fn insert(voter: T::AccountId, weight: VoteWeight) -> Result<(), Error> { - if Self::contains(&voter) { + /// Returns an error if the list already contains `id`. + pub(crate) fn insert(id: T::AccountId, weight: VoteWeight) -> Result<(), Error> { + if Self::contains(&id) { return Err(Error::Duplicate) } let bag_weight = notional_bag_for::(weight); let mut bag = Bag::::get_or_make(bag_weight); - bag.insert(voter.clone()); + // unchecked insertion is okay; we just got the correct `notional_bag_for`. + bag.insert_unchecked(id.clone()); // new inserts are always the tail, so we must write the bag. bag.put(); - crate::CounterForVoters::::mutate(|prev_count| { + crate::CounterForListNodes::::mutate(|prev_count| { *prev_count = prev_count.saturating_add(1) }); crate::log!( debug, "inserted {:?} with weight {} into bag {:?}, new count is {}", - voter, + id, weight, bag_weight, - crate::CounterForVoters::::get(), + crate::CounterForListNodes::::get(), ); Ok(()) } - /// Remove a voter (by id) from the voter list. - pub(crate) fn remove(voter: &T::AccountId) { - Self::remove_many(sp_std::iter::once(voter)); + /// Remove a id from the list. + pub(crate) fn remove(id: &T::AccountId) { + Self::remove_many(sp_std::iter::once(id)); } - /// Remove many voters (by id) from the voter list. + /// Remove many ids from the list. /// /// This is more efficient than repeated calls to `Self::remove`. - fn remove_many<'a>(voters: impl IntoIterator) { + fn remove_many<'a>(ids: impl IntoIterator) { let mut bags = BTreeMap::new(); let mut count = 0; - for voter_id in voters.into_iter() { - let node = match Node::::get(voter_id) { + for id in ids.into_iter() { + let node = match Node::::get(id) { Some(node) => node, None => continue, }; @@ -285,31 +292,32 @@ impl List { let bag = bags .entry(node.bag_upper) .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); - bag.remove_node(&node); + // node.bag_upper must be correct, therefore this bag will contain this node. + bag.remove_node_unchecked(&node); } // now get rid of the node itself - crate::VoterNodes::::remove(voter_id); + crate::ListNodes::::remove(id); } for (_, bag) in bags { bag.put(); } - crate::CounterForVoters::::mutate(|prev_count| { + crate::CounterForListNodes::::mutate(|prev_count| { *prev_count = prev_count.saturating_sub(count) }); } - /// Update a voter's position in the voter list. + /// Update a node's position in the list. /// - /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they + /// If the node was in the correct bag, no effect. If the node was in the incorrect bag, they /// are moved into the correct bag. /// - /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. + /// Returns `Some((old_idx, new_idx))` if the node moved, otherwise `None`. /// /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by - /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient + /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient /// to call [`self.remove_many`] followed by [`self.insert_many`]. pub(crate) fn update_position_for( node: Node, @@ -324,49 +332,49 @@ impl List { node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { // this is a head or tail, so the bag must be updated. - bag.remove_node(&node); + bag.remove_node_unchecked(&node); bag.put(); } else { crate::log!( error, - "Node for voter {:?} did not have a bag; VoterBags is in an inconsistent state", + "Node {:?} did not have a bag; ListBags is in an inconsistent state", node.id, ); debug_assert!(false, "every node must have an extant bag associated with it"); } - // put the voter into the appropriate new bag. + // put the node into the appropriate new bag. let new_bag_upper = notional_bag_for::(new_weight); let mut bag = Bag::::get_or_make(new_bag_upper); // prev, next, and bag_upper of the node are updated inside `insert_node`, also // `node.put` is in there. - bag.insert_node(node); + bag.insert_node_unchecked(node); bag.put(); (old_bag_upper, new_bag_upper) }) } - /// Sanity check the voter list. + /// Sanity check the list. /// /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) /// is being used, after all other staking data (such as counter) has been updated. It checks /// that: /// - /// * Iterate all voters in list and make sure there are no duplicates. - /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. + /// * Iterate all ids in list and make sure there are no duplicates. + /// * Iterate all ids and ensure their count is in sync with `CounterForListNodes`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `List`. pub(crate) fn sanity_check() -> Result<(), &'static str> { use frame_support::ensure; let mut seen_in_list = BTreeSet::new(); ensure!( - Self::iter().map(|node| node.id).all(|voter| seen_in_list.insert(voter)), + Self::iter().map(|node| node.id).all(|id| seen_in_list.insert(id)), "duplicate identified", ); let iter_count = Self::iter().collect::>().len() as u32; - let stored_count = crate::CounterForVoters::::get(); + let stored_count = crate::CounterForListNodes::::get(); ensure!(iter_count == stored_count, "iter_count != stored_count",); let _ = T::BagThresholds::get() @@ -379,13 +387,13 @@ impl List { } } -/// A Bag is a doubly-linked list of voters. +/// A Bag is a doubly-linked list of ids, where each id is mapped to a [`ListNode`]. /// /// Note that we maintain both head and tail pointers. While it would be possible to get away with /// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more /// desirable to ensure that there is some element of first-come, first-serve to the list's -/// iteration so that there's no incentive to churn voter positioning to improve the chances of -/// appearing within the voter set. +/// iteration so that there's no incentive to churn ids positioning to improve the chances of +/// appearing within the ids set. #[derive(DefaultNoBound, Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] #[cfg_attr(test, derive(PartialEq))] @@ -413,7 +421,7 @@ impl Bag { T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, "it is a logic error to attempt to get a bag which is not in the thresholds list" ); - crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { + crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { bag.bag_upper = bag_upper; bag }) @@ -436,9 +444,9 @@ impl Bag { /// Put the bag back into storage. fn put(self) { if self.is_empty() { - crate::VoterBags::::remove(self.bag_upper); + crate::ListBags::::remove(self.bag_upper); } else { - crate::VoterBags::::insert(self.bag_upper, self); + crate::ListBags::::insert(self.bag_upper, self); } } @@ -457,26 +465,28 @@ impl Bag { sp_std::iter::successors(self.head(), |prev| prev.next()) } - /// Insert a new voter into this bag. + /// Insert a new id into this bag. /// /// This is private on purpose because it's naive: it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`List::insert`] instead. + /// appropriate bag for this id at all. Generally, use [`List::insert`] instead. /// /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. - fn insert(&mut self, id: T::AccountId) { - // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. - self.insert_node(Node:: { id, prev: None, next: None, bag_upper: self.bag_upper }); + fn insert_unchecked(&mut self, id: T::AccountId) { + // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long + // as this bag is the correct one, we're good. All calls to this must come after getting the + // correct [`notional_bag_for`], getting the bag, and then `bag.insert_unchecked`. + self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); } - /// Insert a voter node into this bag. + /// Insert a node into this bag. /// /// This is private on purpose because it's naive; it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`List::insert`] instead. + /// appropriate bag for this node at all. Generally, use [`List::insert`] instead. /// /// Storage note: this modifies storage, but only for the node. You still need to call /// `self.put()` after use. - fn insert_node(&mut self, mut node: Node) { + fn insert_node_unchecked(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { // this should never happen, but this check prevents a worst case infinite loop @@ -490,7 +500,7 @@ impl Bag { // to be `self.bag_upper`. node.bag_upper = self.bag_upper; - // update this node now, treating as the new tail. + // update this node now, making it the new tail. let id = node.id.clone(); node.prev = self.tail.clone(); node.next = None; @@ -505,21 +515,21 @@ impl Bag { // ensure head exist. This is only set when the length of the bag is just 1, i.e. if this is // the first insertion into the bag. In this case, both head and tail should point to the - // same voter node. + // same node. if self.head.is_none() { self.head = Some(id.clone()); debug_assert!(self.iter().count() == 1); } } - /// Remove a voter node from this bag. Returns true iff the bag's head or tail is updated. + /// Remove a node from this bag. /// - /// This is private on purpose because it doesn't check whether this bag contains the voter in - /// the first place. Generally, use [`List::remove`] instead. + /// This is private on purpose because it doesn't check whether this bag contains the node in + /// the first place. Generally, use [`List::remove`] instead, similar to `insert_unchecked`. /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()` and `VoterNodes::remove(voter_id)` to update storage for the bag and `node`. - fn remove_node(&mut self, node: &Node) { + /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. + fn remove_node_unchecked(&mut self, node: &Node) { // reassign neighboring nodes. node.excise(); @@ -585,13 +595,13 @@ pub(crate) struct Node { impl Node { /// Get a node by bag idx and account id. - pub(crate) fn get(account_id: &T::AccountId) -> Option> { - crate::VoterNodes::::try_get(account_id).ok() + pub(crate) fn get(id: &T::AccountId) -> Option> { + crate::ListNodes::::try_get(id).ok() } /// Put the node back into storage. fn put(self) { - crate::VoterNodes::::insert(self.id.clone(), self); + crate::ListNodes::::insert(self.id.clone(), self); } /// Update neighboring nodes to point to reach other. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index a5ef455e36e62..223926b4c2b6b 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::{ mock::{ext_builder::*, test_utils::*, *}, - CounterForVoters, VoterBags, VoterNodes, + CounterForListNodes, ListBags, ListNodes, }; use frame_election_provider_support::SortedListProvider; use frame_support::{assert_ok, assert_storage_noop}; @@ -12,29 +12,29 @@ fn basic_setup_works() { // syntactic sugar to create a raw node let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; - assert_eq!(CounterForVoters::::get(), 4); - assert_eq!(VoterNodes::::iter().count(), 4); - assert_eq!(VoterBags::::iter().count(), 2); + assert_eq!(CounterForListNodes::::get(), 4); + assert_eq!(ListNodes::::iter().count(), 4); + assert_eq!(ListBags::::iter().count(), 2); assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); // the state of the bags is as expected assert_eq!( - VoterBags::::get(10).unwrap(), + ListBags::::get(10).unwrap(), Bag:: { head: Some(1), tail: Some(1), bag_upper: 0 } ); assert_eq!( - VoterBags::::get(1_000).unwrap(), + ListBags::::get(1_000).unwrap(), Bag:: { head: Some(2), tail: Some(4), bag_upper: 0 } ); - assert_eq!(VoterNodes::::get(2).unwrap(), node(2, None, Some(3), 1000)); - assert_eq!(VoterNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1000)); - assert_eq!(VoterNodes::::get(4).unwrap(), node(4, Some(3), None, 1000)); - assert_eq!(VoterNodes::::get(1).unwrap(), node(1, None, None, 10)); + assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1000)); + assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1000)); + assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1000)); + assert_eq!(ListNodes::::get(1).unwrap(), node(1, None, None, 10)); // non-existent id does not have a storage footprint - assert_eq!(VoterNodes::::get(41), None); + assert_eq!(ListNodes::::get(41), None); // iteration of the bags would yield: assert_eq!( @@ -68,7 +68,7 @@ fn notional_bag_for_works() { } #[test] -fn remove_last_voter_in_bags_cleans_bag() { +fn remove_last_node_in_bags_cleans_bag() { ExtBuilder::default().build_and_execute(|| { // given assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); @@ -86,7 +86,7 @@ fn remove_last_voter_in_bags_cleans_bag() { }); } -mod voter_list { +mod list { use super::*; #[test] @@ -110,7 +110,7 @@ mod voter_list { ] ); - // when adding a voter that has a higher weight than pre-existing voters in the bag + // when adding a id that has a higher weight than pre-existing ids in the bag assert_ok!(List::::insert(7, 10)); // then @@ -119,13 +119,13 @@ mod voter_list { vec![ 5, 6, // best bag 2, 3, 4, // middle bag - 1, 7, // last bag; new voter is last. + 1, 7, // last bag; new id is last. ] ); }) } - /// This tests that we can `take` x voters, even if that quantity ends midway through a list. + /// This tests that we can `take` x ids, even if that quantity ends midway through a list. #[test] fn take_works() { ExtBuilder::default() @@ -187,16 +187,16 @@ mod voter_list { #[test] fn remove_works() { - use crate::{CounterForVoters, VoterBags, VoterNodes}; + use crate::{CounterForListNodes, ListBags, ListNodes}; let ensure_left = |id, counter| { - assert!(!VoterNodes::::contains_key(id)); - assert_eq!(CounterForVoters::::get(), counter); - assert_eq!(VoterNodes::::iter().count() as u32, counter); + assert!(!ListNodes::::contains_key(id)); + assert_eq!(CounterForListNodes::::get(), counter); + assert_eq!(ListNodes::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { - // removing a non-existent voter is a noop - assert!(!VoterNodes::::contains_key(42)); + // removing a non-existent id is a noop + assert!(!ListNodes::::contains_key(42)); assert_storage_noop!(List::::remove(&42)); // when removing a node from a bag with multiple nodes @@ -215,9 +215,9 @@ mod voter_list { assert_eq!(get_bags(), vec![(1000, vec![3, 4])]); ensure_left(1, 2); // bag 10 is removed - assert!(!VoterBags::::contains_key(10)); + assert!(!ListBags::::contains_key(10)); - // remove remaining voters to make sure storage cleans up as expected + // remove remaining ids to make sure storage cleans up as expected List::::remove(&3); ensure_left(3, 1); assert_eq!(get_list_as_ids(), vec![4]); @@ -227,7 +227,7 @@ mod voter_list { assert_eq!(get_list_as_ids(), Vec::::new()); // bags are deleted via removals - assert_eq!(VoterBags::::iter().count(), 0); + assert_eq!(ListBags::::iter().count(), 0); }); } @@ -291,14 +291,14 @@ mod voter_list { // make sure there are no duplicates. ExtBuilder::default().build_and_execute_no_post_check(|| { - Bag::::get(10).unwrap().insert(2); + Bag::::get(10).unwrap().insert_unchecked(2); assert_eq!(List::::sanity_check(), Err("duplicate identified")); }); - // ensure count is in sync with `CounterForVoters`. + // ensure count is in sync with `CounterForListNodes`. ExtBuilder::default().build_and_execute_no_post_check(|| { - crate::CounterForVoters::::mutate(|counter| *counter += 1); - assert_eq!(crate::CounterForVoters::::get(), 5); + crate::CounterForListNodes::::mutate(|counter| *counter += 1); + assert_eq!(crate::CounterForListNodes::::get(), 5); assert_eq!(List::::sanity_check(), Err("iter_count != stored_count")); }); } @@ -341,7 +341,7 @@ mod bags { .filter(|bag_upper| !vec![10, 1000].contains(bag_upper)) .for_each(|bag_upper| { assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); - assert!(!VoterBags::::contains_key(*bag_upper)); + assert!(!ListBags::::contains_key(*bag_upper)); }); // when we make a pre-existing bag empty @@ -369,10 +369,10 @@ mod bags { assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); let mut bag_10 = Bag::::get(10).unwrap(); - bag_10.insert_node(node(42, 5)); + bag_10.insert_node_unchecked(node(42, 5)); assert_eq!( - VoterNodes::::get(&42).unwrap(), + ListNodes::::get(&42).unwrap(), Node { bag_upper: 10, prev: Some(1), next: None, id: 42 } ); }); @@ -386,26 +386,26 @@ mod bags { // when inserting into a bag with 1 node let mut bag_10 = Bag::::get(10).unwrap(); // (note: bags api does not care about balance or ledger) - bag_10.insert_node(node(42, bag_10.bag_upper)); + bag_10.insert_node_unchecked(node(42, bag_10.bag_upper)); // then assert_eq!(bag_as_ids(&bag_10), vec![1, 42]); // when inserting into a bag with 3 nodes let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(52, bag_1000.bag_upper)); + bag_1000.insert_node_unchecked(node(52, bag_1000.bag_upper)); // then assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 52]); // when inserting into a new bag let mut bag_20 = Bag::::get_or_make(20); - bag_20.insert_node(node(62, bag_20.bag_upper)); + bag_20.insert_node_unchecked(node(62, bag_20.bag_upper)); // then assert_eq!(bag_as_ids(&bag_20), vec![62]); // when inserting a node pointing to the accounts not in the bag let node_61 = Node:: { id: 61, prev: Some(21), next: Some(101), bag_upper: 20 }; - bag_20.insert_node(node_61); + bag_20.insert_node_unchecked(node_61); // then ids are in order assert_eq!(bag_as_ids(&bag_20), vec![62, 61]); // and when the node is re-fetched all the info is correct @@ -431,7 +431,7 @@ mod bags { // when inserting a node with both prev & next pointing at an account in an incorrect // bag. let mut bag_1000 = Bag::::get(1_000).unwrap(); - bag_1000.insert_node(node(42, Some(1), Some(1), 500)); + bag_1000.insert_node_unchecked(node(42, Some(1), Some(1), 500)); // then the proper perv and next is set. assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 42]); @@ -449,7 +449,7 @@ mod bags { assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); // when inserting a node with duplicate id 3 - bag_1000.insert_node(node(3, None, None, bag_1000.bag_upper)); + bag_1000.insert_node_unchecked(node(3, None, None, bag_1000.bag_upper)); // then all the nodes after the duplicate are lost (because it is set as the tail) assert_eq!(bag_as_ids(&bag_1000), vec![2, 3]); @@ -467,7 +467,7 @@ mod bags { // when inserting a duplicate id of the head let mut bag_1000 = Bag::::get(1_000).unwrap(); assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); - bag_1000.insert_node(node(2, None, None, 0)); + bag_1000.insert_node_unchecked(node(2, None, None, 0)); // then all nodes after the head are lost assert_eq!(bag_as_ids(&bag_1000), vec![2]); @@ -496,7 +496,7 @@ mod bags { // when inserting a duplicate id that is already the tail assert_eq!(bag_1000.tail, Some(4)); - bag_1000.insert_node(node(4, None, None, bag_1000.bag_upper)); // panics + bag_1000.insert_node_unchecked(node(4, None, None, bag_1000.bag_upper)); // panics }); } @@ -527,7 +527,7 @@ mod bags { // when removing a node that is not pointing at the head or tail let node_4 = Node::::get(&4).unwrap(); let node_4_pre_remove = node_4.clone(); - bag_1000.remove_node(&node_4); + bag_1000.remove_node_unchecked(&node_4); // then assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 13, 14]); @@ -537,7 +537,7 @@ mod bags { // when removing a head that is not pointing at the tail let node_2 = Node::::get(&2).unwrap(); - bag_1000.remove_node(&node_2); + bag_1000.remove_node_unchecked(&node_2); // then assert_eq!(bag_as_ids(&bag_1000), vec![3, 13, 14]); @@ -545,7 +545,7 @@ mod bags { // when removing a tail that is not pointing at the head let node_14 = Node::::get(&14).unwrap(); - bag_1000.remove_node(&node_14); + bag_1000.remove_node_unchecked(&node_14); // then assert_eq!(bag_as_ids(&bag_1000), vec![3, 13]); @@ -553,7 +553,7 @@ mod bags { // when removing a tail that is pointing at the head let node_13 = Node::::get(&13).unwrap(); - bag_1000.remove_node(&node_13); + bag_1000.remove_node_unchecked(&node_13); // then assert_eq!(bag_as_ids(&bag_1000), vec![3]); @@ -561,7 +561,7 @@ mod bags { // when removing a node that is both the head & tail let node_3 = Node::::get(&3).unwrap(); - bag_1000.remove_node(&node_3); + bag_1000.remove_node_unchecked(&node_3); bag_1000.put(); // put into storage so `get` returns the updated bag // then @@ -569,7 +569,7 @@ mod bags { // when removing a node that is pointing at both the head & tail let node_11 = Node::::get(&11).unwrap(); - bag_10.remove_node(&node_11); + bag_10.remove_node_unchecked(&node_11); // then assert_eq!(bag_as_ids(&bag_10), vec![1, 12]); @@ -577,7 +577,7 @@ mod bags { // when removing a head that is pointing at the tail let node_1 = Node::::get(&1).unwrap(); - bag_10.remove_node(&node_1); + bag_10.remove_node_unchecked(&node_1); // then assert_eq!(bag_as_ids(&bag_10), vec![12]); @@ -588,7 +588,7 @@ mod bags { // when removing a node that is pointing at the head but not the tail let node_16 = Node::::get(&16).unwrap(); - bag_2000.remove_node(&node_16); + bag_2000.remove_node_unchecked(&node_16); // then assert_eq!(bag_as_ids(&bag_2000), vec![15, 17, 18, 19]); @@ -596,7 +596,7 @@ mod bags { // when removing a node that is pointing at tail, but not head let node_18 = Node::::get(&18).unwrap(); - bag_2000.remove_node(&node_18); + bag_2000.remove_node_unchecked(&node_18); // then assert_eq!(bag_as_ids(&bag_2000), vec![15, 17, 19]); @@ -619,7 +619,7 @@ mod bags { let mut bag_1000 = Bag::::get(1_000).unwrap(); // when removing a node that is in the bag but has the wrong upper - bag_1000.remove_node(&bad_upper_node_2); + bag_1000.remove_node_unchecked(&bad_upper_node_2); bag_1000.put(); // then the node is no longer in any bags @@ -640,7 +640,7 @@ mod bags { // when we remove it from bag 10 let mut bag_10 = Bag::::get(10).unwrap(); - bag_10.remove_node(&node_4); // node_101 is in bag 1_000 + bag_10.remove_node_unchecked(&node_4); // node_101 is in bag 1_000 bag_10.put(); // then bag remove was called on is ok, diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index a6df7c07dece8..2cf4f14c299a6 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -47,11 +47,8 @@ impl frame_system::Config for Runtime { type OnSetCode = (); } -/// Thresholds used for bags. -const THRESHOLDS: [VoteWeight; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; - parameter_types! { - pub static BagThresholds: &'static [VoteWeight] = &THRESHOLDS; + pub static BagThresholds: &'static [VoteWeight] = &[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; } impl crate::Config for Runtime { @@ -144,8 +141,4 @@ pub(crate) mod test_utils { pub(crate) fn get_list_as_ids() -> Vec { List::::iter().map(|n| *n.id()).collect::>() } - - pub(crate) fn set_bag_thresholds(thresholds: &'static [VoteWeight]) { - BagThresholds::set(thresholds); - } } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index c22299a0b956c..27e6f98716582 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -18,28 +18,28 @@ mod pallet { NextVoteWeight::set(2000); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); - // then a new bag is created and the voter moves into it + // then a new bag is created and the id moves into it assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); // when decreasing weight within the range of the current bag NextVoteWeight::set(1001); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); - // then the voter does not move + // then the id does not move assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); // when reducing weight to the level of a non-existent bag NextVoteWeight::set(30); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); - // then a new bag is created and the voter moves into it + // then a new bag is created and the id moves into it assert_eq!(get_bags(), vec![(10, vec![1]), (30, vec![42]), (1000, vec![2, 3, 4])]); // when increasing weight to the level of a pre-existing bag NextVoteWeight::set(500); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); - // then the voter moves into that bag + // then the id moves into that bag assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 42])]); }); } @@ -128,20 +128,39 @@ mod pallet { } #[test] - #[should_panic = "Voter bag thresholds must strictly increase, and have no duplicates"] + #[should_panic = "thresholds must strictly increase, and have no duplicates"] fn duplicate_in_bags_threshold_panics() { const DUPE_THRESH: &[VoteWeight; 4] = &[10, 20, 30, 30]; - set_bag_thresholds(DUPE_THRESH); + BagThresholds::set(DUPE_THRESH); BagsList::integrity_test(); } #[test] - #[should_panic = "Voter bag thresholds must strictly increase, and have no duplicates"] + #[should_panic = "thresholds must strictly increase, and have no duplicates"] fn decreasing_in_bags_threshold_panics() { const DECREASING_THRESH: &[VoteWeight; 4] = &[10, 30, 20, 40]; - set_bag_thresholds(DECREASING_THRESH); + BagThresholds::set(DECREASING_THRESH); BagsList::integrity_test(); } + + #[test] + fn empty_threshold_works() { + BagThresholds::set(Default::default()); // which is the same as passing `()` to `Get<_>`. + + ExtBuilder::default().build_and_execute(|| { + // everyone in the same bag. + assert_eq!(get_bags(), vec![(VoteWeight::MAX, vec![1, 2, 3, 4])]); + + // any insertion goes there as well. + assert_ok!(List::::insert(5, 999)); + assert_ok!(List::::insert(6, 0)); + assert_eq!(get_bags(), vec![(VoteWeight::MAX, vec![1, 2, 3, 4, 5, 6])]); + + // any rebag is noop. + assert_storage_noop!(assert!(BagsList::rebag(Origin::signed(0), 1).is_ok())); + assert_storage_noop!(assert!(BagsList::rebag(Origin::signed(0), 10).is_ok())); + }) + } } mod sorted_list_provider { @@ -229,7 +248,7 @@ mod sorted_list_provider { // when increasing weight to the level of non-existent bag BagsList::on_update(&42, 2_000); - // then the bag is created with the voter in it, + // then the bag is created with the id in it, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); // and the id position is updated in the list. assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); @@ -237,7 +256,7 @@ mod sorted_list_provider { // when decreasing weight within the range of the current bag BagsList::on_update(&42, 1_001); - // then the voter does not change bags, + // then the id does not change bags, assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); // or change position in the list. assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); @@ -245,7 +264,7 @@ mod sorted_list_provider { // when increasing weight to the level of a non-existent bag with the max threshold BagsList::on_update(&42, VoteWeight::MAX); - // the the new bag is created with the voter in it, + // the the new bag is created with the id in it, assert_eq!( get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (VoteWeight::MAX, vec![42])] @@ -256,7 +275,7 @@ mod sorted_list_provider { // when decreasing the weight to a pre-existing bag BagsList::on_update(&42, 1_000); - // then voter is moved to the correct bag (as the last member), + // then id is moved to the correct bag (as the last member), assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); // and the id position is updated in the list. assert_eq!(BagsList::iter().collect::>(), vec![2, 3, 4, 42, 1]); @@ -269,15 +288,15 @@ mod sorted_list_provider { #[test] fn on_remove_works() { let ensure_left = |id, counter| { - assert!(!VoterNodes::::contains_key(id)); + assert!(!ListNodes::::contains_key(id)); assert_eq!(BagsList::count(), counter); - assert_eq!(CounterForVoters::::get(), counter); - assert_eq!(VoterNodes::::iter().count() as u32, counter); + assert_eq!(CounterForListNodes::::get(), counter); + assert_eq!(ListNodes::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { - // it is a noop removing a non-existent voter - assert!(!VoterNodes::::contains_key(42)); + // it is a noop removing a non-existent id + assert!(!ListNodes::::contains_key(42)); assert_storage_noop!(BagsList::on_remove(&42)); // when removing a node from a bag with multiple nodes @@ -296,7 +315,7 @@ mod sorted_list_provider { assert_eq!(get_bags(), vec![(1_000, vec![3, 4])]); ensure_left(1, 2); - // when removing all remaining voters + // when removing all remaining ids BagsList::on_remove(&4); assert_eq!(get_list_as_ids(), vec![3]); ensure_left(4, 1); @@ -318,3 +337,8 @@ mod sorted_list_provider { }) } } + +#[test] +fn migrate_works() { + todo!() +} diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index c72b593172d19..cca3221dd2476 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -303,25 +303,23 @@ impl ElectionProvider for () { /// up having use cases outside of the election context, it is easy enough to make it generic over /// the `VoteWeight`. /// -/// Something that implements this trait will do a best-effort sort over voters, and thus can be -/// used on the implementing side of `ElectionDataProvider`. +/// Something that implements this trait will do a best-effort sort over ids, and thus can be +/// used on the implementing side of [`ElectionDataProvider`]. pub trait SortedListProvider { type Error; - /// Returns iterator over voter list, which can have `take` called on it. + /// Returns iterator over the list, which can have `take` called on it. fn iter() -> Box>; - /// get the current count of voters. + /// get the current count of ids in the list. fn count() -> u32; - /// Return true if the insertion can happen. - fn contains(voter: &AccountId) -> bool; - // Hook for inserting a voter. - fn on_insert(voter: AccountId, weight: VoteWeight) -> Result<(), Self::Error>; - /// Hook for updating a single voter. - fn on_update(voter: &AccountId, weight: VoteWeight); - /// Hook for removing a voter from the list. - fn on_remove(voter: &AccountId); - /// Sanity check internal state of list. Only meant for debug compilation. - fn sanity_check() -> Result<(), &'static str>; + /// Return true if the list already contains `id`. + fn contains(id: &AccountId) -> bool; + // Hook for inserting a new id. + fn on_insert(id: AccountId, weight: VoteWeight) -> Result<(), Self::Error>; + /// Hook for updating a single id. + fn on_update(id: &AccountId, weight: VoteWeight); + /// Hook for removing am id from the list. + fn on_remove(id: &AccountId); /// Regenerate this list from scratch. Returns the count of items inserted. /// /// This should typically only be used to enable this flag at a runtime upgrade. @@ -329,8 +327,10 @@ pub trait SortedListProvider { all: impl IntoIterator, weight_of: Box VoteWeight>, ) -> u32; - /// Remove everything + /// Remove everything. fn clear(); + /// Sanity check internal state of list. Only meant for debug compilation. + fn sanity_check() -> Result<(), &'static str>; } /// Something that can provide the `VoteWeight` of an account. Similar to [`ElectionProvider`] and diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 2914294faf4b3..0fff61b9a0288 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -83,8 +83,7 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] -# TODO: clean this flag. -make-bags = [ +generate-bags = [ "chrono", "git2", "num-format", diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index cb1313cd80cc8..0ec5ab4e1ce4d 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1232,17 +1232,17 @@ impl SortedListProvider for UseNominatorsMap { fn count() -> u32 { CounterForNominators::::get() } - fn contains(voter: &T::AccountId) -> bool { - Nominators::::contains_key(voter) + fn contains(id: &T::AccountId) -> bool { + Nominators::::contains_key(id) } - fn on_insert(_voter: T::AccountId, _weight: VoteWeight) -> Result<(), Self::Error> { + fn on_insert(_: T::AccountId, _weight: VoteWeight) -> Result<(), Self::Error> { // nothing to do on insert. Ok(()) } - fn on_update(_voter: &T::AccountId, _weight: VoteWeight) { + fn on_update(_: &T::AccountId, _weight: VoteWeight) { // nothing to do on update. } - fn on_remove(_voter: &T::AccountId) { + fn on_remove(_: &T::AccountId) { // nothing to do on remove. } fn regenerate( From db222f7763c06de03c9a1912da5b5b38ebbcbc84 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Sat, 7 Aug 2021 13:54:39 +0200 Subject: [PATCH 150/241] fix all tests and ci build --- frame/babe/src/mock.rs | 2 +- frame/bags-list/src/benchmarks.rs | 17 +++++++++++++++++ frame/bags-list/src/list/tests.rs | 17 +++++++++++++++++ frame/bags-list/src/mock.rs | 17 +++++++++++++++++ frame/bags-list/src/tests.rs | 17 +++++++++++++++++ frame/bags-list/src/weights.rs | 17 +++++++++++++++++ frame/election-provider-multi-phase/src/lib.rs | 1 + frame/grandpa/src/mock.rs | 2 +- frame/offences/benchmarking/src/mock.rs | 2 +- frame/session/benchmarking/src/mock.rs | 2 +- frame/staking/src/pallet/mod.rs | 2 +- 11 files changed, 91 insertions(+), 5 deletions(-) diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 9e1c67a959389..bcd6f9889280d 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -219,7 +219,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); - type VoterBagThresholds = (); + type SortedListProvider = pallet_staking::UseNominatorsMap; } impl pallet_offences::Config for Test { diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 83cb94d268c4b..63bae8f6c6009 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! Benchmarks for the bags list pallet. use super::*; diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 223926b4c2b6b..79a1f4bdcd270 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use super::*; use crate::{ mock::{ext_builder::*, test_utils::*, *}, diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 2cf4f14c299a6..bd12021ab242b 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use super::*; use frame_election_provider_support::VoteWeight; use frame_support::parameter_types; diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 27e6f98716582..6cc1a9d67a3ce 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use frame_support::{assert_ok, assert_storage_noop, traits::IntegrityTest}; use super::*; diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 0beabb2d49e31..456c4ec5f8a86 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use frame_support::pallet_prelude::Weight; pub trait WeightInfo { diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 01d1efff4e2bc..11f01b9a16482 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1991,6 +1991,7 @@ mod tests { }) } + #[test] fn snapshot_too_big_failure_onchain_fallback() { // the `MockStaking` is designed such that if it has too many targets, it simply fails. ExtBuilder::default().build_and_execute(|| { diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index d94e09d76d89b..a7b6178923b71 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -220,8 +220,8 @@ impl pallet_staking::Config for Test { type NextNewSession = Session; type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; + type SortedListProvider = pallet_staking::UseNominatorsMap; type WeightInfo = (); - type VoterBagThresholds = (); } impl pallet_offences::Config for Test { diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index af6191d4cb8e5..8218b6c8308f1 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -177,8 +177,8 @@ impl pallet_staking::Config for Test { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; + type SortedListProvider = pallet_staking::UseNominatorsMap; type WeightInfo = (); - type VoterBagThresholds = (); } impl pallet_im_online::Config for Test { diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 0bfe033dddce6..4c2612c4f32be 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -185,8 +185,8 @@ impl pallet_staking::Config for Test { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; + type SortedListProvider = pallet_staking::UseNominatorsMap; type WeightInfo = (); - type VoterBagThresholds = (); } impl crate::Config for Test {} diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index f95bd6ab041f6..06d85389f19f6 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -782,7 +782,7 @@ pub mod pallet { // update this staker in the sorted list, if they exist in it. if T::SortedListProvider::contains(&stash) { - T::SortedListProvider::on_update(&stash, Self::weight_of_fn(&ledger.stash)); + T::SortedListProvider::on_update(&stash, Self::weight_of(&ledger.stash)); } Self::deposit_event(Event::::Bonded(stash.clone(), extra)); From 70bfc53816d8a27d8fb2427bf0e78b96d023a32d Mon Sep 17 00:00:00 2001 From: kianenigma Date: Sat, 7 Aug 2021 14:05:00 +0200 Subject: [PATCH 151/241] nit --- frame/bags-list/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 90695565e1be2..7f22bd03f7b46 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -72,5 +72,5 @@ generate-bags = [ "git2", "num-format", "std", - "pallet-staking" + "pallet-staking" ] From f665ea3f0cba529afa435c25353b7b6b7f94ab51 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:43:13 -0700 Subject: [PATCH 152/241] Test and fix migration --- frame/bags-list/src/list/mod.rs | 23 ++++++++----------- frame/bags-list/src/list/tests.rs | 37 +++++++++++++++++++++++-------- frame/bags-list/src/mock.rs | 11 +++++++-- frame/bags-list/src/tests.rs | 5 ----- 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index b815f92af9c01..6795e9464fe28 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -135,12 +135,11 @@ impl List { // a new bag means that all accounts previously using the old bag's threshold must now // be rebagged for inserted_bag in new_bags { - // TODO: why use notional_bag_for here? This affected_bag is a new bag since - // new_set.difference(&old_set) is just the new bags; notional_bag_for is using the - // T::BagThresholds, which are the new threhsholds .. so plugging in a threhold to - // notional_bag_for will just give back that threshold. I think we want to instead - // use notional_bag_for logic with old_thresholds? - let affected_bag = notional_bag_for::(inserted_bag); + let affected_bag = { + // this recreates `notional_bag_for` logic, but with the old thresholds. + let idx = old_thresholds.partition_point(|&threshold| inserted_bag > threshold); + old_thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) + }; if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's // no point iterating through bag 10 twice. @@ -426,10 +425,10 @@ impl Bag { /// Get a bag by its upper vote weight. pub(crate) fn get(bag_upper: VoteWeight) -> Option> { - debug_assert!( - T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); + // debug_assert!( TODO this breaks when we attempt to migrate because thresholds change + // T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + // "it is a logic error to attempt to get a bag which is not in the thresholds list" + // ); crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { bag.bag_upper = bag_upper; bag @@ -438,10 +437,6 @@ impl Bag { /// Get a bag by its upper vote weight or make it, appropriately initialized. fn get_or_make(bag_upper: VoteWeight) -> Bag { - debug_assert!( - T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - "it is a logic error to attempt to get a bag which is not in the thresholds list" - ); Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 79a1f4bdcd270..59acc2a320dae 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -103,6 +103,34 @@ fn remove_last_node_in_bags_cleans_bag() { }); } +#[test] +fn migrate_works() { + ExtBuilder::default().add_ids(vec![(710, 15), (711, 16), (712, 2_000)]).build_and_execute_no_post_check(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![710, 711]), (1000, vec![2, 3, 4]), (2_000, vec![712])]); + let old_thresholds = ::BagThresholds::get(); + assert_eq!(old_thresholds, vec![10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]); + + // when the new thresholds adds `15` and removes `2_000` + const NEW_THRESHOLDS: &'static [VoteWeight] = &[10, 15, 20, 30, 40, 50, 60, 1_000, 10_000]; + BagThresholds::set(NEW_THRESHOLDS); + // and we call + List::::migrate(old_thresholds); + + // then + assert_eq!( + get_bags(), + vec![ + (10, vec![1]), + (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 + (20, vec![711]), + (1000, vec![2, 3, 4]), + (10_000, vec![712]), // nodes in range 1_001 ..= 2_000 move from bag 2_000 to bag 10_000 + ] + ); + }); +} + mod list { use super::*; @@ -369,15 +397,6 @@ mod bags { }); } - #[test] - #[should_panic] - fn get_panics_with_a_bad_threshold() { - // NOTE: panic is only expected with debug compilation - ExtBuilder::default().build_and_execute(|| { - Bag::::get(11); - }); - } - #[test] fn insert_node_sets_proper_bag() { ExtBuilder::default().build_and_execute_no_post_check(|| { diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index bd12021ab242b..440288511d333 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -28,8 +28,13 @@ parameter_types! { pub struct StakingMock; impl frame_election_provider_support::VoteWeightProvider for StakingMock { - fn vote_weight(_: &AccountId) -> VoteWeight { - NextVoteWeight::get() + fn vote_weight(id: &AccountId) -> VoteWeight { + match id { + 710 => 15, + 711 => 16, + 712 => 2_000, // special cases used for migrate test + _ => NextVoteWeight::get() + } } #[cfg(any(feature = "runtime-benchmarks", test))] fn set_vote_weight_of(_: &AccountId, weight: VoteWeight) { @@ -151,10 +156,12 @@ pub(crate) mod test_utils { .collect::>() } + /// Returns the ordered ids within the given bag. pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { bag.iter().map(|n| *n.id()).collect::>() } + /// Returns the ordered ids from the list. pub(crate) fn get_list_as_ids() -> Vec { List::::iter().map(|n| *n.id()).collect::>() } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 6cc1a9d67a3ce..f97e183ef5d13 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -354,8 +354,3 @@ mod sorted_list_provider { }) } } - -#[test] -fn migrate_works() { - todo!() -} From 56b6e23385905208464faaac292595b6565e78a8 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:52:33 -0700 Subject: [PATCH 153/241] nit --- frame/bags-list/src/list/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 59acc2a320dae..a6850bdd99cdc 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -105,7 +105,7 @@ fn remove_last_node_in_bags_cleans_bag() { #[test] fn migrate_works() { - ExtBuilder::default().add_ids(vec![(710, 15), (711, 16), (712, 2_000)]).build_and_execute_no_post_check(|| { + ExtBuilder::default().add_ids(vec![(710, 15), (711, 16), (712, 2_000)]).build_and_execute(|| { // given assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![710, 711]), (1000, vec![2, 3, 4]), (2_000, vec![712])]); let old_thresholds = ::BagThresholds::get(); From c09a2c03bb569ff1916b85102bddc5b533fc765f Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 9 Aug 2021 20:26:51 -0700 Subject: [PATCH 154/241] fmt --- frame/bags-list/src/list/tests.rs | 55 ++++++++++++++++++------------- frame/bags-list/src/mock.rs | 2 +- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index a6850bdd99cdc..92562f9a1fcd7 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -105,30 +105,41 @@ fn remove_last_node_in_bags_cleans_bag() { #[test] fn migrate_works() { - ExtBuilder::default().add_ids(vec![(710, 15), (711, 16), (712, 2_000)]).build_and_execute(|| { - // given - assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![710, 711]), (1000, vec![2, 3, 4]), (2_000, vec![712])]); - let old_thresholds = ::BagThresholds::get(); - assert_eq!(old_thresholds, vec![10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]); + ExtBuilder::default() + .add_ids(vec![(710, 15), (711, 16), (712, 2_000)]) + .build_and_execute(|| { + // given + assert_eq!( + get_bags(), + vec![ + (10, vec![1]), + (20, vec![710, 711]), + (1000, vec![2, 3, 4]), + (2_000, vec![712]) + ] + ); + let old_thresholds = ::BagThresholds::get(); + assert_eq!(old_thresholds, vec![10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]); - // when the new thresholds adds `15` and removes `2_000` - const NEW_THRESHOLDS: &'static [VoteWeight] = &[10, 15, 20, 30, 40, 50, 60, 1_000, 10_000]; - BagThresholds::set(NEW_THRESHOLDS); - // and we call - List::::migrate(old_thresholds); + // when the new thresholds adds `15` and removes `2_000` + const NEW_THRESHOLDS: &'static [VoteWeight] = + &[10, 15, 20, 30, 40, 50, 60, 1_000, 10_000]; + BagThresholds::set(NEW_THRESHOLDS); + // and we call + List::::migrate(old_thresholds); - // then - assert_eq!( - get_bags(), - vec![ - (10, vec![1]), - (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 - (20, vec![711]), - (1000, vec![2, 3, 4]), - (10_000, vec![712]), // nodes in range 1_001 ..= 2_000 move from bag 2_000 to bag 10_000 - ] - ); - }); + // then + assert_eq!( + get_bags(), + vec![ + (10, vec![1]), + (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 + (20, vec![711]), + (1000, vec![2, 3, 4]), + (10_000, vec![712]), // nodes in range 1_001 ..= 2_000 move from bag 2_000 to bag 10_000 + ] + ); + }); } mod list { diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 440288511d333..f845d2be2c38c 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -33,7 +33,7 @@ impl frame_election_provider_support::VoteWeightProvider for StakingM 710 => 15, 711 => 16, 712 => 2_000, // special cases used for migrate test - _ => NextVoteWeight::get() + _ => NextVoteWeight::get(), } } #[cfg(any(feature = "runtime-benchmarks", test))] From b81cd60f57b92a4fd8dcc30b8fd460e437eacbd6 Mon Sep 17 00:00:00 2001 From: Parity Benchmarking Bot Date: Tue, 10 Aug 2021 06:15:46 +0000 Subject: [PATCH 155/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_staking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/staking/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/staking/src/weights.rs | 334 +++++++++++++++++------------------ 1 file changed, 161 insertions(+), 173 deletions(-) diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 3cd09225736b8..c1df7dfc898d1 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-08-10, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -73,8 +73,6 @@ pub trait WeightInfo { fn get_npos_targets(v: u32, ) -> Weight; fn set_staking_limits() -> Weight; fn chill_other() -> Weight; - fn rebag() -> Weight; - fn regenerate(v: u32, n: u32, ) -> Weight; } /// Weights for pallet_staking using the Substrate node and recommended hardware. @@ -87,16 +85,17 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (72_423_000 as Weight) + (74_199_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:1) + // Storage: BagsList ListNodes (r:1 w:0) // Storage: Balances Locks (r:1 w:1) fn bond_extra() -> Weight { - (56_157_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (62_360_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Staking Ledger (r:1 w:1) @@ -105,9 +104,10 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:1 w:0) fn unbond() -> Weight { - (59_039_000 as Weight) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) + (64_317_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Staking Ledger (r:1 w:1) @@ -115,9 +115,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (51_503_000 as Weight) + (53_293_000 as Weight) // Standard Error: 0 - .saturating_add((59_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((52_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -132,9 +132,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (84_211_000 as Weight) - // Standard Error: 4_000 - .saturating_add((2_391_000 as Weight).saturating_mul(s as Weight)) + (85_914_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_449_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -146,16 +146,16 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Nominators (r:1 w:0) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (34_206_000 as Weight) + (35_121_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (22_863_000 as Weight) - // Standard Error: 13_000 - .saturating_add((16_208_000 as Weight).saturating_mul(k as Weight)) + (22_505_000 as Weight) + // Standard Error: 14_000 + .saturating_add((16_774_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -167,60 +167,64 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Validators (r:2 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListNodes (r:1 w:1) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (41_047_000 as Weight) - // Standard Error: 10_000 - .saturating_add((5_611_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (70_642_000 as Weight) + // Standard Error: 15_000 + .saturating_add((5_780_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(T::DbWeight::get().writes(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Validators (r:1 w:0) // Storage: Staking Nominators (r:1 w:0) fn chill() -> Weight { - (17_489_000 as Weight) + (17_992_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_384_000 as Weight) + (13_736_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (27_863_000 as Weight) + (27_961_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_468_000 as Weight) + (2_707_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_798_000 as Weight) + (3_009_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_763_000 as Weight) + (3_024_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_707_000 as Weight) + (2_992_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_353_000 as Weight) + (3_521_000 as Weight) // Standard Error: 0 - .saturating_add((56_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((55_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) @@ -233,18 +237,18 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (60_682_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_384_000 as Weight).saturating_mul(s as Weight)) + (61_777_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_455_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_368_335_000 as Weight) - // Standard Error: 221_000 - .saturating_add((19_815_000 as Weight).saturating_mul(s as Weight)) + (3_397_386_000 as Weight) + // Standard Error: 225_000 + .saturating_add((19_941_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -259,9 +263,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (108_594_000 as Weight) - // Standard Error: 15_000 - .saturating_add((46_477_000 as Weight).saturating_mul(n as Weight)) + (120_312_000 as Weight) + // Standard Error: 17_000 + .saturating_add((48_314_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -279,9 +283,9 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (157_564_000 as Weight) - // Standard Error: 20_000 - .saturating_add((59_781_000 as Weight).saturating_mul(n as Weight)) + (158_876_000 as Weight) + // Standard Error: 23_000 + .saturating_add((61_974_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) @@ -290,11 +294,12 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:1 w:0) fn rebond(l: u32, ) -> Weight { - (48_497_000 as Weight) - // Standard Error: 3_000 - .saturating_add((89_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (54_398_000 as Weight) + // Standard Error: 4_000 + .saturating_add((55_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Staking CurrentEra (r:1 w:0) @@ -308,8 +313,8 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 73_000 - .saturating_add((34_176_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 72_000 + .saturating_add((35_311_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -325,9 +330,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (71_895_000 as Weight) - // Standard Error: 0 - .saturating_add((2_376_000 as Weight).saturating_mul(s as Weight)) + (72_621_000 as Weight) + // Standard Error: 3_000 + .saturating_add((2_466_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -338,7 +343,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Validators (r:2 w:0) // Storage: Staking Bonded (r:101 w:0) // Storage: Staking Ledger (r:101 w:0) - // Storage: Staking Nominators (r:101 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:100 w:0) + // Storage: Staking Nominators (r:100 w:0) // Storage: Staking ValidatorCount (r:1 w:0) // Storage: System BlockWeight (r:1 w:1) // Storage: Staking MinimumValidatorCount (r:1 w:0) @@ -351,30 +358,34 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 980_000 - .saturating_add((300_866_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 49_000 - .saturating_add((46_397_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(10 as Weight)) + // Standard Error: 928_000 + .saturating_add((312_443_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 46_000 + .saturating_add((51_477_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(209 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } + // Storage: Staking CounterForNominators (r:1 w:0) + // Storage: Staking CounterForValidators (r:1 w:0) // Storage: Staking Validators (r:501 w:0) // Storage: Staking Bonded (r:1500 w:0) // Storage: Staking Ledger (r:1500 w:0) // Storage: Staking SlashingSpans (r:21 w:0) - // Storage: Staking Nominators (r:1001 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:1000 w:0) + // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 98_000 - .saturating_add((24_916_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 98_000 - .saturating_add((26_575_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_335_000 - .saturating_add((22_464_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + // Standard Error: 101_000 + .saturating_add((27_748_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 101_000 + .saturating_add((32_798_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_455_000 + .saturating_add((7_943_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(204 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) @@ -383,7 +394,7 @@ impl WeightInfo for SubstrateWeight { fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) // Standard Error: 32_000 - .saturating_add((10_706_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((11_334_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -393,7 +404,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_463_000 as Weight) + (6_625_000 as Weight) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) @@ -404,28 +415,10 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking CounterForValidators (r:1 w:1) // Storage: Staking MinValidatorBond (r:1 w:0) fn chill_other() -> Weight { - (56_717_000 as Weight) + (58_129_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } - fn rebag() -> Weight { - (84_501_000 as Weight) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(5 as Weight)) - } - fn regenerate(v: u32, n: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 128_000 - .saturating_add((40_328_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 128_000 - .saturating_add((42_763_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) - .saturating_add(T::DbWeight::get().writes(14 as Weight)) - .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) - } } // For backwards compatibility and tests @@ -437,16 +430,17 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (72_423_000 as Weight) + (74_199_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:1) + // Storage: BagsList ListNodes (r:1 w:0) // Storage: Balances Locks (r:1 w:1) fn bond_extra() -> Weight { - (56_157_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (62_360_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Staking Ledger (r:1 w:1) @@ -455,9 +449,10 @@ impl WeightInfo for () { // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:1 w:0) fn unbond() -> Weight { - (59_039_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + (64_317_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Staking Ledger (r:1 w:1) @@ -465,9 +460,9 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (51_503_000 as Weight) + (53_293_000 as Weight) // Standard Error: 0 - .saturating_add((59_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((52_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -482,9 +477,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (84_211_000 as Weight) - // Standard Error: 4_000 - .saturating_add((2_391_000 as Weight).saturating_mul(s as Weight)) + (85_914_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_449_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -496,16 +491,16 @@ impl WeightInfo for () { // Storage: Staking Nominators (r:1 w:0) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (34_206_000 as Weight) + (35_121_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (22_863_000 as Weight) - // Standard Error: 13_000 - .saturating_add((16_208_000 as Weight).saturating_mul(k as Weight)) + (22_505_000 as Weight) + // Standard Error: 14_000 + .saturating_add((16_774_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -517,60 +512,64 @@ impl WeightInfo for () { // Storage: Staking Validators (r:2 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListNodes (r:1 w:1) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (41_047_000 as Weight) - // Standard Error: 10_000 - .saturating_add((5_611_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (70_642_000 as Weight) + // Standard Error: 15_000 + .saturating_add((5_780_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(RocksDbWeight::get().writes(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Validators (r:1 w:0) // Storage: Staking Nominators (r:1 w:0) fn chill() -> Weight { - (17_489_000 as Weight) + (17_992_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_384_000 as Weight) + (13_736_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (27_863_000 as Weight) + (27_961_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_468_000 as Weight) + (2_707_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_798_000 as Weight) + (3_009_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_763_000 as Weight) + (3_024_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_707_000 as Weight) + (2_992_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_353_000 as Weight) + (3_521_000 as Weight) // Standard Error: 0 - .saturating_add((56_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((55_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) @@ -583,18 +582,18 @@ impl WeightInfo for () { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (60_682_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_384_000 as Weight).saturating_mul(s as Weight)) + (61_777_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_455_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_368_335_000 as Weight) - // Standard Error: 221_000 - .saturating_add((19_815_000 as Weight).saturating_mul(s as Weight)) + (3_397_386_000 as Weight) + // Standard Error: 225_000 + .saturating_add((19_941_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -609,9 +608,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (108_594_000 as Weight) - // Standard Error: 15_000 - .saturating_add((46_477_000 as Weight).saturating_mul(n as Weight)) + (120_312_000 as Weight) + // Standard Error: 17_000 + .saturating_add((48_314_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) @@ -629,9 +628,9 @@ impl WeightInfo for () { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (157_564_000 as Weight) - // Standard Error: 20_000 - .saturating_add((59_781_000 as Weight).saturating_mul(n as Weight)) + (158_876_000 as Weight) + // Standard Error: 23_000 + .saturating_add((61_974_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) @@ -640,11 +639,12 @@ impl WeightInfo for () { // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: BagsList ListNodes (r:1 w:0) fn rebond(l: u32, ) -> Weight { - (48_497_000 as Weight) - // Standard Error: 3_000 - .saturating_add((89_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (54_398_000 as Weight) + // Standard Error: 4_000 + .saturating_add((55_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Staking CurrentEra (r:1 w:0) @@ -658,8 +658,8 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 73_000 - .saturating_add((34_176_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 72_000 + .saturating_add((35_311_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -675,9 +675,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (71_895_000 as Weight) - // Standard Error: 0 - .saturating_add((2_376_000 as Weight).saturating_mul(s as Weight)) + (72_621_000 as Weight) + // Standard Error: 3_000 + .saturating_add((2_466_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -688,7 +688,9 @@ impl WeightInfo for () { // Storage: Staking Validators (r:2 w:0) // Storage: Staking Bonded (r:101 w:0) // Storage: Staking Ledger (r:101 w:0) - // Storage: Staking Nominators (r:101 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:100 w:0) + // Storage: Staking Nominators (r:100 w:0) // Storage: Staking ValidatorCount (r:1 w:0) // Storage: System BlockWeight (r:1 w:1) // Storage: Staking MinimumValidatorCount (r:1 w:0) @@ -701,30 +703,34 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 980_000 - .saturating_add((300_866_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 49_000 - .saturating_add((46_397_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + // Standard Error: 928_000 + .saturating_add((312_443_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 46_000 + .saturating_add((51_477_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(209 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } + // Storage: Staking CounterForNominators (r:1 w:0) + // Storage: Staking CounterForValidators (r:1 w:0) // Storage: Staking Validators (r:501 w:0) // Storage: Staking Bonded (r:1500 w:0) // Storage: Staking Ledger (r:1500 w:0) // Storage: Staking SlashingSpans (r:21 w:0) - // Storage: Staking Nominators (r:1001 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:1000 w:0) + // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 98_000 - .saturating_add((24_916_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 98_000 - .saturating_add((26_575_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_335_000 - .saturating_add((22_464_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + // Standard Error: 101_000 + .saturating_add((27_748_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 101_000 + .saturating_add((32_798_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_455_000 + .saturating_add((7_943_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(204 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) @@ -733,7 +739,7 @@ impl WeightInfo for () { fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) // Standard Error: 32_000 - .saturating_add((10_706_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((11_334_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -743,7 +749,7 @@ impl WeightInfo for () { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_463_000 as Weight) + (6_625_000 as Weight) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) @@ -754,26 +760,8 @@ impl WeightInfo for () { // Storage: Staking CounterForValidators (r:1 w:1) // Storage: Staking MinValidatorBond (r:1 w:0) fn chill_other() -> Weight { - (56_717_000 as Weight) + (58_129_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } - fn rebag() -> Weight { - (84_501_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(5 as Weight)) - } - fn regenerate(v: u32, n: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 128_000 - .saturating_add((40_328_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 128_000 - .saturating_add((42_763_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) - .saturating_add(RocksDbWeight::get().writes(14 as Weight)) - .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) - } } From 601521f47cbad9e2990ee622294eb0926b3e4bf3 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 16 Aug 2021 13:22:06 +0200 Subject: [PATCH 156/241] fmt --- Cargo.lock | 3 --- frame/bags-list/src/generate_bags.rs | 4 ++-- frame/bags-list/src/lib.rs | 4 ++-- frame/bags-list/src/list/mod.rs | 8 ++++---- frame/bags-list/src/list/tests.rs | 3 ++- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f88f8ef39f8b..441beb0fbc883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5539,7 +5539,6 @@ dependencies = [ "frame-support", "frame-system", "git2", - "hex", "log 0.4.14", "num-format", "pallet-authorship", @@ -5549,8 +5548,6 @@ dependencies = [ "pallet-staking-reward-curve", "pallet-timestamp", "parity-scale-codec", - "parking_lot 0.11.1", - "paste 1.0.4", "rand 0.8.4", "rand_chacha 0.2.2", "serde", diff --git a/frame/bags-list/src/generate_bags.rs b/frame/bags-list/src/generate_bags.rs index b6d65cb2dda88..3f47798a87e2b 100644 --- a/frame/bags-list/src/generate_bags.rs +++ b/frame/bags-list/src/generate_bags.rs @@ -37,8 +37,8 @@ //! ``` //! //! 2. Write a little program to generate the definitions. This can be a near-identical copy of -//! `substrate/frame/bags-list/generate-bags`. This program exists only to hook together the runtime -//! definitions with the various calculations here. +//! `substrate/frame/bags-list/generate-bags`. This program exists only to hook together the +//! runtime definitions with the various calculations here. //! //! 3. Run that program: //! diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 6f67e8017f2b1..7a2180200e50a 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -142,8 +142,8 @@ pub mod pallet { /// the procedure given above, then the constant ratio is equal to 2. /// - If `BagThresholds::get().len() == 200`, and the thresholds are determined according to /// the procedure given above, then the constant ratio is approximately equal to 1.248. - /// - If the threshold list begins `[1, 2, 3, ...]`, then an id with weight 0 or 1 will - /// fall into bag 0, an id with weight 2 will fall into bag 1, etc. + /// - If the threshold list begins `[1, 2, 3, ...]`, then an id with weight 0 or 1 will fall + /// into bag 0, an id with weight 2 will fall into bag 1, etc. /// /// # Migration /// diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 6795e9464fe28..15bcb1de3a71b 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -111,10 +111,10 @@ impl List { /// Postconditions: /// /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. - /// - No id is changed unless required to by the difference between the old threshold list - /// and the new. - /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the - /// new threshold set. + /// - No id is changed unless required to by the difference between the old threshold list and + /// the new. + /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the new + /// threshold set. #[allow(dead_code)] pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { // we can't check all preconditions, but we can check one diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 92562f9a1fcd7..28d3cb031a59f 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -136,7 +136,8 @@ fn migrate_works() { (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 (20, vec![711]), (1000, vec![2, 3, 4]), - (10_000, vec![712]), // nodes in range 1_001 ..= 2_000 move from bag 2_000 to bag 10_000 + (10_000, vec![712]), /* nodes in range 1_001 ..= 2_000 move from bag 2_000 + * to bag 10_000 */ ] ); }); From 81b3aaa5fc6e6eff85673be7795b11e6b037c895 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 16 Aug 2021 13:22:46 +0200 Subject: [PATCH 157/241] remove unused --- frame/bags-list/src/mock.rs | 2 +- frame/staking/src/mock.rs | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index f845d2be2c38c..fe57a54e44e80 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -45,7 +45,7 @@ impl frame_election_provider_support::VoteWeightProvider for StakingM impl frame_system::Config for Runtime { type SS58Prefix = (); - type BaseCallFilter = frame_support::traits::AllowAll; + type BaseCallFilter = frame_support::traits::Everything; type Origin = Origin; type Index = u64; type BlockNumber = u64; diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index f5602dec2cac3..c8d702917bd58 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -870,15 +870,3 @@ pub(crate) fn staking_events() -> Vec> { pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { (Balances::free_balance(who), Balances::reserved_balance(who)) } - -/// ensure that the given staking ledger has `total`, `own`, and is being only backed by `others`. -pub(crate) fn assert_eq_exposure( - exposure: Exposure, - total: Balance, - own: Balance, - others: Vec>, -) { - assert_eq!(exposure.total, total); - assert_eq!(exposure.own, own); - substrate_test_utils::assert_eq_uvec!(exposure.others, others); -} From 66f9eda66991618483af7e6a5220fb41eeb8301c Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 18 Aug 2021 13:59:01 +0200 Subject: [PATCH 158/241] make a few things pub --- frame/bags-list/src/lib.rs | 4 +++- frame/bags-list/src/list/mod.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 7a2180200e50a..f8b09fb512762 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -163,8 +163,10 @@ pub mod pallet { /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. + // TODO: we make this public for now only for the sake of the remote-ext tests, find another way + // around it. #[pallet::storage] - pub(crate) type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; + pub type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 15bcb1de3a71b..685142d227b8c 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -465,7 +465,8 @@ impl Bag { } /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { + // TODO: we make this public for the remote-ext test. + pub fn iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } From 864d3486f7996b553e135885eddd0b666a43e614 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 18 Aug 2021 14:02:37 +0200 Subject: [PATCH 159/241] make node also pub.. for remote-ext test --- frame/bags-list/src/list/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 685142d227b8c..c2e42cc48ec3e 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -592,7 +592,7 @@ impl Bag { #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] #[cfg_attr(test, derive(PartialEq, Clone))] -pub(crate) struct Node { +pub struct Node { id: T::AccountId, prev: Option, next: Option, From 4c65bde741200f22689f23c472c131e93958beac Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 19 Aug 2021 11:12:31 +0200 Subject: [PATCH 160/241] Fix all tests again --- frame/staking/src/tests.rs | 56 +++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 568f98705df70..c573dab4eec14 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -542,8 +542,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 800, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 400 }, IndividualExposure { who: 1, value: 400 }, + IndividualExposure { who: 3, value: 400 }, ] }, ); @@ -553,8 +553,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 1200, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 600 }, IndividualExposure { who: 1, value: 600 }, + IndividualExposure { who: 3, value: 600 }, ] }, ); @@ -1907,8 +1907,8 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider() { assert_eq!( supports, vec![ - (21, Support { total: 1800, voters: vec![(21, 1000), (3, 400), (1, 400)] }), - (31, Support { total: 2200, voters: vec![(31, 1000), (3, 600), (1, 600)] }) + (21, Support { total: 1800, voters: vec![(21, 1000), (1, 400), (3, 400)] }), + (31, Support { total: 2200, voters: vec![(31, 1000), (1, 600), (3, 600)] }) ], ); }); @@ -1952,7 +1952,7 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider_elected() { supports, vec![ (11, Support { total: 1500, voters: vec![(11, 1000), (1, 500)] }), - (21, Support { total: 2500, voters: vec![(21, 1000), (3, 1000), (1, 500)] }) + (21, Support { total: 2500, voters: vec![(21, 1000), (1, 500), (3, 1000)] }) ], ); }); @@ -3877,26 +3877,32 @@ mod election_data_provider { #[test] fn respects_snapshot_len_limits() { - ExtBuilder::default().build_and_execute(|| { - // sum of all nominators who'd be voters, plus the self votes. - assert_eq!( - ::SortedListProvider::count() + - >::iter().count() as u32, - 5 - ); - - // if limits is less.. - assert_eq!(Staking::voters(Some(1)).unwrap().len(), 1); - - // if limit is equal.. - assert_eq!(Staking::voters(Some(5)).unwrap().len(), 5); - - // if limit is more. - assert_eq!(Staking::voters(Some(55)).unwrap().len(), 5); - - // if target limit is less, then we return an error. - assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); - }); + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // sum of all nominators who'd be voters (1), plus the self-votes (4). + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 5 + ); + + // if limits is less.. + assert_eq!(Staking::voters(Some(1)).unwrap().len(), 1); + + // if limit is equal.. + assert_eq!(Staking::voters(Some(5)).unwrap().len(), 5); + + // if limit is more. + assert_eq!(Staking::voters(Some(55)).unwrap().len(), 5); + + // if target limit is more.. + assert_eq!(Staking::targets(Some(6)).unwrap().len(), 4); + assert_eq!(Staking::targets(Some(4)).unwrap().len(), 4); + + // if target limit is less, then we return an error. + assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); + }); } #[test] From bd835d4794437bfd0b8651d8b6fe876d631b4e3e Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Sat, 28 Aug 2021 21:27:00 +0200 Subject: [PATCH 161/241] Force bag changes in relevant benchmarks (targets #9507) (#9529) * force rebag for unbond, rebond, and bond_extra * nit * Improve utils * fmt * nits * Move generate_bags to its own pallet * Get runtime-benchmarks feature setup with prepare_on_update_benchmark * Withdraw unbonded kill working * Nominate bench working * some cleanup * WIP * update to check head pre & post conditions * Add some post condition verification stuff for on_remove * Update nominate * fmt * Improvements * Fix build * fix build with polkadot companion * Update frame/bags-list/src/list/tests.rs Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> * move generate-bag from frame to utils * wip * refactor WIP * WIP save * Refactor working * some variable renaming * WIP: prepare to remove head checks * Finish MvP refactor * Some cleanup * Soem more cleanup * save * fix a lot of stuff * Update client/db/src/bench.rs Co-authored-by: Shawn Tabrizi * Apply suggestions from code review * Apply suggestions from code review * Fix some issues that came from trying to merge comments on github * some small changes * simplify it Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: kianenigma Co-authored-by: Shawn Tabrizi --- Cargo.lock | 162 +++++----- Cargo.toml | 3 +- client/db/src/bench.rs | 2 +- frame/bags-list/Cargo.toml | 15 +- frame/bags-list/src/lib.rs | 24 +- frame/bags-list/src/list/mod.rs | 9 +- frame/bags-list/src/list/tests.rs | 4 +- frame/election-provider-support/src/lib.rs | 7 + frame/session/benchmarking/src/lib.rs | 2 +- frame/staking/Cargo.toml | 18 +- frame/staking/src/benchmarking.rs | 299 ++++++++++++++++-- frame/staking/src/pallet/impls.rs | 1 - frame/staking/src/pallet/mod.rs | 3 +- frame/staking/src/testing_utils.rs | 38 ++- utils/frame/generate-bags/Cargo.toml | 28 ++ .../generate-bags/node-runtime}/Cargo.toml | 9 +- .../generate-bags/node-runtime}/src/main.rs | 7 +- .../frame/generate-bags/src/lib.rs | 5 +- 18 files changed, 469 insertions(+), 167 deletions(-) create mode 100644 utils/frame/generate-bags/Cargo.toml rename {frame/bags-list/generate-bags => utils/frame/generate-bags/node-runtime}/Cargo.toml (57%) rename {frame/bags-list/generate-bags => utils/frame/generate-bags/node-runtime}/src/main.rs (82%) rename frame/bags-list/src/generate_bags.rs => utils/frame/generate-bags/src/lib.rs (97%) diff --git a/Cargo.lock b/Cargo.lock index fb706d7d97d16..8550c6f769457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,7 +318,7 @@ dependencies = [ "memchr", "num_cpus", "once_cell", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", "pin-utils", "slab", "wasm-bindgen-futures", @@ -365,7 +365,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", ] [[package]] @@ -378,7 +378,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", ] [[package]] @@ -695,9 +695,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "byte-slice-cast" @@ -1344,9 +1344,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8f45d9ad417bcef4817d614a501ab55cdd96a6fdb24f49aab89a54acfd66b19" +checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" dependencies = [ "quote", "syn", @@ -2262,7 +2262,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", "waker-fn", ] @@ -2329,7 +2329,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -2342,6 +2342,22 @@ version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" +[[package]] +name = "generate-bags" +version = "3.0.0" +dependencies = [ + "chrono", + "frame-election-provider-support", + "frame-support", + "frame-system", + "git2", + "node-runtime", + "num-format", + "pallet-staking", + "sp-io", + "structopt", +] + [[package]] name = "generic-array" version = "0.12.4" @@ -2532,9 +2548,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] @@ -2612,9 +2628,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" dependencies = [ "bytes 1.0.1", "fnv", @@ -2623,13 +2639,13 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" dependencies = [ "bytes 1.0.1", "http", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", ] [[package]] @@ -2693,8 +2709,8 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.6", - "socket2 0.4.0", + "pin-project-lite 0.2.7", + "socket2 0.4.1", "tokio 1.10.0", "tower-service", "tracing", @@ -2838,9 +2854,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" dependencies = [ "cfg-if 1.0.0", ] @@ -3232,9 +3248,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" +checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" [[package]] name = "libgit2-sys" @@ -3485,7 +3501,7 @@ dependencies = [ "log 0.4.14", "rand 0.8.4", "smallvec 1.6.1", - "socket2 0.4.0", + "socket2 0.4.1", "void", ] @@ -3658,7 +3674,7 @@ dependencies = [ "libc", "libp2p-core", "log 0.4.14", - "socket2 0.4.0", + "socket2 0.4.1", ] [[package]] @@ -3863,9 +3879,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] @@ -4116,7 +4132,7 @@ checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" dependencies = [ "libc", "log 0.4.14", - "miow 0.3.6", + "miow 0.3.7", "ntapi", "winapi 0.3.9", ] @@ -4158,11 +4174,10 @@ dependencies = [ [[package]] name = "miow" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "socket2 0.3.19", "winapi 0.3.9", ] @@ -4642,9 +4657,8 @@ name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ "frame-support", + "generate-bags", "node-runtime", - "pallet-bags-list", - "pallet-staking", "sp-io", "structopt", ] @@ -4898,9 +4912,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" dependencies = [ "parking_lot 0.11.1", ] @@ -5082,16 +5096,12 @@ dependencies = [ name = "pallet-bags-list" version = "4.0.0-dev" dependencies = [ - "chrono", "frame-benchmarking", "frame-election-provider-support", "frame-support", "frame-system", - "git2", "log 0.4.14", - "num-format", "pallet-balances", - "pallet-staking", "parity-scale-codec", "sp-core", "sp-io", @@ -5750,14 +5760,11 @@ dependencies = [ name = "pallet-staking" version = "4.0.0-dev" dependencies = [ - "chrono", "frame-benchmarking", "frame-election-provider-support", "frame-support", "frame-system", - "git2", "log 0.4.14", - "num-format", "pallet-authorship", "pallet-bags-list", "pallet-balances", @@ -6142,7 +6149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" dependencies = [ "instant", - "lock_api 0.4.2", + "lock_api 0.4.4", "parking_lot_core 0.8.3", ] @@ -6184,7 +6191,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.5", + "redox_syscall 0.2.10", "smallvec 1.6.1", "winapi 0.3.9", ] @@ -6352,9 +6359,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" [[package]] name = "pin-utils" @@ -7074,9 +7081,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ "bitflags", ] @@ -7088,7 +7095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ "getrandom 0.2.3", - "redox_syscall 0.2.5", + "redox_syscall 0.2.10", ] [[package]] @@ -8644,9 +8651,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.126" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" +checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" dependencies = [ "serde_derive", ] @@ -8672,9 +8679,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.126" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" +checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc" dependencies = [ "proc-macro2", "quote", @@ -8683,9 +8690,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" dependencies = [ "itoa", "ryu", @@ -8787,9 +8794,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] @@ -8814,9 +8821,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" [[package]] name = "slog" @@ -8879,9 +8886,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" dependencies = [ "libc", "winapi 0.3.9", @@ -10066,9 +10073,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.69" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" dependencies = [ "proc-macro2", "quote", @@ -10125,7 +10132,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.4", - "redox_syscall 0.2.5", + "redox_syscall 0.2.10", "remove_dir_all", "winapi 0.3.9", ] @@ -10374,7 +10381,7 @@ dependencies = [ "num_cpus", "once_cell", "parking_lot 0.11.1", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", "signal-hook-registry", "tokio-macros 1.3.0", "winapi 0.3.9", @@ -10514,7 +10521,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" dependencies = [ "futures-core", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", "tokio 1.10.0", ] @@ -10640,7 +10647,7 @@ dependencies = [ "futures-core", "futures-sink", "log 0.4.14", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", "tokio 1.10.0", ] @@ -10661,12 +10668,12 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.7", "tracing-attributes", "tracing-core", ] @@ -10684,9 +10691,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" dependencies = [ "lazy_static", ] @@ -10978,9 +10985,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "universal-hash" @@ -11053,11 +11060,12 @@ dependencies = [ [[package]] name = "value-bag" -version = "1.0.0-alpha.6" +version = "1.0.0-alpha.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" +checksum = "dd320e1520f94261153e96f7534476ad869c14022aee1e59af7c778075d840ae" dependencies = [ "ctor", + "version_check 0.9.2", ] [[package]] @@ -11675,9 +11683,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.47" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c40dc691fc48003eba817c38da7113c15698142da971298003cac3ef175680b3" +checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index eb62ff712e465..d2232257d879e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,7 +129,6 @@ members = [ "frame/utility", "frame/vesting", "frame/bags-list", - "frame/bags-list/generate-bags", "primitives/api", "primitives/api/proc-macro", "primitives/api/test", @@ -201,6 +200,8 @@ members = [ "utils/frame/try-runtime/cli", "utils/frame/rpc/support", "utils/frame/rpc/system", + "utils/frame/generate-bags", + "utils/frame/generate-bags/node-runtime", "utils/prometheus", "utils/wasm-builder", ] diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index a4b8f6696ea6a..5f5cbae12749c 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -544,7 +544,7 @@ impl StateBackend> for BenchmarkingState { if tracker.writes > 0 { writes += 1; - repeat_writes += tracker.reads - 1; + repeat_writes += tracker.writes - 1; } } }); diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 7f22bd03f7b46..75423a78b6654 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -35,12 +35,6 @@ sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = tr sp-io = { version = "4.0.0-dev", path = "../../primitives/io", optional = true, default-features = false } sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true, default-features = false } -# optional imports for making voter bags lists -chrono = { version = "0.4.19", optional = true } -git2 = { version = "0.13.20", default-features = false, optional = true } -num-format = { version = "0.4.0", optional = true } -pallet-staking = { version = "4.0.0-dev", path = "../staking", optional = true } - [dev-dependencies] sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} sp-io = { version = "4.0.0-dev", path = "../../primitives/io"} @@ -66,11 +60,6 @@ runtime-benchmarks = [ "sp-io", "pallet-balances", "sp-tracing", + "frame-election-provider-support/runtime-benchmarks", ] -generate-bags = [ - "chrono", - "git2", - "num-format", - "std", - "pallet-staking" -] + diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index f8b09fb512762..677bd0ee8c956 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -56,8 +56,6 @@ use sp_std::prelude::*; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; -#[cfg(feature = "generate-bags")] -pub mod generate_bags; mod list; #[cfg(test)] mod mock; @@ -267,4 +265,26 @@ impl SortedListProvider for Pallet { fn clear() { List::::clear() } + + #[cfg(feature = "runtime-benchmarks")] + fn weight_update_worst_case(who: &T::AccountId, is_increase: bool) -> VoteWeight { + use frame_support::traits::Get as _; + let thresholds = T::BagThresholds::get(); + let node = list::Node::::get(who).unwrap(); + let current_bag_idx = thresholds + .iter() + .chain(sp_std::iter::once(&VoteWeight::MAX)) + .position(|w| w == &node.bag_upper) + .unwrap(); + + if is_increase { + let next_threshold_idx = current_bag_idx + 1; + assert!(thresholds.len() > next_threshold_idx); + thresholds[next_threshold_idx] + } else { + assert!(current_bag_idx != 0); + let prev_threshold_idx = current_bag_idx - 1; + thresholds[prev_threshold_idx] + } + } } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index c2e42cc48ec3e..11c0f413c593e 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -52,7 +52,7 @@ mod tests; /// /// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this /// function behaves as if it does. -fn notional_bag_for(weight: VoteWeight) -> VoteWeight { +pub(crate) fn notional_bag_for(weight: VoteWeight) -> VoteWeight { let thresholds = T::BagThresholds::get(); let idx = thresholds.partition_point(|&threshold| weight > threshold); thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) @@ -586,6 +586,11 @@ impl Bag { Ok(()) } + + #[cfg(feature = "runtime-benchmarks")] + pub(crate) fn contains(&self, id: &T::AccountId) -> bool { + self.iter().any(|n| n.id() == id) + } } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. @@ -596,7 +601,7 @@ pub struct Node { id: T::AccountId, prev: Option, next: Option, - bag_upper: VoteWeight, + pub(crate) bag_upper: VoteWeight, } impl Node { diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 28d3cb031a59f..49efcca8bf416 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -136,8 +136,8 @@ fn migrate_works() { (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 (20, vec![711]), (1000, vec![2, 3, 4]), - (10_000, vec![712]), /* nodes in range 1_001 ..= 2_000 move from bag 2_000 - * to bag 10_000 */ + // nodes in range 1_001 ..= 2_000 move from bag 2_000 to bag 10_000 + (10_000, vec![712]), ] ); }); diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index c461ae56b9c2b..9eae6fc636a88 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -333,6 +333,13 @@ pub trait SortedListProvider { fn clear(); /// Sanity check internal state of list. Only meant for debug compilation. fn sanity_check() -> Result<(), &'static str>; + + /// If `who` changes by the returned amount they are guaranteed to have a worst case change + /// in their list position. + #[cfg(feature = "runtime-benchmarks")] + fn weight_update_worst_case(_who: &AccountId, _is_increase: bool) -> VoteWeight { + VoteWeight::MAX + } } /// Something that can provide the `VoteWeight` of an account. Similar to [`ElectionProvider`] and diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index 8b84145c1acfd..349043bdbf991 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -126,7 +126,7 @@ fn check_membership_proof_setup( pallet_staking::ValidatorCount::::put(n); // create validators and set random session keys - for (n, who) in create_validators::(n, 1000).unwrap().into_iter().enumerate() { + for (n, who) in create_validators::(n, 1000, 0).unwrap().into_iter().enumerate() { use rand::{RngCore, SeedableRng}; let validator = T::Lookup::lookup(who).unwrap(); diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 684cd6800a279..18fcd46ecc5f2 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -40,11 +40,6 @@ log = { version = "0.4.14", default-features = false } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } rand_chacha = { version = "0.2", default-features = false, optional = true } -# Optional imports for making voter bags lists -chrono = { version = "0.4.19", optional = true } -git2 = { version = "0.13.20", default-features = false, optional = true } -num-format = { version = "0.4.0", optional = true } - [dev-dependencies] sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } @@ -52,7 +47,7 @@ sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elect pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } -pallet-bags-list = { version = "4.0.0-dev", path = "../bags-list" } +pallet-bags-list = { version = "4.0.0-dev", features = ["runtime-benchmarks"], path = "../bags-list" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } @@ -81,14 +76,3 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] -generate-bags = [ - "chrono", - "git2", - "num-format", - "pallet-staking-reward-curve", - "pallet-balances", - "pallet-timestamp", - "sp-core", - "sp-tracing", - "std", -] diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index d8eef256fd782..233c628f2f925 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -21,9 +21,10 @@ use super::*; use crate::Pallet as Staking; use testing_utils::*; +use frame_election_provider_support::{SortedListProvider, VoteWeight}; use frame_support::{ pallet_prelude::*, - traits::{Currency, Get, Imbalance}, + traits::{Currency, CurrencyToVote, Get, Imbalance}, }; use sp_runtime::{ traits::{StaticLookup, Zero}, @@ -131,6 +132,95 @@ pub fn create_validator_with_nominators( Ok((v_stash, nominators)) } +struct ListScenario { + dest_stash1: T::AccountId, + /// Stash that is expected to be moved. + origin_stash1: T::AccountId, + /// Controller of the Stash that is expected to be moved. + origin_controller1: T::AccountId, + origin_stash2: T::AccountId, + origin_weight_as_vote: VoteWeight, + dest_weight_as_vote: VoteWeight, + dest_weight: BalanceOf, +} + +impl ListScenario { + /// An expensive scenario for bags-list implementation: + /// + /// - the node to be updated (r) is the head of a bag that has at least one other node. The bag + /// itself will need to be read and written to update its head. The node pointed to by r.next + /// will need to be read and written as it will need to have its prev pointer updated. Note + /// that there are two other worst case scenarios for bag removal: 1) the node is a tail and + /// 2) the node is a middle node with prev and next; all scenarios end up with the same number + /// of storage reads and writes. + /// + /// - the destination bag has at least one node, which will need its next pointer updated. + /// + /// NOTE: while this scenario specifically targets a worst case for the bags-list, it should + /// also elicit a worst case for other known `SortedListProvider` implementations; although + /// this may not be true against unknown `SortedListProvider` implementations. + fn new(origin_weight: BalanceOf, is_increase: bool) -> Result { + ensure!(!origin_weight.is_zero(), "origin weight must be greater than 0"); + + // burn the entire issuance. + let i = T::Currency::burn(T::Currency::total_issuance()); + sp_std::mem::forget(i); + + // create accounts with the origin weight + + let (origin_stash1, origin_controller1) = create_stash_controller_with_balance::( + USER_SEED + 2, + origin_weight, + Default::default(), + )?; + Staking::::nominate( + RawOrigin::Signed(origin_controller1.clone()).into(), + // NOTE: these don't really need to be validators. + vec![T::Lookup::unlookup(account("random_validator", 0, SEED))], + )?; + + let (origin_stash2, origin_controller2) = create_stash_controller_with_balance::( + USER_SEED + 3, + origin_weight, + Default::default(), + )?; + Staking::::nominate( + RawOrigin::Signed(origin_controller2.clone()).into(), + vec![T::Lookup::unlookup(account("random_validator", 0, SEED))].clone(), + )?; + + // find a destination weight that will trigger the worst case scenario + let dest_weight_as_vote = + T::SortedListProvider::weight_update_worst_case(&origin_stash1, is_increase); + + let total_issuance = T::Currency::total_issuance(); + + let dest_weight = + T::CurrencyToVote::to_currency(dest_weight_as_vote as u128, total_issuance); + + // create an account with the worst case destination weight + let (dest_stash1, dest_controller1) = create_stash_controller_with_balance::( + USER_SEED + 1, + dest_weight, + Default::default(), + )?; + Staking::::nominate( + RawOrigin::Signed(dest_controller1).into(), + vec![T::Lookup::unlookup(account("random_validator", 0, SEED))], + )?; + + Ok(ListScenario { + dest_stash1, + origin_stash1, + origin_controller1, + origin_stash2, + origin_weight_as_vote: T::CurrencyToVote::to_vote(origin_weight, total_issuance), + dest_weight_as_vote, + dest_weight, + }) + } +} + const USER_SEED: u32 = 999666; benchmarks! { @@ -148,10 +238,24 @@ benchmarks! { } bond_extra { - let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - let max_additional = T::Currency::minimum_balance() * 10u32.into(); - let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; - let original_bonded: BalanceOf = ledger.active; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup the worst case list scenario. + + // the weight the nominator will start at. + let scenario = ListScenario::::new(origin_weight, true)?; + + let max_additional = scenario.dest_weight.clone() - origin_weight; + + let stash = scenario.origin_stash1.clone(); + let controller = scenario.origin_controller1.clone(); + let original_bonded: BalanceOf = Ledger::::get(&controller).map(|l| l.active).ok_or("ledger not created after")?; + + T::Currency::deposit_into_existing(&stash, max_additional).unwrap(); + whitelist_account!(stash); }: _(RawOrigin::Signed(stash), max_additional) verify { @@ -161,10 +265,25 @@ benchmarks! { } unbond { - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - let amount = T::Currency::minimum_balance() * 10u32.into(); + use sp_std::convert::TryFrom; + // clean up any existing state. + clear_validators_and_nominators::(); + + // setup the worst case list scenario. + let total_issuance = T::Currency::total_issuance(); + // the weight the nominator will start at. The value used here is expected to be + // significantly higher than the first position in a list (e.g. the first bag threshold). + let origin_weight = BalanceOf::::try_from(952_994_955_240_703u128) + .map_err(|_| "balance expected to be a u128") + .unwrap(); + let scenario = ListScenario::::new(origin_weight, false)?; + + let stash = scenario.origin_stash1.clone(); + let controller = scenario.origin_controller1.clone(); + let amount = origin_weight - scenario.dest_weight.clone(); let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; let original_bonded: BalanceOf = ledger.active; + whitelist_account!(controller); }: _(RawOrigin::Signed(controller.clone()), amount) verify { @@ -196,26 +315,50 @@ benchmarks! { withdraw_unbonded_kill { // Slashing Spans let s in 0 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0, 100, Default::default())?; - add_slashing_spans::(&stash, s); - let amount = T::Currency::minimum_balance() * 10u32.into(); - Staking::::unbond(RawOrigin::Signed(controller.clone()).into(), amount)?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + + let ed = T::Currency::minimum_balance(); + let mut ledger = Ledger::::get(&controller).unwrap(); + ledger.active = ed - One::one(); + Ledger::::insert(&controller, ledger); CurrentEra::::put(EraIndex::max_value()); - let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; - let original_total: BalanceOf = ledger.total; + whitelist_account!(controller); }: withdraw_unbonded(RawOrigin::Signed(controller.clone()), s) verify { assert!(!Ledger::::contains_key(controller)); + assert!(!T::SortedListProvider::contains(&stash)); } validate { - let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case scenario where the user calling validate was formerly a nominator so + // they must be removed from the list. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + let prefs = ValidatorPrefs::default(); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), prefs) verify { - assert!(Validators::::contains_key(stash)); + assert!(Validators::::contains_key(&stash)); + assert!(!T::SortedListProvider::contains(&stash)); } kick { @@ -227,7 +370,7 @@ benchmarks! { // these are the other validators; there are `T::MAX_NOMINATIONS - 1` of them, so // there are a total of `T::MAX_NOMINATIONS` validators in the system. - let rest_of_validators = create_validators::(T::MAX_NOMINATIONS - 1, 100)?; + let rest_of_validators = create_validators::(T::MAX_NOMINATIONS - 1, 100, 0)?; // this is the validator that will be kicking. let (stash, controller) = create_stash_controller::( @@ -284,18 +427,49 @@ benchmarks! { // Worst case scenario, T::MAX_NOMINATIONS nominate { let n in 1 .. T::MAX_NOMINATIONS; - let (stash, controller) = create_stash_controller::(n + 1, 100, Default::default())?; - let validators = create_validators::(n, 100)?; + + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. + let scenario = ListScenario::::new(origin_weight, true)?; + let (stash, controller) = create_stash_controller_with_balance::( + SEED + T::MAX_NOMINATIONS + 1, // make sure the account does not conflict with others + origin_weight, + Default::default(), + ).unwrap(); + + assert!(!Nominators::::contains_key(&stash)); + assert!(!T::SortedListProvider::contains(&stash)); + + let validators = create_validators::(n, 100, 0).unwrap(); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), validators) verify { - assert!(Nominators::::contains_key(stash)); + assert!(Nominators::::contains_key(&stash)); + assert!(T::SortedListProvider::contains(&stash)) } chill { - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + whitelist_account!(controller); }: _(RawOrigin::Signed(controller)) + verify { + assert!(!T::SortedListProvider::contains(&stash)); + } set_payee { let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; @@ -347,11 +521,23 @@ benchmarks! { force_unstake { // Slashing Spans let s in 0 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0, 100, Default::default())?; + // Clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); add_slashing_spans::(&stash, s); - }: _(RawOrigin::Root, stash, s) + + }: _(RawOrigin::Root, stash.clone(), s) verify { assert!(!Ledger::::contains_key(&controller)); + assert!(!T::SortedListProvider::contains(&stash)); } cancel_deferred_slash { @@ -440,19 +626,44 @@ benchmarks! { rebond { let l in 1 .. MAX_UNLOCKING_CHUNKS as u32; - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); + + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get() + .max(T::Currency::minimum_balance()) + // we use 100 to play friendly with the bags, list threshold values in the mock. + .max(100u32.into()); + + // setup a worst case list scenario. + let scenario = ListScenario::::new(origin_weight, true)?; + let dest_weight = scenario.dest_weight.clone(); + + // rebond an amount that will give the use dest_weight + let rebond_amount = dest_weight - origin_weight; + + // spread that amount to rebond across `l` unlocking chunks, + let value = rebond_amount / l.into(); + // so the sum of unlocking chunks puts voter into the dest bag. + assert!(value * l.into() + origin_weight > origin_weight); + assert!(value * l.into() + origin_weight <= dest_weight); let unlock_chunk = UnlockChunk::> { - value: 1u32.into(), + value, era: EraIndex::zero(), }; + + let stash = scenario.origin_stash1.clone(); + let controller = scenario.origin_controller1.clone(); + let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); + for _ in 0 .. l { staking_ledger.unlocking.push(unlock_chunk.clone()) } Ledger::::insert(controller.clone(), staking_ledger.clone()); let original_bonded: BalanceOf = staking_ledger.active; + whitelist_account!(controller); - }: _(RawOrigin::Signed(controller.clone()), (l + 100).into()) + }: _(RawOrigin::Signed(controller.clone()), rebond_amount) verify { let ledger = Ledger::::get(&controller).ok_or("ledger not created after")?; let new_bonded: BalanceOf = ledger.active; @@ -479,19 +690,28 @@ benchmarks! { reap_stash { let s in 1 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0, 100, Default::default())?; - Staking::::validate(RawOrigin::Signed(controller.clone()).into(), ValidatorPrefs::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + add_slashing_spans::(&stash, s); T::Currency::make_free_balance_be(&stash, T::Currency::minimum_balance()); - whitelist_account!(controller); assert!(Bonded::::contains_key(&stash)); - assert!(Validators::::contains_key(&stash)); + assert!(T::SortedListProvider::contains(&stash)); + whitelist_account!(controller); }: _(RawOrigin::Signed(controller), stash.clone(), s) verify { assert!(!Bonded::::contains_key(&stash)); - assert!(!Validators::::contains_key(&stash)); + assert!(!T::SortedListProvider::contains(&stash)); } new_era { @@ -637,8 +857,18 @@ benchmarks! { } chill_other { - let (_, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; - Staking::::validate(RawOrigin::Signed(controller.clone()).into(), ValidatorPrefs::default())?; + // clean up any existing state. + clear_validators_and_nominators::(); + + let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); + + // setup a worst case list scenario. Note that we don't care about the setup of the + // destination position because we are doing a removal from the list but no insert. + let scenario = ListScenario::::new(origin_weight, true)?; + let controller = scenario.origin_controller1.clone(); + let stash = scenario.origin_stash1.clone(); + assert!(T::SortedListProvider::contains(&stash)); + Staking::::set_staking_limits( RawOrigin::Root.into(), BalanceOf::::max_value(), @@ -647,10 +877,11 @@ benchmarks! { Some(0), Some(Percent::from_percent(0)) )?; + let caller = whitelisted_caller(); }: _(RawOrigin::Signed(caller), controller.clone()) verify { - assert!(!Validators::::contains_key(controller)); + assert!(!T::SortedListProvider::contains(&stash)); } } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index b6df94adb3b73..bd971e15ae45b 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1273,7 +1273,6 @@ impl SortedListProvider for UseNominatorsMap { fn sanity_check() -> Result<(), &'static str> { Ok(()) } - fn clear() { Nominators::::remove_all(None); CounterForNominators::::kill(); diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index fe1f63982687e..695a57c06a58f 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -768,13 +768,14 @@ pub mod pallet { Error::::InsufficientBond ); + // ledger must be updated prior to calling `Self::weight_of`. + Self::update_ledger(&controller, &ledger); // update this staker in the sorted list, if they exist in it. if T::SortedListProvider::contains(&stash) { T::SortedListProvider::on_update(&stash, Self::weight_of(&ledger.stash)); } Self::deposit_event(Event::::Bonded(stash.clone(), extra)); - Self::update_ledger(&controller, &ledger); } Ok(()) } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 10348761268ae..2e57931c6f3cf 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -53,9 +53,18 @@ pub fn create_funded_user( ) -> T::AccountId { let user = account(string, n, SEED); let balance = T::Currency::minimum_balance() * balance_factor.into(); - T::Currency::make_free_balance_be(&user, balance); - // ensure T::CurrencyToVote will work correctly. - T::Currency::issue(balance); + let _ = T::Currency::make_free_balance_be(&user, balance); + user +} + +/// Grab a funded user with max Balance. +pub fn create_funded_user_with_balance( + string: &'static str, + n: u32, + balance: BalanceOf, +) -> T::AccountId { + let user = account(string, n, SEED); + let _ = T::Currency::make_free_balance_be(&user, balance); user } @@ -79,6 +88,26 @@ pub fn create_stash_controller( return Ok((stash, controller)) } +/// Create a stash and controller pair with fixed balance. +pub fn create_stash_controller_with_balance( + n: u32, + balance: crate::BalanceOf, + destination: RewardDestination, +) -> Result<(T::AccountId, T::AccountId), &'static str> { + let stash = create_funded_user_with_balance::("stash", n, balance); + let controller = create_funded_user_with_balance::("controller", n, balance); + let controller_lookup: ::Source = + T::Lookup::unlookup(controller.clone()); + + Staking::::bond( + RawOrigin::Signed(stash.clone()).into(), + controller_lookup, + balance, + destination, + )?; + Ok((stash, controller)) +} + /// Create a stash and controller pair, where the controller is dead, and payouts go to controller. /// This is used to test worst case payout scenarios. pub fn create_stash_and_dead_controller( @@ -105,11 +134,12 @@ pub fn create_stash_and_dead_controller( pub fn create_validators( max: u32, balance_factor: u32, + seed: u32, ) -> Result::Source>, &'static str> { let mut validators: Vec<::Source> = Vec::with_capacity(max as usize); for i in 0..max { let (stash, controller) = - create_stash_controller::(i, balance_factor, RewardDestination::Staked)?; + create_stash_controller::(i + seed, balance_factor, RewardDestination::Staked)?; let validator_prefs = ValidatorPrefs { commission: Perbill::from_percent(50), ..Default::default() }; Staking::::validate(RawOrigin::Signed(controller).into(), validator_prefs)?; diff --git a/utils/frame/generate-bags/Cargo.toml b/utils/frame/generate-bags/Cargo.toml new file mode 100644 index 0000000000000..ecbc09b05f7f7 --- /dev/null +++ b/utils/frame/generate-bags/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "generate-bags" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Bag threshold generation script for pallet-bag-list" +readme = "README.md" + +[dependencies] +node-runtime = { version = "3.0.0-dev", path = "../../../bin/node/runtime" } + +# FRAME +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/support" } +frame-election-provider-support = { version = "4.0.0-dev", path = "../../../frame/election-provider-support", features = ["runtime-benchmarks"] } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/staking" } + +# primitives +sp-io = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/io" } + +# third party +chrono = { version = "0.4.19" } +git2 = { version = "0.13.20", default-features = false } +num-format = { version = "0.4.0" } +structopt = "0.3.21" diff --git a/frame/bags-list/generate-bags/Cargo.toml b/utils/frame/generate-bags/node-runtime/Cargo.toml similarity index 57% rename from frame/bags-list/generate-bags/Cargo.toml rename to utils/frame/generate-bags/node-runtime/Cargo.toml index 7d86865f15adb..5b517e4162b2a 100644 --- a/frame/bags-list/generate-bags/Cargo.toml +++ b/utils/frame/generate-bags/node-runtime/Cargo.toml @@ -10,15 +10,14 @@ description = "Bag threshold generation script for pallet-bag-list and node-runt readme = "README.md" [dependencies] -node-runtime = { version = "3.0.0-dev", path = "../../../bin/node/runtime" } +node-runtime = { version = "3.0.0-dev", path = "../../../../bin/node/runtime" } +generate-bags = { version = "3.0.0", path = "../" } # FRAME -frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } -pallet-bags-list = { version = "4.0.0-dev", path = "../../bags-list", features = ["generate-bags"] } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../staking" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/support" } # primitives -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } +sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } # third-party structopt = "0.3.21" diff --git a/frame/bags-list/generate-bags/src/main.rs b/utils/frame/generate-bags/node-runtime/src/main.rs similarity index 82% rename from frame/bags-list/generate-bags/src/main.rs rename to utils/frame/generate-bags/node-runtime/src/main.rs index 7f0bb5c7be74b..f99acdbfcea15 100644 --- a/frame/bags-list/generate-bags/src/main.rs +++ b/utils/frame/generate-bags/node-runtime/src/main.rs @@ -15,11 +15,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Make the set of bag thresholds to be used in pallet-bags-list. +//! Make the set of bag thresholds to be used with pallet-bags-list. -use pallet_bags_list::generate_bags::generate_thresholds_module; +use generate_bags::generate_thresholds; use std::path::PathBuf; use structopt::StructOpt; + #[derive(Debug, StructOpt)] struct Opt { /// How many bags to generate. @@ -34,5 +35,5 @@ fn main() -> Result<(), std::io::Error> { let Opt { n_bags, output } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) + ext.execute_with(|| generate_thresholds::(n_bags, &output)) } diff --git a/frame/bags-list/src/generate_bags.rs b/utils/frame/generate-bags/src/lib.rs similarity index 97% rename from frame/bags-list/src/generate_bags.rs rename to utils/frame/generate-bags/src/lib.rs index 3f47798a87e2b..dc79812682373 100644 --- a/frame/bags-list/src/generate_bags.rs +++ b/utils/frame/generate-bags/src/lib.rs @@ -38,7 +38,7 @@ //! //! 2. Write a little program to generate the definitions. This can be a near-identical copy of //! `substrate/frame/bags-list/generate-bags`. This program exists only to hook together the -//! runtime definitions with the various calculations here. +//! runtime definitions with the various calculations here. //! //! 3. Run that program: //! @@ -66,7 +66,6 @@ use std::{ /// Note that this value depends on the current issuance, a quantity known to change over time. /// This makes the project of computing a static value suitable for inclusion in a static, /// generated file _excitingly unstable_. -#[cfg(any(feature = "std", feature = "generate-bags"))] fn existential_weight() -> VoteWeight { use frame_support::traits::{Currency, CurrencyToVote}; @@ -160,7 +159,7 @@ pub fn thresholds( /// - Module documentation noting that this is autogenerated and when. /// - Some associated constants. /// - The constant array of thresholds. -pub fn generate_thresholds_module( +pub fn generate_thresholds( n_bags: usize, output: &Path, ) -> Result<(), std::io::Error> { From 65682bbce35c9bc4c298fafa5c14d021975df62d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 31 Aug 2021 22:49:40 +0200 Subject: [PATCH 162/241] Build works --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2aa8d1440056e..0660d0cc2ab6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2674,7 +2674,7 @@ dependencies = [ "itoa", "pin-project-lite 0.2.7", "socket2 0.4.1", - "tokio 1.10.0", + "tokio", "tower-service", "tracing", "want", @@ -10377,7 +10377,7 @@ checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" dependencies = [ "futures-core", "pin-project-lite 0.2.7", - "tokio 1.10.0", + "tokio", ] [[package]] @@ -10427,7 +10427,7 @@ dependencies = [ "futures-sink", "log 0.4.14", "pin-project-lite 0.2.7", - "tokio 1.10.0", + "tokio", ] [[package]] From 6d699567b894730a8c297f2fd5c198d9aadee88a Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Tue, 31 Aug 2021 23:01:39 +0200 Subject: [PATCH 163/241] Apply suggestions from code review Co-authored-by: Guillaume Thiolliere --- frame/bags-list/src/list/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 11c0f413c593e..05c944bca0d74 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -81,7 +81,7 @@ impl List { /// Regenerate all of the data from the given ids. /// - /// This is expensive and should only ever be performed during a migration a migration, or when + /// This is expensive and should only ever be performed during a migration, or when /// the data needs to be generated from scratch again. /// /// This may or may not need to be called at genesis as well, based on the configuration of the @@ -126,7 +126,7 @@ impl List { let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); let new_set: BTreeSet<_> = T::BagThresholds::get().iter().copied().collect(); - // accounts that need to be rebaged + // accounts that need to be rebagged let mut affected_accounts = BTreeSet::new(); // track affected old bags to make sure we only iterate them once let mut affected_old_bags = BTreeSet::new(); @@ -165,7 +165,7 @@ impl List { // migrate the voters whose bag has changed let weight_of = T::VoteWeightProvider::vote_weight; - Self::remove_many(affected_accounts.iter().map(|id| id)); + Self::remove_many(&affected_accounts); let num_affected = affected_accounts.len() as u32; let _ = Self::insert_many(affected_accounts.into_iter(), weight_of); From fa4844fe6ca83cd7f3608f65edf8cdac4f0b8a1e Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Tue, 31 Aug 2021 23:04:48 +0200 Subject: [PATCH 164/241] Apply suggestions from code review Co-authored-by: Guillaume Thiolliere --- frame/election-provider-support/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 9eae6fc636a88..e5dd1db105b8b 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -316,7 +316,7 @@ pub trait SortedListProvider { fn count() -> u32; /// Return true if the list already contains `id`. fn contains(id: &AccountId) -> bool; - // Hook for inserting a new id. + /// Hook for inserting a new id. fn on_insert(id: AccountId, weight: VoteWeight) -> Result<(), Self::Error>; /// Hook for updating a single id. fn on_update(id: &AccountId, weight: VoteWeight); From 9166eed61b18deb9a16ee231a983e3249fe22d44 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 1 Sep 2021 09:28:43 +0200 Subject: [PATCH 165/241] Remove commented out debug assert --- frame/bags-list/src/list/mod.rs | 4 ---- frame/staking/src/migrations.rs | 1 - 2 files changed, 5 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 05c944bca0d74..e80572dc09b8d 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -425,10 +425,6 @@ impl Bag { /// Get a bag by its upper vote weight. pub(crate) fn get(bag_upper: VoteWeight) -> Option> { - // debug_assert!( TODO this breaks when we attempt to migrate because thresholds change - // T::BagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, - // "it is a logic error to attempt to get a bag which is not in the thresholds list" - // ); crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { bag.bag_upper = bag_upper; bag diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 0c81279c3bc0a..89661a75a85b2 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -42,7 +42,6 @@ use super::*; // ); // T::WeightInfo::regenerate( -// CounterForValidators::::get(), // CounterForNominators::::get(), // ) // .saturating_add(T::DbWeight::get().reads(2)) From e7052d9844e170b5ae5b7622d6195a41b65efd5d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 1 Sep 2021 10:27:24 +0200 Subject: [PATCH 166/241] Remove some unused deps and some unused benchmarking stuff --- frame/bags-list/src/list/mod.rs | 8 ++++---- frame/election-provider-support/src/lib.rs | 19 ++++++++++++++----- frame/staking/Cargo.toml | 7 +------ frame/staking/src/benchmarking.rs | 20 ++++---------------- 4 files changed, 23 insertions(+), 31 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index e80572dc09b8d..6a2abeb16ce60 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -20,9 +20,8 @@ //! data structures, where multiple storage items are masked by one outer API. See [`ListNodes`], //! [`CounterForListNodes`] and [`ListBags`] for more information. //! -//! //! The outer API of this module is the [`List`] struct. It wraps all acceptable operations on top -//! of the aggregate linked list. All operations with the linked list should happen through this +//! of the aggregate linked list. All operations with the bags list should happen through this //! interface. use crate::Config; @@ -490,7 +489,8 @@ impl Bag { fn insert_node_unchecked(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { - // this should never happen, but this check prevents a worst case infinite loop. + // this should never happen, but this check prevents one path to a worst case + // infinite loop. debug_assert!(false, "system logic error: inserting a node who has the id of tail"); crate::log!(warn, "system logic error: inserting a node who has the id of tail"); return @@ -546,7 +546,7 @@ impl Bag { /// Sanity check this bag. /// - /// Should be called by the call-site, after each mutating operation on a bag. The call site of + /// Should be called by the call-site, after any mutating operation on a bag. The call site of /// this struct is always `List`. /// /// * Ensures head has no prev. diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index e5dd1db105b8b..61027c4672e1c 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -297,40 +297,49 @@ impl ElectionProvider for () { /// A utility trait for something to implement `ElectionDataProvider` in a sensible way. /// -/// This is generic over the `AccountId`'d sole, and it can represent a validator, a nominator, or -/// any other entity. +/// This is generic over `AccountId` and it can represent a validator, a nominator, or any other +/// entity. /// /// On the contrary, to simplify the trait, the `VoteWeight` is hardcoded as the weight of each -/// entity. The weights are scending, the higher, the better. In the long term, if this trait ends +/// entity. The weights are ascending, the higher, the better. In the long term, if this trait ends /// up having use cases outside of the election context, it is easy enough to make it generic over /// the `VoteWeight`. /// /// Something that implements this trait will do a best-effort sort over ids, and thus can be /// used on the implementing side of [`ElectionDataProvider`]. pub trait SortedListProvider { + /// The list's error type. type Error; /// Returns iterator over the list, which can have `take` called on it. fn iter() -> Box>; + /// get the current count of ids in the list. fn count() -> u32; + /// Return true if the list already contains `id`. fn contains(id: &AccountId) -> bool; + /// Hook for inserting a new id. fn on_insert(id: AccountId, weight: VoteWeight) -> Result<(), Self::Error>; + /// Hook for updating a single id. fn on_update(id: &AccountId, weight: VoteWeight); + /// Hook for removing am id from the list. fn on_remove(id: &AccountId); + /// Regenerate this list from scratch. Returns the count of items inserted. /// - /// This should typically only be used to enable this flag at a runtime upgrade. + /// This should typically only be used at a runtime upgrade. fn regenerate( all: impl IntoIterator, weight_of: Box VoteWeight>, ) -> u32; - /// Remove everything. + + /// Remove the list and all its associated storage items. fn clear(); + /// Sanity check internal state of list. Only meant for debug compilation. fn sanity_check() -> Result<(), &'static str>; diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 18fcd46ecc5f2..b389c34258a6d 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -29,11 +29,6 @@ pallet-session = { version = "4.0.0-dev", default-features = false, features = [ pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../authorship" } sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = true } -pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp", optional = true } -pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve", optional = true } -pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true } log = { version = "0.4.14", default-features = false } # Optional imports for benchmarking @@ -46,7 +41,7 @@ sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } -pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } +pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } pallet-bags-list = { version = "4.0.0-dev", features = ["runtime-benchmarks"], path = "../bags-list" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 233c628f2f925..27955d78d2119 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -21,7 +21,7 @@ use super::*; use crate::Pallet as Staking; use testing_utils::*; -use frame_election_provider_support::{SortedListProvider, VoteWeight}; +use frame_election_provider_support::SortedListProvider; use frame_support::{ pallet_prelude::*, traits::{Currency, CurrencyToVote, Get, Imbalance}, @@ -133,14 +133,10 @@ pub fn create_validator_with_nominators( } struct ListScenario { - dest_stash1: T::AccountId, /// Stash that is expected to be moved. origin_stash1: T::AccountId, /// Controller of the Stash that is expected to be moved. origin_controller1: T::AccountId, - origin_stash2: T::AccountId, - origin_weight_as_vote: VoteWeight, - dest_weight_as_vote: VoteWeight, dest_weight: BalanceOf, } @@ -179,7 +175,7 @@ impl ListScenario { vec![T::Lookup::unlookup(account("random_validator", 0, SEED))], )?; - let (origin_stash2, origin_controller2) = create_stash_controller_with_balance::( + let (_origin_stash2, origin_controller2) = create_stash_controller_with_balance::( USER_SEED + 3, origin_weight, Default::default(), @@ -199,7 +195,7 @@ impl ListScenario { T::CurrencyToVote::to_currency(dest_weight_as_vote as u128, total_issuance); // create an account with the worst case destination weight - let (dest_stash1, dest_controller1) = create_stash_controller_with_balance::( + let (_dest_stash1, dest_controller1) = create_stash_controller_with_balance::( USER_SEED + 1, dest_weight, Default::default(), @@ -209,15 +205,7 @@ impl ListScenario { vec![T::Lookup::unlookup(account("random_validator", 0, SEED))], )?; - Ok(ListScenario { - dest_stash1, - origin_stash1, - origin_controller1, - origin_stash2, - origin_weight_as_vote: T::CurrencyToVote::to_vote(origin_weight, total_issuance), - dest_weight_as_vote, - dest_weight, - }) + Ok(ListScenario { origin_stash1, origin_controller1, dest_weight }) } } From c9263bc027bbb9aaecdb27f866ebb2f3d74881e6 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 1 Sep 2021 10:42:45 +0200 Subject: [PATCH 167/241] Fix stakings ElectionDataProvider clear --- frame/bags-list/src/list/mod.rs | 5 ----- frame/staking/src/lib.rs | 2 +- frame/staking/src/pallet/impls.rs | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 6a2abeb16ce60..1ae2a69762556 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -582,11 +582,6 @@ impl Bag { Ok(()) } - - #[cfg(feature = "runtime-benchmarks")] - pub(crate) fn contains(&self, id: &T::AccountId) -> bool { - self.iter().any(|n| n.id() == id) - } } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index a11069790d653..3ff2e91b7dfb9 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -103,7 +103,7 @@ //! #### Voting //! //! Staking is closely related to elections; actual validators are chosen from among all potential -//! validators by election by the potential validators and nominators. To reduce use of the phrase +//! validators via election by the potential validators and nominators. To reduce use of the phrase //! "potential validators and nominators", we often use the term **voters**, who are simply //! the union of potential validators and nominators. //! diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index bd971e15ae45b..bdd6622b1d61b 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -938,6 +938,9 @@ impl ElectionDataProvider> for Pallet >::remove_all(None); >::remove_all(None); >::remove_all(None); + >::kill(); + >::kill(); + T::SortedListProvider::clear(); } #[cfg(feature = "runtime-benchmarks")] From ecddb8293780d57ca6d6988805ac2019b3f7f11d Mon Sep 17 00:00:00 2001 From: Parity Benchmarking Bot Date: Wed, 1 Sep 2021 13:55:25 +0000 Subject: [PATCH 168/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bags_list --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bags-list/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/bags-list/src/weights.rs | 51 ++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 456c4ec5f8a86..6eee2a22b0b0b 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -7,7 +7,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -15,21 +15,62 @@ // See the License for the specific language governing permissions and // limitations under the License. -use frame_support::pallet_prelude::Weight; +//! Autogenerated weights for pallet_bags_list +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2021-09-01, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +// Executed Command: +// target/release/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_bags_list +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/bags-list/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_bags_list. pub trait WeightInfo { fn rebag() -> Weight; } -pub struct SubstrateWeight(sp_std::marker::PhantomData); +/// Weights for pallet_bags_list using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - 0 + (74_658_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) } } +// For backwards compatibility and tests impl WeightInfo for () { + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - 0 + (74_658_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } } From e69ad5df081f568f1a32a66ea7f8aa5d16b58ff9 Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Thu, 2 Sep 2021 02:36:18 +0000 Subject: [PATCH 169/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bags_list --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bags-list/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/bags-list/src/weights.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 6eee2a22b0b0b..27be0587188ce 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_bags_list //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-09-01, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-09-02, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -56,7 +56,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - (74_658_000 as Weight) + (76_182_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -69,7 +69,7 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - (74_658_000 as Weight) + (76_182_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } From 708672dd7a8cb170fbb64c4a663bf56febfbd102 Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Thu, 2 Sep 2021 02:41:05 +0000 Subject: [PATCH 170/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bags_list --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bags-list/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/bags-list/src/weights.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 27be0587188ce..aa291d2c719fc 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -56,7 +56,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - (76_182_000 as Weight) + (74_954_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -69,7 +69,7 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - (76_182_000 as Weight) + (74_954_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } From bb41a291eaa836769386f5ef4a0eb2c99eed36ed Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Thu, 2 Sep 2021 02:49:40 +0000 Subject: [PATCH 171/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bags_list --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bags-list/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/bags-list/src/weights.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index aa291d2c719fc..456bed6538555 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -56,7 +56,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - (74_954_000 as Weight) + (75_718_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -69,7 +69,7 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag() -> Weight { - (74_954_000 as Weight) + (75_718_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } From a3c96247c6593acc9dd6df8f0d34e141085f6063 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 2 Sep 2021 10:42:10 +0200 Subject: [PATCH 172/241] Improving staking pallet-bags-list migration --- frame/staking/src/lib.rs | 2 +- frame/staking/src/migrations.rs | 75 ++++++++++++++++++++------------- frame/staking/src/pallet/mod.rs | 16 +++++++ 3 files changed, 63 insertions(+), 30 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 3ff2e91b7dfb9..d08f860294818 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -732,7 +732,7 @@ enum Releases { V5_0_0, // blockable validators. V6_0_0, // removal of all storage associated with offchain phragmen. V7_0_0, // keep track of number of nominators / validators in map - V8_0_0, // VoterList and efficient semi-sorted iteration + V8_0_0, // Populate `SortedListProvider` with the assumption that it is pallet-bags-list } impl Default for Releases { diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 89661a75a85b2..db2a2b908684e 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -18,35 +18,52 @@ use super::*; -// pub mod v8 { -// use super::{voter_bags::VoterList, *}; -// use frame_support::ensure; - -// pub fn pre_migrate() -> Result<(), &'static str> { -// ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); -// ensure!(VoterList::::iter().count() == 0, "voter list already exists"); -// Ok(()) -// } - -// pub fn migrate() -> Weight { -// log!(info, "Migrating staking to Releases::V8_0_0"); - -// let migrated = VoterList::::regenerate(); -// debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - -// StorageVersion::::put(Releases::V8_0_0); -// log!( -// info, -// "Completed staking migration to Releases::V8_0_0 with {} voters migrated", -// migrated, -// ); - -// T::WeightInfo::regenerate( -// CounterForNominators::::get(), -// ) -// .saturating_add(T::DbWeight::get().reads(2)) -// } -// } +pub mod v8 { + use frame_election_provider_support::SortedListProvider; + use frame_support::traits::Get; + + use crate::{Config, Nominators, Pallet, StorageVersion, Weight}; + + #[cfg(feature = "try-runtime")] + pub fn pre_migrate() -> Result<(), &'static str> { + ensure!(StorageVersion::::get() == crate::Releases::V7_0_0, "must upgrade linearly"); + ensure!(T::SortedListProvider::iter().count() == 0, "voter list already exists"); + crate::log!(info, "👜 staking bags-list migration passes PRE migrate checks ✅",); + Ok(()) + } + + /// Migration to sorted [`SortedListProvider`]. + /// NOTE: this migration makes the assumption that pallet-bags-list is used as the + /// [`SortedListProvider`]. + pub fn migrate() -> Weight { + if StorageVersion::::get() == crate::Releases::V7_0_0 { + crate::log!(info, "migrating staking to Releases::V8_0_0"); + + let migrated = T::SortedListProvider::regenerate( + Nominators::::iter().map(|(id, _)| id), + Pallet::::weight_of_fn(), + ); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + + StorageVersion::::put(crate::Releases::V8_0_0); + crate::log!( + info, + "👜 completed staking migration to Releases::V8_0_0 with {} voters migrated", + migrated, + ); + + T::BlockWeights::get().max_block + } else { + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + fn post_migrate() -> Result<(), &'static str> { + T::SortedListProvider::sanity_check(); + crate::log!(info, "👜 staking bags-list migration passes POST migrate checks ✅",); + } +} pub mod v7 { use super::*; diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 695a57c06a58f..354700fc4f7d9 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -630,6 +630,11 @@ pub mod pallet { fn on_runtime_upgrade() -> Weight { if StorageVersion::::get() == Releases::V6_0_0 { migrations::v7::migrate::() + } else if StorageVersion::::get() == Releases::V7_0_0 { + // NOTE: this migration assumes that pallet-bags-list is being used as the + // [`SortedListProvider`]. If not using pallet-bags-list the runtime author should + // carefully consider if this migration is necessary. + migrations::v8::migrate::() } else { T::DbWeight::get().reads(1) } @@ -639,6 +644,17 @@ pub mod pallet { fn pre_upgrade() -> Result<(), &'static str> { if StorageVersion::::get() == Releases::V6_0_0 { migrations::v7::pre_migrate::() + } else if StorageVersion::::get() == Releases::V7_0_0 { + migrations::v8::pre_migrate::() + } else { + Ok(()) + } + } + + #[cfg(feature = "try-runtime")] + fn post_migrate() -> Result<(), &'static str> { + if StorageVersion::::get() == Releases::V7_0_0 { + migrations::v8::post_migrate::() } else { Ok(()) } From f25a2e7e190777319ef4fb0dd82897c9615751d6 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 2 Sep 2021 14:30:15 +0200 Subject: [PATCH 173/241] fix build and some comments; --- frame/bags-list/src/list/mod.rs | 3 ++- frame/staking/src/migrations.rs | 18 ++++++++++++------ frame/staking/src/pallet/impls.rs | 3 ++- frame/staking/src/pallet/mod.rs | 9 ++++----- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 1ae2a69762556..2d70c8ba5b1d6 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -643,7 +643,8 @@ impl Node { } /// Get the underlying voter. - pub(crate) fn id(&self) -> &T::AccountId { + // TODO: this is public for remote-ext test + pub fn id(&self) -> &T::AccountId { &self.id } } diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index db2a2b908684e..3abfcebabcea3 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -26,15 +26,19 @@ pub mod v8 { #[cfg(feature = "try-runtime")] pub fn pre_migrate() -> Result<(), &'static str> { - ensure!(StorageVersion::::get() == crate::Releases::V7_0_0, "must upgrade linearly"); - ensure!(T::SortedListProvider::iter().count() == 0, "voter list already exists"); + frame_support::ensure!( + StorageVersion::::get() == crate::Releases::V7_0_0, + "must upgrade linearly" + ); + frame_support::ensure!( + T::SortedListProvider::iter().count() == 0, + "voter list already exists" + ); crate::log!(info, "👜 staking bags-list migration passes PRE migrate checks ✅",); Ok(()) } /// Migration to sorted [`SortedListProvider`]. - /// NOTE: this migration makes the assumption that pallet-bags-list is used as the - /// [`SortedListProvider`]. pub fn migrate() -> Weight { if StorageVersion::::get() == crate::Releases::V7_0_0 { crate::log!(info, "migrating staking to Releases::V8_0_0"); @@ -59,9 +63,11 @@ pub mod v8 { } #[cfg(feature = "try-runtime")] - fn post_migrate() -> Result<(), &'static str> { - T::SortedListProvider::sanity_check(); + pub fn post_migrate() -> Result<(), &'static str> { + T::SortedListProvider::sanity_check() + .map_err(|_| "SortedListProvider is not in a sane state.")?; crate::log!(info, "👜 staking bags-list migration passes POST migrate checks ✅",); + Ok(()) } } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index bdd6622b1d61b..6d1a7ce3882a4 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1241,7 +1241,8 @@ impl VoteWeightProvider for Pallet { } } -/// A simple voter list implementation that does not require any additional pallets. +/// A simple voter list implementation that does not require any additional pallets. Note, this +/// does not provided nominators in sorted ordered. pub struct UseNominatorsMap(sp_std::marker::PhantomData); impl SortedListProvider for UseNominatorsMap { type Error = (); diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 354700fc4f7d9..58b164c9e2f84 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -141,7 +141,9 @@ pub mod pallet { #[pallet::constant] type MaxNominatorRewardedPerValidator: Get; - /// Something that can provide a sorted list of voters is a somewhat sorted way. + /// Something that can provide a sorted list of voters is a somewhat sorted way. The original + /// use case for this was designed with [`pallet-bags-list::Pallet`] in mind. If the + /// bags-list is not desired type SortedListProvider: SortedListProvider; /// Weight information for extrinsics in this pallet. @@ -631,9 +633,6 @@ pub mod pallet { if StorageVersion::::get() == Releases::V6_0_0 { migrations::v7::migrate::() } else if StorageVersion::::get() == Releases::V7_0_0 { - // NOTE: this migration assumes that pallet-bags-list is being used as the - // [`SortedListProvider`]. If not using pallet-bags-list the runtime author should - // carefully consider if this migration is necessary. migrations::v8::migrate::() } else { T::DbWeight::get().reads(1) @@ -652,7 +651,7 @@ pub mod pallet { } #[cfg(feature = "try-runtime")] - fn post_migrate() -> Result<(), &'static str> { + fn post_upgrade() -> Result<(), &'static str> { if StorageVersion::::get() == Releases::V7_0_0 { migrations::v8::post_migrate::() } else { From b83afc337e98114d7acc1759068efb9e3a5a86de Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 13:44:24 +0200 Subject: [PATCH 174/241] comment --- frame/staking/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index d08f860294818..be5f1f398a47b 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -732,7 +732,7 @@ enum Releases { V5_0_0, // blockable validators. V6_0_0, // removal of all storage associated with offchain phragmen. V7_0_0, // keep track of number of nominators / validators in map - V8_0_0, // Populate `SortedListProvider` with the assumption that it is pallet-bags-list + V8_0_0, // populate `SortedListProvider`. } impl Default for Releases { From 9de3c56dd4160eac0a0c916087f1d680e768d267 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 14:40:04 +0200 Subject: [PATCH 175/241] Reduce visibility in bags list components --- frame/bags-list/src/list/mod.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 2d70c8ba5b1d6..b468c6323540f 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -460,8 +460,7 @@ impl Bag { } /// Iterate over the nodes in this bag. - // TODO: we make this public for the remote-ext test. - pub fn iter(&self) -> impl Iterator> { + pub(crate) fn iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } @@ -582,6 +581,12 @@ impl Bag { Ok(()) } + + /// Iterate over the nodes in this bag (publicly accessible for testing and benchmarks). + #[cfg(feature = "std")] + pub fn std_iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. @@ -592,7 +597,7 @@ pub struct Node { id: T::AccountId, prev: Option, next: Option, - pub(crate) bag_upper: VoteWeight, + bag_upper: VoteWeight, } impl Node { @@ -643,8 +648,13 @@ impl Node { } /// Get the underlying voter. - // TODO: this is public for remote-ext test - pub fn id(&self) -> &T::AccountId { + pub(crate) fn id(&self) -> &T::AccountId { + &self.id + } + + /// Get the underlying voter (publicly accessible for testing and benchmarks). + #[cfg(feature = "std")] + pub fn std_id(&self) -> &T::AccountId { &self.id } } From 6610e2daf7d54a22dc7d2a18b0ffcac9997846b2 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 14:46:19 +0200 Subject: [PATCH 176/241] make node.bag_upper only accesible to benchmarks --- frame/bags-list/src/lib.rs | 2 +- frame/bags-list/src/list/mod.rs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 677bd0ee8c956..9cbd20578f9b6 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -274,7 +274,7 @@ impl SortedListProvider for Pallet { let current_bag_idx = thresholds .iter() .chain(sp_std::iter::once(&VoteWeight::MAX)) - .position(|w| w == &node.bag_upper) + .position(|w| w == &node.bag_upper()) .unwrap(); if is_increase { diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index b468c6323540f..14ef149fac7b9 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -657,4 +657,10 @@ impl Node { pub fn std_id(&self) -> &T::AccountId { &self.id } + + /// The bag this nodes belongs to (publicly accessible for testing and benchmarks). + #[cfg(feature = "runtime-benchmarks")] + pub fn bag_upper(&self) -> VoteWeight { + self.bag_upper + } } From 627bb160b13c2ee015a49ede2a3b6b779feafd93 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 19:44:30 +0200 Subject: [PATCH 177/241] Address some feedback; comments updates --- Cargo.lock | 153 +++++++++--------- bin/node/runtime/src/lib.rs | 2 +- frame/bags-list/src/benchmarks.rs | 8 +- frame/bags-list/src/lib.rs | 12 +- frame/bags-list/src/list/mod.rs | 23 +-- frame/bags-list/src/list/tests.rs | 79 ++++----- frame/bags-list/src/mock.rs | 2 + frame/bags-list/src/tests.rs | 31 ++-- .../election-provider-multi-phase/src/lib.rs | 2 +- frame/staking/src/pallet/mod.rs | 19 ++- 10 files changed, 179 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58b94660e9bec..6ba2c9c9a1ec3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,7 +318,7 @@ dependencies = [ "memchr", "num_cpus", "once_cell", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", "pin-utils", "slab", "wasm-bindgen-futures", @@ -365,7 +365,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", ] [[package]] @@ -378,7 +378,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", ] [[package]] @@ -695,9 +695,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.7.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" [[package]] name = "byte-slice-cast" @@ -1307,9 +1307,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.20" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +checksum = "e8f45d9ad417bcef4817d614a501ab55cdd96a6fdb24f49aab89a54acfd66b19" dependencies = [ "quote", "syn", @@ -2225,7 +2225,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", "waker-fn", ] @@ -2292,7 +2292,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -2404,9 +2404,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.13.20" +version = "0.13.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba" +checksum = "659cd14835e75b64d9dba5b660463506763cf0aa6cb640aeeb0e98d841093490" dependencies = [ "bitflags", "libc", @@ -2511,9 +2511,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -2591,9 +2591,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.4" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ "bytes 1.0.1", "fnv", @@ -2602,13 +2602,13 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ "bytes 1.0.1", "http", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", ] [[package]] @@ -2672,8 +2672,8 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.7", - "socket2 0.4.1", + "pin-project-lite 0.2.6", + "socket2 0.4.0", "tokio", "tower-service", "tracing", @@ -2817,9 +2817,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" dependencies = [ "cfg-if 1.0.0", ] @@ -3211,15 +3211,15 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.99" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" +checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" [[package]] name = "libgit2-sys" -version = "0.12.21+1.1.0" +version = "0.12.22+1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825" +checksum = "89c53ac117c44f7042ad8d8f5681378dfbc6010e49ec2c0d1f11dfedc7a4a1c3" dependencies = [ "cc", "libc", @@ -3464,7 +3464,7 @@ dependencies = [ "log 0.4.14", "rand 0.8.4", "smallvec 1.6.1", - "socket2 0.4.1", + "socket2 0.4.0", "void", ] @@ -3637,7 +3637,7 @@ dependencies = [ "libc", "libp2p-core", "log 0.4.14", - "socket2 0.4.1", + "socket2 0.4.0", ] [[package]] @@ -3842,9 +3842,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.4" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" dependencies = [ "scopeguard", ] @@ -4086,7 +4086,7 @@ checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" dependencies = [ "libc", "log 0.4.14", - "miow 0.3.7", + "miow 0.3.6", "ntapi", "winapi 0.3.9", ] @@ -4117,10 +4117,11 @@ dependencies = [ [[package]] name = "miow" -version = "0.3.7" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ + "socket2 0.3.19", "winapi 0.3.9", ] @@ -4855,9 +4856,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.8.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" dependencies = [ "parking_lot 0.11.1", ] @@ -6092,7 +6093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" dependencies = [ "instant", - "lock_api 0.4.4", + "lock_api 0.4.2", "parking_lot_core 0.8.3", ] @@ -6134,7 +6135,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.10", + "redox_syscall 0.2.5", "smallvec 1.6.1", "winapi 0.3.9", ] @@ -6302,9 +6303,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.7" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -6973,9 +6974,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" dependencies = [ "bitflags", ] @@ -6987,7 +6988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ "getrandom 0.2.3", - "redox_syscall 0.2.10", + "redox_syscall 0.2.5", ] [[package]] @@ -7024,13 +7025,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.6" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", "regex-syntax", + "thread_local", ] [[package]] @@ -7045,9 +7047,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "region" @@ -8540,9 +8542,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.127" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] @@ -8568,9 +8570,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.127" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote", @@ -8579,9 +8581,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.66" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "itoa", "ryu", @@ -8683,9 +8685,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" dependencies = [ "libc", ] @@ -8710,9 +8712,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.4" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "slog" @@ -8775,9 +8777,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" dependencies = [ "libc", "winapi 0.3.9", @@ -9964,9 +9966,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.74" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" +checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" dependencies = [ "proc-macro2", "quote", @@ -10023,7 +10025,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.4", - "redox_syscall 0.2.10", + "redox_syscall 0.2.5", "remove_dir_all", "winapi 0.3.9", ] @@ -10227,7 +10229,7 @@ dependencies = [ "num_cpus", "once_cell", "parking_lot 0.11.1", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", "signal-hook-registry", "tokio-macros", "winapi 0.3.9", @@ -10323,7 +10325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" dependencies = [ "futures-core", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", "tokio", ] @@ -10373,7 +10375,7 @@ dependencies = [ "futures-io", "futures-sink", "log 0.4.14", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", "tokio", ] @@ -10394,12 +10396,12 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.26" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", - "pin-project-lite 0.2.7", + "pin-project-lite 0.2.6", "tracing-attributes", "tracing-core", ] @@ -10417,9 +10419,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.18" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" dependencies = [ "lazy_static", ] @@ -10711,9 +10713,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" [[package]] name = "universal-hash" @@ -10786,12 +10788,11 @@ dependencies = [ [[package]] name = "value-bag" -version = "1.0.0-alpha.7" +version = "1.0.0-alpha.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd320e1520f94261153e96f7534476ad869c14022aee1e59af7c778075d840ae" +checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" dependencies = [ "ctor", - "version_check 0.9.2", ] [[package]] @@ -11410,9 +11411,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.50" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" +checksum = "c40dc691fc48003eba817c38da7113c15698142da971298003cac3ef175680b3" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 38310e427b1a1..3e08dfe7e1927 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -517,7 +517,7 @@ impl pallet_staking::Config for Runtime { pallet_election_provider_multi_phase::OnChainConfig, >; // Alternatively, use pallet_staking::UseNominatorsMap to just use the nominators map. - // Note that this method does not scale to a very large number of nominators. + // Note that the aforementioned does not scale to a very large number of nominators. type SortedListProvider = BagsList; type WeightInfo = pallet_staking::weights::SubstrateWeight; } diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 63bae8f6c6009..b95eadf1d6659 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -43,6 +43,10 @@ frame_benchmarking::benchmarks! { // when it is removed. // - The destination bag is not empty, because then we need to update the `next` pointer // of the previous node in addition to the work we do otherwise. + // + // NOTE: another expensive case for rebag-ing is for a terminal node, because in that case + // the tail node of each back will need to be updated and both bags will need to be read and + // written to storage. // clear any pre-existing storage. List::::clear(); @@ -55,7 +59,7 @@ frame_benchmarking::benchmarks! { let origin_head: T::AccountId = account("origin_head", 0, 0); assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); - let origin_middle: T::AccountId = account("origin_middle", 0, 0); + let origin_middle: T::AccountId = account("origin_middle", 0, 0); assert_ok!(List::::insert(origin_middle.clone(), origin_bag_thresh)); let origin_tail: T::AccountId = account("origin_tail", 0, 0); @@ -65,7 +69,7 @@ frame_benchmarking::benchmarks! { let dest_head: T::AccountId = account("dest_head", 0, 0); assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); - // and the bags are in the expected state after insertions. + // the bags are in the expected state after insertions. assert_eq!( get_bags::(), vec![ diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 9cbd20578f9b6..f350c132adb91 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -43,9 +43,9 @@ //! it will worsen its position in list iteration; this reduces incentives for some types of spam //! that involve consistently removing and inserting for better position. Further, ordering //! granularity is thus dictated by range between each bag threshold. -//! - if an items weight changes to a value no longer within the range of its current bag the item's -//! position will need to be updated by an external actor with rebag (update), or removal and -//! insertion. +//! - if an item's weight changes to a value no longer within the range of its current bag the +//! item's position will need to be updated by an external actor with rebag (update), or removal +//! and insertion. #![cfg_attr(not(feature = "std"), no_std)] @@ -77,7 +77,7 @@ macro_rules! log { ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { log::$level!( target: crate::LOG_TARGET, - concat!("[{:?}] ", $patter), >::block_number() $(, $values)* + concat!("[{:?}] 👜", $patter), >::block_number() $(, $values)* ) }; } @@ -164,7 +164,7 @@ pub mod pallet { // TODO: we make this public for now only for the sake of the remote-ext tests, find another way // around it. #[pallet::storage] - pub type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; + pub(crate) type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] @@ -183,7 +183,7 @@ pub mod pallet { /// Anyone can call this function about any potentially dislocated account. /// /// Will never return an error; if `dislocated` does not exist or doesn't need a rebag, then - /// it is a noop and only fees are collected from `origin`. + /// it is a noop and fees are still collected from `origin`. #[pallet::weight(T::WeightInfo::rebag())] pub fn rebag(origin: OriginFor, dislocated: T::AccountId) -> DispatchResult { ensure_signed(origin)?; diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 14ef149fac7b9..e9fa3ca9724c1 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -273,7 +273,7 @@ impl List { Ok(()) } - /// Remove a id from the list. + /// Remove an id from the list. pub(crate) fn remove(id: &T::AccountId) { Self::remove_many(sp_std::iter::once(id)); } @@ -369,7 +369,7 @@ impl List { /// is being used, after all other staking data (such as counter) has been updated. It checks: /// /// * there are no duplicate ids, - /// * length of this list are in sync with `CounterForListNodes`. + /// * length of this list is in sync with `CounterForListNodes`, /// * and sanity-checks all bags. This will cascade down all the checks and makes sure all bags /// are checked per *any* update to `List`. pub(crate) fn sanity_check() -> Result<(), &'static str> { @@ -380,7 +380,7 @@ impl List { "duplicate identified", ); - let iter_count = Self::iter().collect::>().len() as u32; + let iter_count = Self::iter().count() as u32; let stored_count = crate::CounterForListNodes::::get(); ensure!(iter_count == stored_count, "iter_count != stored_count",); @@ -430,7 +430,8 @@ impl Bag { }) } - /// Get a bag by its upper vote weight or make it, appropriately initialized. + /// Get a bag by its upper vote weight or make it, appropriately initialized. Does not check if + /// if `bag_upper` is a valid threshold. fn get_or_make(bag_upper: VoteWeight) -> Bag { Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } @@ -474,7 +475,7 @@ impl Bag { fn insert_unchecked(&mut self, id: T::AccountId) { // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long // as this bag is the correct one, we're good. All calls to this must come after getting the - // correct [`notional_bag_for`], getting the bag, and then `bag.insert_unchecked`. + // correct [`notional_bag_for`]. self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); } @@ -500,7 +501,6 @@ impl Bag { // to be `self.bag_upper`. node.bag_upper = self.bag_upper; - // update this node now, making it the new tail. let id = node.id.clone(); // update this node now, treating it as the new tail. node.prev = self.tail.clone(); @@ -582,8 +582,9 @@ impl Bag { Ok(()) } - /// Iterate over the nodes in this bag (publicly accessible for testing and benchmarks). + /// Iterate over the nodes in this bag (public for tests). #[cfg(feature = "std")] + #[allow(dead_code)] pub fn std_iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } @@ -613,7 +614,8 @@ impl Node { /// Update neighboring nodes to point to reach other. /// - /// Does _not_ update storage, so the user may need to call `self.put`. + /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call + /// `self.put`. fn excise(&self) { // Update previous node. if let Some(mut prev) = self.prev() { @@ -652,14 +654,15 @@ impl Node { &self.id } - /// Get the underlying voter (publicly accessible for testing and benchmarks). + /// Get the underlying voter (public fo tests). #[cfg(feature = "std")] pub fn std_id(&self) -> &T::AccountId { &self.id } - /// The bag this nodes belongs to (publicly accessible for testing and benchmarks). + /// The bag this nodes belongs to (public for benchmarks). #[cfg(feature = "runtime-benchmarks")] + #[allow(dead_code)] pub fn bag_upper(&self) -> VoteWeight { self.bag_upper } diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 49efcca8bf416..1cbd981b9cb79 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -33,7 +33,7 @@ fn basic_setup_works() { assert_eq!(ListNodes::::iter().count(), 4); assert_eq!(ListBags::::iter().count(), 2); - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // the state of the bags is as expected assert_eq!( @@ -45,13 +45,13 @@ fn basic_setup_works() { Bag:: { head: Some(2), tail: Some(4), bag_upper: 0 } ); - assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1000)); - assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1000)); - assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1000)); + assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1_000)); + assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1_000)); + assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1_000)); assert_eq!(ListNodes::::get(1).unwrap(), node(1, None, None, 10)); // non-existent id does not have a storage footprint - assert_eq!(ListNodes::::get(41), None); + assert_eq!(ListNodes::::get(42), None); // iteration of the bags would yield: assert_eq!( @@ -71,15 +71,16 @@ fn notional_bag_for_works() { // at a threshold gives that threshold. assert_eq!(notional_bag_for::(10), 10); - // above the threshold. + // above the threshold, gives the next threshold. assert_eq!(notional_bag_for::(11), 20); let max_explicit_threshold = *::BagThresholds::get().last().unwrap(); assert_eq!(max_explicit_threshold, 10_000); + // if the max explicit threshold is less than VoteWeight::MAX, assert!(VoteWeight::MAX > max_explicit_threshold); - // anything above it will belong to the VoteWeight::MAX bag. + // then anything above it will belong to the VoteWeight::MAX bag. assert_eq!(notional_bag_for::(max_explicit_threshold), max_explicit_threshold); assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); } @@ -88,7 +89,7 @@ fn notional_bag_for_works() { fn remove_last_node_in_bags_cleans_bag() { ExtBuilder::default().build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // bump 1 to a bigger bag List::::remove(&1); @@ -114,7 +115,7 @@ fn migrate_works() { vec![ (10, vec![1]), (20, vec![710, 711]), - (1000, vec![2, 3, 4]), + (1_000, vec![2, 3, 4]), (2_000, vec![712]) ] ); @@ -135,7 +136,7 @@ fn migrate_works() { (10, vec![1]), (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 (20, vec![711]), - (1000, vec![2, 3, 4]), + (1_000, vec![2, 3, 4]), // nodes in range 1_001 ..= 2_000 move from bag 2_000 to bag 10_000 (10_000, vec![712]), ] @@ -154,10 +155,8 @@ mod list { // given assert_eq!( get_bags(), - vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![5, 6])] + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![5, 6])] ); - - // then assert_eq!( get_list_as_ids(), vec![ @@ -167,7 +166,7 @@ mod list { ] ); - // when adding a id that has a higher weight than pre-existing ids in the bag + // when adding an id that has a higher weight than pre-existing ids in the bag assert_ok!(List::::insert(7, 10)); // then @@ -182,7 +181,7 @@ mod list { }) } - /// This tests that we can `take` x ids, even if that quantity ends midway through a list. + /// we can `take` x ids, even if that quantity ends midway through a list. #[test] fn take_works() { ExtBuilder::default() @@ -191,7 +190,7 @@ mod list { // given assert_eq!( get_bags(), - vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![5, 6])] + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![5, 6])] ); // when @@ -216,14 +215,17 @@ mod list { assert_ok!(List::::insert(5, 1_000)); // then - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); assert_eq!(get_list_as_ids(), vec![2, 3, 4, 5, 1]); // when inserting into a non-existent bag assert_ok!(List::::insert(6, 1_001)); // then - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 5]), (2000, vec![6])]); + assert_eq!( + get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5]), (2_000, vec![6])] + ); assert_eq!(get_list_as_ids(), vec![6, 2, 3, 4, 5, 1]); }); } @@ -256,12 +258,12 @@ mod list { assert!(!ListNodes::::contains_key(42)); assert_storage_noop!(List::::remove(&42)); - // when removing a node from a bag with multiple nodes + // when removing a node from a bag with multiple nodes: List::::remove(&2); // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); ensure_left(2, 3); // when removing a node from a bag with only one node: @@ -269,7 +271,7 @@ mod list { // then assert_eq!(get_list_as_ids(), vec![3, 4]); - assert_eq!(get_bags(), vec![(1000, vec![3, 4])]); + assert_eq!(get_bags(), vec![(1_000, vec![3, 4])]); ensure_left(1, 2); // bag 10 is removed assert!(!ListBags::::contains_key(10)); @@ -323,18 +325,18 @@ mod list { None )); - // then move it to bag 1000 by giving it weight 500. + // then move it to bag 1_000 by giving it weight 500. assert_eq!(List::::update_position_for(node.clone(), 500), Some((20, 1_000))); assert_eq!(get_bags(), vec![(1_000, vec![2, 3, 4, 1])]); - // moving withing that bag again is a noop + // moving within that bag again is a noop let node = Node::::get(&1).unwrap(); assert_storage_noop!(assert_eq!( List::::update_position_for(node.clone(), 750), None, )); assert_storage_noop!(assert_eq!( - List::::update_position_for(node, 1000), + List::::update_position_for(node, 1_000), None, )); }); @@ -385,17 +387,17 @@ mod bags { assert_eq!(bag_ids, ids); }; - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // we can fetch them check_bag(10, Some(1), Some(1), vec![1]); - check_bag(1000, Some(2), Some(4), vec![2, 3, 4]); + check_bag(1_000, Some(2), Some(4), vec![2, 3, 4]); - // and all other uppers don't get bags. + // and all other bag thresholds don't get bags. ::BagThresholds::get() .iter() .chain(iter::once(&VoteWeight::MAX)) - .filter(|bag_upper| !vec![10, 1000].contains(bag_upper)) + .filter(|bag_upper| !vec![10, 1_000].contains(bag_upper)) .for_each(|bag_upper| { assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); assert!(!ListBags::::contains_key(*bag_upper)); @@ -414,7 +416,7 @@ mod bags { ExtBuilder::default().build_and_execute_no_post_check(|| { let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); let mut bag_10 = Bag::::get(10).unwrap(); bag_10.insert_node_unchecked(node(42, 5)); @@ -433,7 +435,6 @@ mod bags { // when inserting into a bag with 1 node let mut bag_10 = Bag::::get(10).unwrap(); - // (note: bags api does not care about balance or ledger) bag_10.insert_node_unchecked(node(42, bag_10.bag_upper)); // then assert_eq!(bag_as_ids(&bag_10), vec![1, 42]); @@ -463,7 +464,7 @@ mod bags { ); // state of all bags is as expected - bag_20.put(); // need to put this bag so its in the storage map + bag_20.put(); // need to put this newly created bag so its in the storage map assert_eq!( get_bags(), vec![(10, vec![1, 42]), (20, vec![62, 61]), (1_000, vec![2, 3, 4, 52])] @@ -481,7 +482,7 @@ mod bags { let mut bag_1000 = Bag::::get(1_000).unwrap(); bag_1000.insert_node_unchecked(node(42, Some(1), Some(1), 500)); - // then the proper perv and next is set. + // then the proper prev and next is set. assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4, 42]); // and when the node is re-fetched all the info is correct @@ -492,7 +493,7 @@ mod bags { }); ExtBuilder::default().build_and_execute_no_post_check(|| { - // given 3 is in in bag_1000 (and not a tail node) + // given 3 is in bag_1000 (and not a tail node) let mut bag_1000 = Bag::::get(1_000).unwrap(); assert_eq!(bag_as_ids(&bag_1000), vec![2, 3, 4]); @@ -501,7 +502,7 @@ mod bags { // then all the nodes after the duplicate are lost (because it is set as the tail) assert_eq!(bag_as_ids(&bag_1000), vec![2, 3]); - // also in the full iteration, 2 and 3 are from the 1000 bag and 1 from bag 10. + // also in the full iteration, 2 and 3 are from bag_1000 and 1 is from bag_10. assert_eq!(get_list_as_ids(), vec![2, 3, 1]); // and the last accessible node has an **incorrect** prev pointer. @@ -539,7 +540,7 @@ mod bags { let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; // given - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])],); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])],); let mut bag_1000 = Bag::::get(1_000).unwrap(); // when inserting a duplicate id that is already the tail @@ -650,7 +651,7 @@ mod bags { assert_eq!(bag_as_ids(&bag_2000), vec![15, 17, 19]); assert_ok!(bag_2000.sanity_check()); - // finally, when read from storage the state of all bags is as expected + // finally, when reading from storage, the state of all bags is as expected assert_eq!(get_bags(), vec![(10, vec![12]), (2_000, vec![15, 17, 19])]); }); } @@ -676,19 +677,19 @@ mod bags { let bag_1000 = Bag::::get(1_000).unwrap(); // is sane assert_ok!(bag_1000.sanity_check()); - // and has the correct head and tail + // and has the correct head and tail. assert_eq!(bag_1000.head, Some(3)); assert_eq!(bag_1000.tail, Some(4)); }); - // Removing a node that is in another bag, will mess up the other bag. + // Removing a node that is in another bag, will mess up that other bag. ExtBuilder::default().build_and_execute_no_post_check(|| { // given a tail node is in bag 1_000 let node_4 = Node::::get(&4).unwrap(); // when we remove it from bag 10 let mut bag_10 = Bag::::get(10).unwrap(); - bag_10.remove_node_unchecked(&node_4); // node_101 is in bag 1_000 + bag_10.remove_node_unchecked(&node_4); bag_10.put(); // then bag remove was called on is ok, diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index fe57a54e44e80..e7baf9886dbd2 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -15,6 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Mock runtime for pallet-bags-lists tests. + use super::*; use frame_election_provider_support::VoteWeight; use frame_support::parameter_types; diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index f97e183ef5d13..b74403b1051f7 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -29,35 +29,35 @@ mod pallet { fn rebag_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1000, vec![2, 3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])]); // when increasing vote weight to the level of non-existent bag - NextVoteWeight::set(2000); + NextVoteWeight::set(2_000); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then a new bag is created and the id moves into it - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])]); // when decreasing weight within the range of the current bag NextVoteWeight::set(1001); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then the id does not move - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4]), (2000, vec![42])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])]); // when reducing weight to the level of a non-existent bag NextVoteWeight::set(30); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then a new bag is created and the id moves into it - assert_eq!(get_bags(), vec![(10, vec![1]), (30, vec![42]), (1000, vec![2, 3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (30, vec![42]), (1_000, vec![2, 3, 4])]); // when increasing weight to the level of a pre-existing bag NextVoteWeight::set(500); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then the id moves into that bag - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4, 42])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); }); } @@ -67,24 +67,24 @@ mod pallet { fn rebag_tail_works() { ExtBuilder::default().build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![1]), (1000, vec![2, 3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // when NextVoteWeight::set(10); assert_ok!(BagsList::rebag(Origin::signed(0), 4)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 4]), (1000, vec![2, 3])]); + assert_eq!(get_bags(), vec![(10, vec![1, 4]), (1_000, vec![2, 3])]); assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(3), 1_000)); // when assert_ok!(BagsList::rebag(Origin::signed(0), 3)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 4, 3]), (1000, vec![2])]); + assert_eq!(get_bags(), vec![(10, vec![1, 4, 3]), (1_000, vec![2])]); assert_eq!(Bag::::get(10).unwrap(), Bag::new(Some(1), Some(3), 10)); - assert_eq!(Bag::::get(1000).unwrap(), Bag::new(Some(2), Some(2), 1000)); + assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(2), 1_000)); assert_eq!(get_list_as_ids(), vec![2u32, 1, 4, 3]); // when @@ -92,7 +92,7 @@ mod pallet { // then assert_eq!(get_bags(), vec![(10, vec![1, 4, 3, 2])]); - assert_eq!(Bag::::get(1000), None); + assert_eq!(Bag::::get(1_000), None); }); } @@ -106,14 +106,14 @@ mod pallet { assert_ok!(BagsList::rebag(Origin::signed(0), 2)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 2]), (1000, vec![3, 4])]); + assert_eq!(get_bags(), vec![(10, vec![1, 2]), (1_000, vec![3, 4])]); assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(3), Some(4), 1_000)); // when assert_ok!(BagsList::rebag(Origin::signed(0), 3)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 2, 3]), (1000, vec![4])]); + assert_eq!(get_bags(), vec![(10, vec![1, 2, 3]), (1_000, vec![4])]); assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(4), Some(4), 1_000)); // when @@ -233,7 +233,10 @@ mod sorted_list_provider { assert_ok!(BagsList::on_insert(7, 1_001)); // then the bags - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2000, vec![7])]); + assert_eq!( + get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2_000, vec![7])] + ); // and list correctly include the new id, assert_eq!(BagsList::iter().collect::>(), vec![7, 2, 3, 4, 6, 1]); // and the count is incremented. diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 6844b4472b2d0..a989ffa13dfd1 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -638,7 +638,7 @@ pub mod pallet { /// The number of snapshot voters to fetch per block. /// - /// In the future, this value will make more sense with multi-block snapshot. + /// In the future, once m /// /// Also, note the data type: If the voters are represented by a `u32` in `type /// CompactSolution`, the same `u32` is used here to ensure bounds are respected. diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 58b164c9e2f84..368374d4cc3e4 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -141,9 +141,9 @@ pub mod pallet { #[pallet::constant] type MaxNominatorRewardedPerValidator: Get; - /// Something that can provide a sorted list of voters is a somewhat sorted way. The original - /// use case for this was designed with [`pallet-bags-list::Pallet`] in mind. If the - /// bags-list is not desired + /// Something that can provide a sorted list of voters is a somewhat sorted way. The + /// original use case for this was designed with [`pallet-bags-list::Pallet`] in mind. If + /// the bags-list is not desired, [`impls::UseNominatorsMap`] is likely a good option. type SortedListProvider: SortedListProvider; /// Weight information for extrinsics in this pallet. @@ -677,6 +677,19 @@ pub mod pallet { } // `on_finalize` weight is tracked in `on_initialize` } + + fn integrity_test() { + sp_std::if_std! { + sp_io::TestExternalities::new_empty().execute_with(|| + assert!( + T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, + "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", + T::SlashDeferDuration::get(), + T::BondingDuration::get(), + ) + ); + } + } } #[pallet::call] From dba68757f23b59ef46fdbbb413dfcd3979a1d8ff Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 20:27:24 +0200 Subject: [PATCH 178/241] use nominator map comment --- frame/staking/src/pallet/impls.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 6d1a7ce3882a4..a9edc011f67b9 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1242,7 +1242,8 @@ impl VoteWeightProvider for Pallet { } /// A simple voter list implementation that does not require any additional pallets. Note, this -/// does not provided nominators in sorted ordered. +/// does not provided nominators in sorted ordered. If you desire nominators in a sorted order take +/// a look at [`pallet-bags-list]. pub struct UseNominatorsMap(sp_std::marker::PhantomData); impl SortedListProvider for UseNominatorsMap { type Error = (); From 0ca98f953341ff5173aaf1d2551b2684d834a8a7 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 21:01:48 +0200 Subject: [PATCH 179/241] fix vec capacity debug assert --- frame/bags-list/src/list/mod.rs | 5 +++-- frame/staking/src/pallet/impls.rs | 16 +++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index e9fa3ca9724c1..44a6b71715d62 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -584,7 +584,7 @@ impl Bag { /// Iterate over the nodes in this bag (public for tests). #[cfg(feature = "std")] - #[allow(dead_code)] + #[allow(dead_code)] pub fn std_iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } @@ -656,13 +656,14 @@ impl Node { /// Get the underlying voter (public fo tests). #[cfg(feature = "std")] + #[allow(dead_code)] pub fn std_id(&self) -> &T::AccountId { &self.id } /// The bag this nodes belongs to (public for benchmarks). #[cfg(feature = "runtime-benchmarks")] - #[allow(dead_code)] + #[allow(dead_code)] pub fn bag_upper(&self) -> VoteWeight { self.bag_upper } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index a9edc011f67b9..81c393785831a 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -653,14 +653,12 @@ impl Pallet { pub fn get_npos_voters( maybe_max_len: Option, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { - let nominator_count = CounterForNominators::::get() as usize; - let validator_count = CounterForValidators::::get() as usize; - let all_voter_count = validator_count.saturating_add(nominator_count); - drop(validator_count); - drop(nominator_count); - - let max_allowed_len = maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count); - drop(all_voter_count); + let max_allowed_len = { + let nominator_count = CounterForNominators::::get() as usize; + let validator_count = CounterForValidators::::get() as usize; + let all_voter_count = validator_count.saturating_add(nominator_count); + maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count) + }; let mut all_voters = Vec::<_>::with_capacity(max_allowed_len); @@ -697,7 +695,7 @@ impl Pallet { } // all_voters should have not re-allocated. - debug_assert!(all_voters.capacity() == all_voters.len()); + debug_assert!(all_voters.capacity() >= all_voters.len()); Self::register_weight(T::WeightInfo::get_npos_voters( validators_taken, From accfafa8a970fe03473e3ced7c9f69b97c76845a Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 21:04:31 +0200 Subject: [PATCH 180/241] Apply suggestions from code review Co-authored-by: Guillaume Thiolliere --- frame/staking/src/pallet/impls.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 81c393785831a..b0e45a1259522 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -639,8 +639,7 @@ impl Pallet { /// Get all of the voters that are eligible for the npos election. /// - /// `voter_count` imposes a cap on the number of voters returned; care should be taken to ensure - /// that it is accurate. + /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator are included in no particular order, then the nominator in the order returned by the configured [`Config::SortedListProvider`]. /// /// This will use on-chain nominators, and all the validators will inject a self vote. /// From 80ec27f6ff069b33f1c803a956ed72d3645fec44 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 21:24:46 +0200 Subject: [PATCH 181/241] clarify VoterSnapshotPerBlock --- frame/election-provider-multi-phase/src/lib.rs | 6 +++--- frame/staking/src/pallet/impls.rs | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index a989ffa13dfd1..ae938179ee1df 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -636,9 +636,9 @@ pub mod pallet { #[pallet::constant] type SignedDepositWeight: Get>; - /// The number of snapshot voters to fetch per block. - /// - /// In the future, once m + /// The maximum number of voters to put in the snapshot. At the moment, snapshots are only + /// over a single block, but once multi-block elections are introduced they will take place + /// over multiple blocks. /// /// Also, note the data type: If the voters are represented by a `u32` in `type /// CompactSolution`, the same `u32` is used here to ensure bounds are respected. diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index b0e45a1259522..c5879c8d7dfa3 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -639,7 +639,9 @@ impl Pallet { /// Get all of the voters that are eligible for the npos election. /// - /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator are included in no particular order, then the nominator in the order returned by the configured [`Config::SortedListProvider`]. + /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator + /// are included in no particular order, then the nominator in the order returned by the + /// configured [`Config::SortedListProvider`]. /// /// This will use on-chain nominators, and all the validators will inject a self vote. /// From 0c3a081d73574d5d4e6ae9122706776a10605557 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 21:55:32 +0200 Subject: [PATCH 182/241] Reduce diff on create_validators by wrapping with_seed --- frame/election-provider-support/src/lib.rs | 6 +++--- frame/session/benchmarking/src/lib.rs | 2 +- frame/staking/src/benchmarking.rs | 4 ++-- frame/staking/src/testing_utils.rs | 9 ++++++++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 61027c4672e1c..83106dc551f40 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -300,7 +300,7 @@ impl ElectionProvider for () { /// This is generic over `AccountId` and it can represent a validator, a nominator, or any other /// entity. /// -/// On the contrary, to simplify the trait, the `VoteWeight` is hardcoded as the weight of each +/// To simplify the trait, the `VoteWeight` is hardcoded as the weight of each /// entity. The weights are ascending, the higher, the better. In the long term, if this trait ends /// up having use cases outside of the election context, it is easy enough to make it generic over /// the `VoteWeight`. @@ -311,10 +311,10 @@ pub trait SortedListProvider { /// The list's error type. type Error; - /// Returns iterator over the list, which can have `take` called on it. + /// An iterator over the list, which can have `take` called on it. fn iter() -> Box>; - /// get the current count of ids in the list. + /// The current count of ids in the list. fn count() -> u32; /// Return true if the list already contains `id`. diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index 349043bdbf991..8b84145c1acfd 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -126,7 +126,7 @@ fn check_membership_proof_setup( pallet_staking::ValidatorCount::::put(n); // create validators and set random session keys - for (n, who) in create_validators::(n, 1000, 0).unwrap().into_iter().enumerate() { + for (n, who) in create_validators::(n, 1000).unwrap().into_iter().enumerate() { use rand::{RngCore, SeedableRng}; let validator = T::Lookup::lookup(who).unwrap(); diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 27955d78d2119..3ecb16e80ba88 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -358,7 +358,7 @@ benchmarks! { // these are the other validators; there are `T::MAX_NOMINATIONS - 1` of them, so // there are a total of `T::MAX_NOMINATIONS` validators in the system. - let rest_of_validators = create_validators::(T::MAX_NOMINATIONS - 1, 100, 0)?; + let rest_of_validators = create_validators_with_seed::(T::MAX_NOMINATIONS - 1, 100)?; // this is the validator that will be kicking. let (stash, controller) = create_stash_controller::( @@ -432,7 +432,7 @@ benchmarks! { assert!(!Nominators::::contains_key(&stash)); assert!(!T::SortedListProvider::contains(&stash)); - let validators = create_validators::(n, 100, 0).unwrap(); + let validators = create_validators::(n, 100).unwrap(); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), validators) verify { diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 2e57931c6f3cf..8969acfc023c2 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -130,10 +130,17 @@ pub fn create_stash_and_dead_controller( return Ok((stash, controller)) } -/// create `max` validators. pub fn create_validators( max: u32, balance_factor: u32, +) -> Result::Source>, &'static str> { + create_validators_with_seed::(max, balance_factor, 0) +} + +/// create `max` validators. +pub fn create_validators_with_seed( + max: u32, + balance_factor: u32, seed: u32, ) -> Result::Source>, &'static str> { let mut validators: Vec<::Source> = Vec::with_capacity(max as usize); From a182f587cbd8315ebdafa9cc832a37ffbd944128 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 22:54:44 +0200 Subject: [PATCH 183/241] Some small improvements to staking benches --- frame/staking/src/benchmarking.rs | 11 +++++++---- frame/staking/src/pallet/impls.rs | 16 ++++++++-------- frame/staking/src/pallet/mod.rs | 11 +++++++---- frame/staking/src/testing_utils.rs | 3 ++- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 3ecb16e80ba88..58a3a044a9256 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -358,7 +358,7 @@ benchmarks! { // these are the other validators; there are `T::MAX_NOMINATIONS - 1` of them, so // there are a total of `T::MAX_NOMINATIONS` validators in the system. - let rest_of_validators = create_validators_with_seed::(T::MAX_NOMINATIONS - 1, 100)?; + let rest_of_validators = create_validators_with_seed::(T::MAX_NOMINATIONS - 1, 100, 415)?; // this is the validator that will be kicking. let (stash, controller) = create_stash_controller::( @@ -421,7 +421,8 @@ benchmarks! { let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); - // setup a worst case list scenario. + // setup a worst case list scenario. Note we don't care about the destination position, because + // we are just doing an insert into the origin position. let scenario = ListScenario::::new(origin_weight, true)?; let (stash, controller) = create_stash_controller_with_balance::( SEED + T::MAX_NOMINATIONS + 1, // make sure the account does not conflict with others @@ -620,18 +621,20 @@ benchmarks! { let origin_weight = MinNominatorBond::::get() .max(T::Currency::minimum_balance()) - // we use 100 to play friendly with the bags, list threshold values in the mock. + // we use 100 to play friendly with the list threshold values in the mock .max(100u32.into()); // setup a worst case list scenario. let scenario = ListScenario::::new(origin_weight, true)?; let dest_weight = scenario.dest_weight.clone(); - // rebond an amount that will give the use dest_weight + // rebond an amount that will give the user dest_weight let rebond_amount = dest_weight - origin_weight; // spread that amount to rebond across `l` unlocking chunks, let value = rebond_amount / l.into(); + // if `value` is zero, we need a greater delta between dest <=> origin weight + assert_ne!(value, Zero::zero()); // so the sum of unlocking chunks puts voter into the dest bag. assert!(value * l.into() + origin_weight > origin_weight); assert!(value * l.into() + origin_weight <= dest_weight); diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index c5879c8d7dfa3..e7b169d8fca82 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -640,17 +640,17 @@ impl Pallet { /// Get all of the voters that are eligible for the npos election. /// /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator - /// are included in no particular order, then the nominator in the order returned by the - /// configured [`Config::SortedListProvider`]. + /// are included in no particular order, then remainder is taken from the nominators, as returned + /// by [`Config::SortedListProvider`]. /// - /// This will use on-chain nominators, and all the validators will inject a self vote. + /// This will use nominators, and all the validators will inject a self vote. /// /// This function is self-weighing as [`DispatchClass::Mandatory`]. /// /// ### Slashing /// /// All nominations that have been submitted before the last non-zero slash of the validator are - /// auto-chilled. + /// auto-chilled, but still count towards the limit imposed by `maybe_max_len`. pub fn get_npos_voters( maybe_max_len: Option, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { @@ -663,7 +663,7 @@ impl Pallet { let mut all_voters = Vec::<_>::with_capacity(max_allowed_len); - // first, grab all validators, capped by the maximum allowed length. + // first, grab all validators in no particular order, capped by the maximum allowed length. let mut validators_taken = 0u32; for (validator, _) in >::iter().take(max_allowed_len) { // Append self vote. @@ -732,7 +732,7 @@ impl Pallet { } /// This function will add a nominator to the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. + /// [`SortedListProvider`] and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. /// @@ -744,7 +744,7 @@ impl Pallet { // maybe update the counter. CounterForNominators::::mutate(|x| x.saturating_inc()); - // maybe update sorted list. Defensive-only: this should never fail. + // maybe update sorted list. Error checking is defensive-only - this should never fail. if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { log!(warn, "attempt to insert duplicate nominator ({:#?})", who); debug_assert!(false, "attempt to insert duplicate nominator"); @@ -757,7 +757,7 @@ impl Pallet { } /// This function will remove a nominator from the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. + /// [`SortedListProvider`] and keep track of the `CounterForNominators`. /// /// Returns true if `who` was removed from `Nominators`, otherwise false. /// diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 368374d4cc3e4..a575e16745b5e 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -141,9 +141,9 @@ pub mod pallet { #[pallet::constant] type MaxNominatorRewardedPerValidator: Get; - /// Something that can provide a sorted list of voters is a somewhat sorted way. The - /// original use case for this was designed with [`pallet-bags-list::Pallet`] in mind. If - /// the bags-list is not desired, [`impls::UseNominatorsMap`] is likely a good option. + /// Something that can provide a sorted list of voters in a somewhat sorted way. The + /// original use case for this was designed with [`pallet_bags_list::Pallet`] in mind. If + /// the bags-list is not desired, [`impls::UseNominatorsMap`] is likely the desired option. type SortedListProvider: SortedListProvider; /// Weight information for extrinsics in this pallet. @@ -796,7 +796,7 @@ pub mod pallet { Error::::InsufficientBond ); - // ledger must be updated prior to calling `Self::weight_of`. + // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); // update this staker in the sorted list, if they exist in it. if T::SortedListProvider::contains(&stash) { @@ -862,6 +862,7 @@ pub mod pallet { // Note: in case there is no current era it is fine to bond one era more. let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); + // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); // update this staker in the sorted list, if they exist in it. @@ -1364,6 +1365,8 @@ pub mod pallet { ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); + + // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); if T::SortedListProvider::contains(&ledger.stash) { T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 8969acfc023c2..27367048cc6a9 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -130,6 +130,7 @@ pub fn create_stash_and_dead_controller( return Ok((stash, controller)) } +/// create `max` validators. pub fn create_validators( max: u32, balance_factor: u32, @@ -137,7 +138,7 @@ pub fn create_validators( create_validators_with_seed::(max, balance_factor, 0) } -/// create `max` validators. +/// create `max` validators, with a seed to help unintentional prevent account collisions. pub fn create_validators_with_seed( max: u32, balance_factor: u32, From a87a6c5d240635502f515b16998153782aa7eb10 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 23:22:29 +0200 Subject: [PATCH 184/241] Soem comment updates --- frame/staking/src/mock.rs | 3 +-- frame/staking/src/pallet/impls.rs | 4 ++-- utils/frame/generate-bags/src/lib.rs | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index fcd6d21732785..0b98cbe5c6683 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -410,7 +410,6 @@ impl ExtBuilder { } fn build(self) -> sp_io::TestExternalities { sp_tracing::try_init_simple(); - let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); let _ = pallet_balances::GenesisConfig:: { @@ -553,7 +552,7 @@ fn check_count() { assert_eq!(nominator_count, CounterForNominators::::get()); assert_eq!(validator_count, CounterForValidators::::get()); - // the voters that the voter list is storing for us. + // the voters that the `SortedListProvider` list is storing for us. let external_voters = ::SortedListProvider::count(); assert_eq!(external_voters, nominator_count); } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index e7b169d8fca82..0421aaeded8a0 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -640,8 +640,8 @@ impl Pallet { /// Get all of the voters that are eligible for the npos election. /// /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator - /// are included in no particular order, then remainder is taken from the nominators, as returned - /// by [`Config::SortedListProvider`]. + /// are included in no particular order, then remainder is taken from the nominators, as + /// returned by [`Config::SortedListProvider`]. /// /// This will use nominators, and all the validators will inject a self vote. /// diff --git a/utils/frame/generate-bags/src/lib.rs b/utils/frame/generate-bags/src/lib.rs index dc79812682373..b42867d5e5007 100644 --- a/utils/frame/generate-bags/src/lib.rs +++ b/utils/frame/generate-bags/src/lib.rs @@ -30,15 +30,15 @@ //! pub const BagThresholds: &'static [u64] = &[]; //! } //! -//! impl pallet_staking::Config for Runtime { +//! impl pallet_bags_list::Config for Runtime { //! // //! type BagThresholds = BagThresholds; //! } //! ``` //! -//! 2. Write a little program to generate the definitions. This can be a near-identical copy of -//! `substrate/frame/bags-list/generate-bags`. This program exists only to hook together the -//! runtime definitions with the various calculations here. +//! 2. Write a little program to generate the definitions. This program exists only to hook together +//! the runtime definitions with the various calculations here. Take a look at +//! _utils/frame/node-runtime_ for an example. //! //! 3. Run that program: //! From d547cb4bf4c74ae9af9dca1aabfe60bb0c3bcd69 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 3 Sep 2021 23:38:30 +0200 Subject: [PATCH 185/241] fix vec capacity debug assert ... for real this time --- frame/staking/src/pallet/impls.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 0421aaeded8a0..83dff6a94e9ba 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -696,7 +696,7 @@ impl Pallet { } // all_voters should have not re-allocated. - debug_assert!(all_voters.capacity() >= all_voters.len()); + debug_assert!(all_voters.capacity() == max_allowed_len); Self::register_weight(T::WeightInfo::get_npos_voters( validators_taken, From 0a97cdbd277438e7f1d04a9eac69f981dccac2db Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Sat, 4 Sep 2021 00:05:47 +0200 Subject: [PATCH 186/241] Reduce ListBags viz --- frame/bags-list/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index f350c132adb91..feb69a3d54cc7 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -161,8 +161,6 @@ pub mod pallet { /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. - // TODO: we make this public for now only for the sake of the remote-ext tests, find another way - // around it. #[pallet::storage] pub(crate) type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; @@ -222,6 +220,12 @@ impl Pallet { }; maybe_movement } + + /// Equivalent to `ListBags::get`, but public. Useful for tests in outside of this crate. + #[cfg(feature = "std")] + pub fn list_bags_get(weight: VoteWeight) -> Option> { + ListBags::get(weight) + } } impl SortedListProvider for Pallet { From 478adb6821defe3ffeb38fe78fe4c2bac240f042 Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Sat, 4 Sep 2021 08:28:30 +0000 Subject: [PATCH 187/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_staking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/staking/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/staking/src/weights.rs | 454 +++++++++++++++++++---------------- 1 file changed, 253 insertions(+), 201 deletions(-) diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 0667b83844ad6..6b1cd9755d738 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-18, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-09-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -85,39 +85,42 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_523_000 as Weight) + (73_343_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:1) - // Storage: BagsList ListNodes (r:1 w:0) // Storage: Balances Locks (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (58_129_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (115_584_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:0) + // Storage: Staking MinNominatorBond (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: BagsList ListNodes (r:1 w:0) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (61_542_000 as Weight) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) + (124_084_000 as Weight) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (53_160_000 as Weight) + (52_377_000 as Weight) // Standard Error: 0 - .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -126,36 +129,40 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) - // Storage: Staking SpanSlash (r:0 w:2) - fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (85_826_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_453_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) - .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + fn withdraw_unbonded_kill(_s: u32, ) -> Weight { + (104_269_000 as Weight) + .saturating_add(T::DbWeight::get().reads(13 as Weight)) + .saturating_add(T::DbWeight::get().writes(11 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking MinValidatorBond (r:1 w:0) // Storage: Staking Validators (r:1 w:1) // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (34_936_000 as Weight) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (72_339_000 as Weight) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (23_493_000 as Weight) - // Standard Error: 17_000 - .saturating_add((16_632_000 as Weight).saturating_mul(k as Weight)) + (22_679_000 as Weight) + // Standard Error: 16_000 + .saturating_add((16_705_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -168,61 +175,66 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) // Storage: Staking Bonded (r:1 w:0) - // Storage: BagsList ListNodes (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (41_733_000 as Weight) - // Standard Error: 11_000 - .saturating_add((5_840_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (85_089_000 as Weight) + // Standard Error: 18_000 + .saturating_add((5_753_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(T::DbWeight::get().writes(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (17_901_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (71_457_000 as Weight) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_760_000 as Weight) + (13_705_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (28_388_000 as Weight) + (28_406_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_537_000 as Weight) + (2_609_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_749_000 as Weight) + (2_783_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_834_000 as Weight) + (2_704_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_800_000 as Weight) + (2_843_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_429_000 as Weight) + (3_584_000 as Weight) // Standard Error: 0 .saturating_add((55_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) @@ -230,25 +242,29 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (61_799_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_451_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) + (99_538_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_415_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_383_988_000 as Weight) - // Standard Error: 223_000 - .saturating_add((19_981_000 as Weight).saturating_mul(s as Weight)) + (3_410_256_000 as Weight) + // Standard Error: 225_000 + .saturating_add((19_993_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -263,9 +279,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (124_714_000 as Weight) - // Standard Error: 23_000 - .saturating_add((47_575_000 as Weight).saturating_mul(n as Weight)) + (120_896_000 as Weight) + // Standard Error: 19_000 + .saturating_add((47_666_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -283,9 +299,9 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (160_203_000 as Weight) - // Standard Error: 24_000 - .saturating_add((61_321_000 as Weight).saturating_mul(n as Weight)) + (185_434_000 as Weight) + // Standard Error: 27_000 + .saturating_add((61_581_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) @@ -294,13 +310,15 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: BagsList ListNodes (r:1 w:0) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (49_593_000 as Weight) + (113_970_000 as Weight) // Standard Error: 3_000 - .saturating_add((78_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) + .saturating_add((71_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(T::DbWeight::get().reads(9 as Weight)) + .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking HistoryDepth (r:1 w:1) @@ -313,8 +331,8 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 71_000 - .saturating_add((35_237_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 75_000 + .saturating_add((35_484_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -322,19 +340,22 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:1 w:1) // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:1) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (72_484_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_452_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(8 as Weight)) + (102_451_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_413_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking CounterForNominators (r:1 w:0) @@ -343,7 +364,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:101 w:0) // Storage: Staking Ledger (r:101 w:0) // Storage: Staking SlashingSpans (r:1 w:0) - // Storage: Staking Nominators (r:101 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:100 w:0) + // Storage: Staking Nominators (r:100 w:0) // Storage: Staking ValidatorCount (r:1 w:0) // Storage: Staking MinimumValidatorCount (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:1) @@ -355,13 +378,13 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 856_000 - .saturating_add((305_057_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 43_000 - .saturating_add((47_890_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(9 as Weight)) + // Standard Error: 882_000 + .saturating_add((312_808_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 44_000 + .saturating_add((51_780_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(208 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } @@ -376,22 +399,22 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 98_000 - .saturating_add((25_610_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 98_000 - .saturating_add((28_064_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_346_000 - .saturating_add((18_123_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + // Standard Error: 82_000 + .saturating_add((27_544_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 82_000 + .saturating_add((32_927_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 2_797_000 + .saturating_add((15_097_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(204 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { - (30_422_000 as Weight) - // Standard Error: 33_000 - .saturating_add((11_252_000 as Weight).saturating_mul(v as Weight)) + (41_187_000 as Weight) + // Standard Error: 31_000 + .saturating_add((10_913_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -401,20 +424,23 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_486_000 as Weight) + (6_505_000 as Weight) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking ChillThreshold (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking MinValidatorBond (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking MaxNominatorsCount (r:1 w:0) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking MinNominatorBond (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (58_222_000 as Weight) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (85_442_000 as Weight) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } } @@ -427,39 +453,42 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_523_000 as Weight) + (73_343_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:1) - // Storage: BagsList ListNodes (r:1 w:0) // Storage: Balances Locks (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (58_129_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (115_584_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:0) + // Storage: Staking MinNominatorBond (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: BagsList ListNodes (r:1 w:0) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (61_542_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + (124_084_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:1) // Storage: Staking CurrentEra (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (53_160_000 as Weight) + (52_377_000 as Weight) // Standard Error: 0 - .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -468,36 +497,40 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) - // Storage: Staking SpanSlash (r:0 w:2) - fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (85_826_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_453_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + fn withdraw_unbonded_kill(_s: u32, ) -> Weight { + (104_269_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(13 as Weight)) + .saturating_add(RocksDbWeight::get().writes(11 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking MinValidatorBond (r:1 w:0) // Storage: Staking Validators (r:1 w:1) // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (34_936_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (72_339_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (23_493_000 as Weight) - // Standard Error: 17_000 - .saturating_add((16_632_000 as Weight).saturating_mul(k as Weight)) + (22_679_000 as Weight) + // Standard Error: 16_000 + .saturating_add((16_705_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -510,61 +543,66 @@ impl WeightInfo for () { // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) // Storage: Staking Bonded (r:1 w:0) - // Storage: BagsList ListNodes (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (41_733_000 as Weight) - // Standard Error: 11_000 - .saturating_add((5_840_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (85_089_000 as Weight) + // Standard Error: 18_000 + .saturating_add((5_753_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (17_901_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (71_457_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_760_000 as Weight) + (13_705_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (28_388_000 as Weight) + (28_406_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_537_000 as Weight) + (2_609_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_749_000 as Weight) + (2_783_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_834_000 as Weight) + (2_704_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_800_000 as Weight) + (2_843_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_429_000 as Weight) + (3_584_000 as Weight) // Standard Error: 0 .saturating_add((55_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) @@ -572,25 +610,29 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:0) // Storage: Staking Validators (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (61_799_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_451_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + (99_538_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_415_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_383_988_000 as Weight) - // Standard Error: 223_000 - .saturating_add((19_981_000 as Weight).saturating_mul(s as Weight)) + (3_410_256_000 as Weight) + // Standard Error: 225_000 + .saturating_add((19_993_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -605,9 +647,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (124_714_000 as Weight) - // Standard Error: 23_000 - .saturating_add((47_575_000 as Weight).saturating_mul(n as Weight)) + (120_896_000 as Weight) + // Standard Error: 19_000 + .saturating_add((47_666_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) @@ -625,9 +667,9 @@ impl WeightInfo for () { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (160_203_000 as Weight) - // Standard Error: 24_000 - .saturating_add((61_321_000 as Weight).saturating_mul(n as Weight)) + (185_434_000 as Weight) + // Standard Error: 27_000 + .saturating_add((61_581_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) @@ -636,13 +678,15 @@ impl WeightInfo for () { // Storage: Staking Ledger (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: BagsList ListNodes (r:1 w:0) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: Staking Bonded (r:1 w:0) + // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (49_593_000 as Weight) + (113_970_000 as Weight) // Standard Error: 3_000 - .saturating_add((78_000 as Weight).saturating_mul(l as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + .saturating_add((71_000 as Weight).saturating_mul(l as Weight)) + .saturating_add(RocksDbWeight::get().reads(9 as Weight)) + .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking CurrentEra (r:1 w:0) // Storage: Staking HistoryDepth (r:1 w:1) @@ -655,8 +699,8 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 71_000 - .saturating_add((35_237_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 75_000 + .saturating_add((35_484_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -664,19 +708,22 @@ impl WeightInfo for () { // Storage: System Account (r:1 w:1) // Storage: Staking Bonded (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:1) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking Nominators (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (72_484_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_452_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(8 as Weight)) + (102_451_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_413_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking CounterForNominators (r:1 w:0) @@ -685,7 +732,9 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:101 w:0) // Storage: Staking Ledger (r:101 w:0) // Storage: Staking SlashingSpans (r:1 w:0) - // Storage: Staking Nominators (r:101 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:100 w:0) + // Storage: Staking Nominators (r:100 w:0) // Storage: Staking ValidatorCount (r:1 w:0) // Storage: Staking MinimumValidatorCount (r:1 w:0) // Storage: Staking CurrentEra (r:1 w:1) @@ -697,13 +746,13 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 856_000 - .saturating_add((305_057_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 43_000 - .saturating_add((47_890_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(9 as Weight)) + // Standard Error: 882_000 + .saturating_add((312_808_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 44_000 + .saturating_add((51_780_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(208 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } @@ -718,22 +767,22 @@ impl WeightInfo for () { // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 98_000 - .saturating_add((25_610_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 98_000 - .saturating_add((28_064_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_346_000 - .saturating_add((18_123_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + // Standard Error: 82_000 + .saturating_add((27_544_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 82_000 + .saturating_add((32_927_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 2_797_000 + .saturating_add((15_097_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(204 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { - (30_422_000 as Weight) - // Standard Error: 33_000 - .saturating_add((11_252_000 as Weight).saturating_mul(v as Weight)) + (41_187_000 as Weight) + // Standard Error: 31_000 + .saturating_add((10_913_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -743,19 +792,22 @@ impl WeightInfo for () { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_486_000 as Weight) + (6_505_000 as Weight) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking ChillThreshold (r:1 w:0) - // Storage: Staking Nominators (r:1 w:0) - // Storage: Staking Validators (r:1 w:1) - // Storage: Staking MaxValidatorsCount (r:1 w:0) - // Storage: Staking CounterForValidators (r:1 w:1) - // Storage: Staking MinValidatorBond (r:1 w:0) + // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking MaxNominatorsCount (r:1 w:0) + // Storage: Staking CounterForNominators (r:1 w:1) + // Storage: Staking MinNominatorBond (r:1 w:0) + // Storage: Staking Validators (r:1 w:0) + // Storage: BagsList ListNodes (r:2 w:2) + // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (58_222_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (85_442_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } } From 8da3202ab865f5d47f0f5b3e02af92a738ffc046 Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Sat, 4 Sep 2021 22:15:22 +0000 Subject: [PATCH 188/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_staking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/staking/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/staking/src/weights.rs | 220 +++++++++++++++++------------------ 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 6b1cd9755d738..32c8dc80da158 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -85,7 +85,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_343_000 as Weight) + (73_865_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } @@ -95,7 +95,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (115_584_000 as Weight) + (114_296_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(7 as Weight)) } @@ -109,7 +109,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (124_084_000 as Weight) + (121_737_000 as Weight) .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) } @@ -118,7 +118,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (52_377_000 as Weight) + (51_631_000 as Weight) // Standard Error: 0 .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) @@ -138,7 +138,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn withdraw_unbonded_kill(_s: u32, ) -> Weight { - (104_269_000 as Weight) + (101_870_000 as Weight) .saturating_add(T::DbWeight::get().reads(13 as Weight)) .saturating_add(T::DbWeight::get().writes(11 as Weight)) } @@ -153,16 +153,16 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (72_339_000 as Weight) + (69_092_000 as Weight) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (22_679_000 as Weight) - // Standard Error: 16_000 - .saturating_add((16_705_000 as Weight).saturating_mul(k as Weight)) + (21_468_000 as Weight) + // Standard Error: 19_000 + .saturating_add((16_415_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -179,9 +179,9 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (85_089_000 as Weight) - // Standard Error: 18_000 - .saturating_add((5_753_000 as Weight).saturating_mul(n as Weight)) + (82_389_000 as Weight) + // Standard Error: 14_000 + .saturating_add((5_597_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(6 as Weight)) @@ -194,49 +194,49 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (71_457_000 as Weight) + (69_655_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_705_000 as Weight) + (12_770_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (28_406_000 as Weight) + (27_756_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_609_000 as Weight) + (2_446_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_783_000 as Weight) + (2_720_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_704_000 as Weight) + (2_711_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_843_000 as Weight) + (2_796_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_584_000 as Weight) + (3_141_000 as Weight) // Standard Error: 0 - .saturating_add((55_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) @@ -253,18 +253,18 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (99_538_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_415_000 as Weight).saturating_mul(s as Weight)) + (97_394_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_370_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_410_256_000 as Weight) - // Standard Error: 225_000 - .saturating_add((19_993_000 as Weight).saturating_mul(s as Weight)) + (2_783_746_000 as Weight) + // Standard Error: 182_000 + .saturating_add((16_223_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -279,9 +279,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (120_896_000 as Weight) - // Standard Error: 19_000 - .saturating_add((47_666_000 as Weight).saturating_mul(n as Weight)) + (109_233_000 as Weight) + // Standard Error: 17_000 + .saturating_add((47_612_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -299,9 +299,9 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (185_434_000 as Weight) - // Standard Error: 27_000 - .saturating_add((61_581_000 as Weight).saturating_mul(n as Weight)) + (177_392_000 as Weight) + // Standard Error: 20_000 + .saturating_add((60_771_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) @@ -314,9 +314,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (113_970_000 as Weight) - // Standard Error: 3_000 - .saturating_add((71_000 as Weight).saturating_mul(l as Weight)) + (111_858_000 as Weight) + // Standard Error: 4_000 + .saturating_add((36_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(9 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) } @@ -331,8 +331,8 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 75_000 - .saturating_add((35_484_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 68_000 + .saturating_add((33_495_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -351,9 +351,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (102_451_000 as Weight) + (100_178_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_413_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_358_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -378,10 +378,10 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 882_000 - .saturating_add((312_808_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 44_000 - .saturating_add((51_780_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 860_000 + .saturating_add((298_721_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 43_000 + .saturating_add((49_427_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(208 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -399,12 +399,12 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 82_000 - .saturating_add((27_544_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 82_000 - .saturating_add((32_927_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 2_797_000 - .saturating_add((15_097_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 91_000 + .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 91_000 + .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_122_000 + .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(204 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -412,9 +412,9 @@ impl WeightInfo for SubstrateWeight { } // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { - (41_187_000 as Weight) - // Standard Error: 31_000 - .saturating_add((10_913_000 as Weight).saturating_mul(v as Weight)) + (0 as Weight) + // Standard Error: 34_000 + .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -424,7 +424,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_505_000 as Weight) + (6_353_000 as Weight) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) @@ -438,7 +438,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (85_442_000 as Weight) + (83_389_000 as Weight) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } @@ -453,7 +453,7 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_343_000 as Weight) + (73_865_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } @@ -463,7 +463,7 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (115_584_000 as Weight) + (114_296_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } @@ -477,7 +477,7 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (124_084_000 as Weight) + (121_737_000 as Weight) .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } @@ -486,7 +486,7 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (52_377_000 as Weight) + (51_631_000 as Weight) // Standard Error: 0 .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) @@ -506,7 +506,7 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn withdraw_unbonded_kill(_s: u32, ) -> Weight { - (104_269_000 as Weight) + (101_870_000 as Weight) .saturating_add(RocksDbWeight::get().reads(13 as Weight)) .saturating_add(RocksDbWeight::get().writes(11 as Weight)) } @@ -521,16 +521,16 @@ impl WeightInfo for () { // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (72_339_000 as Weight) + (69_092_000 as Weight) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (22_679_000 as Weight) - // Standard Error: 16_000 - .saturating_add((16_705_000 as Weight).saturating_mul(k as Weight)) + (21_468_000 as Weight) + // Standard Error: 19_000 + .saturating_add((16_415_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -547,9 +547,9 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (85_089_000 as Weight) - // Standard Error: 18_000 - .saturating_add((5_753_000 as Weight).saturating_mul(n as Weight)) + (82_389_000 as Weight) + // Standard Error: 14_000 + .saturating_add((5_597_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) @@ -562,49 +562,49 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (71_457_000 as Weight) + (69_655_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (13_705_000 as Weight) + (12_770_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (28_406_000 as Weight) + (27_756_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_609_000 as Weight) + (2_446_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_783_000 as Weight) + (2_720_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_704_000 as Weight) + (2_711_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_843_000 as Weight) + (2_796_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_584_000 as Weight) + (3_141_000 as Weight) // Standard Error: 0 - .saturating_add((55_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) @@ -621,18 +621,18 @@ impl WeightInfo for () { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (99_538_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_415_000 as Weight).saturating_mul(s as Weight)) + (97_394_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_370_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_410_256_000 as Weight) - // Standard Error: 225_000 - .saturating_add((19_993_000 as Weight).saturating_mul(s as Weight)) + (2_783_746_000 as Weight) + // Standard Error: 182_000 + .saturating_add((16_223_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -647,9 +647,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (120_896_000 as Weight) - // Standard Error: 19_000 - .saturating_add((47_666_000 as Weight).saturating_mul(n as Weight)) + (109_233_000 as Weight) + // Standard Error: 17_000 + .saturating_add((47_612_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) @@ -667,9 +667,9 @@ impl WeightInfo for () { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (185_434_000 as Weight) - // Standard Error: 27_000 - .saturating_add((61_581_000 as Weight).saturating_mul(n as Weight)) + (177_392_000 as Weight) + // Standard Error: 20_000 + .saturating_add((60_771_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) @@ -682,9 +682,9 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (113_970_000 as Weight) - // Standard Error: 3_000 - .saturating_add((71_000 as Weight).saturating_mul(l as Weight)) + (111_858_000 as Weight) + // Standard Error: 4_000 + .saturating_add((36_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(9 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } @@ -699,8 +699,8 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 75_000 - .saturating_add((35_484_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 68_000 + .saturating_add((33_495_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) @@ -719,9 +719,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (102_451_000 as Weight) + (100_178_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_413_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_358_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -746,10 +746,10 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 882_000 - .saturating_add((312_808_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 44_000 - .saturating_add((51_780_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 860_000 + .saturating_add((298_721_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 43_000 + .saturating_add((49_427_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(208 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -767,12 +767,12 @@ impl WeightInfo for () { // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 82_000 - .saturating_add((27_544_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 82_000 - .saturating_add((32_927_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 2_797_000 - .saturating_add((15_097_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 91_000 + .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 91_000 + .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_122_000 + .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(204 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -780,9 +780,9 @@ impl WeightInfo for () { } // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { - (41_187_000 as Weight) - // Standard Error: 31_000 - .saturating_add((10_913_000 as Weight).saturating_mul(v as Weight)) + (0 as Weight) + // Standard Error: 34_000 + .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } @@ -792,7 +792,7 @@ impl WeightInfo for () { // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) fn set_staking_limits() -> Weight { - (6_505_000 as Weight) + (6_353_000 as Weight) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Staking Ledger (r:1 w:0) @@ -806,7 +806,7 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (85_442_000 as Weight) + (83_389_000 as Weight) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } From 9ade86d03401e914dcf172f3128d029a59fbe443 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 6 Sep 2021 09:36:57 -0700 Subject: [PATCH 189/241] Remove supports_eq_unordered & Support eq_unordered --- primitives/npos-elections/src/lib.rs | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index 3bccec4305077..6a7e7e8c23cc0 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -349,18 +349,6 @@ pub struct Support { pub voters: Vec<(AccountId, ExtendedBalance)>, } -#[cfg(feature = "mocks")] -impl Support { - /// `true` when the support is identical except for the ordering of the voters. - pub fn eq_unordered(&self, other: &Self) -> bool { - self.total == other.total && { - let my_voters: BTreeMap<_, _> = self.voters.iter().cloned().collect(); - let other_voters: BTreeMap<_, _> = other.voters.iter().cloned().collect(); - my_voters == other_voters - } - } -} - /// A target-major representation of the the election outcome. /// /// Essentially a flat variant of [`SupportMap`]. @@ -368,21 +356,6 @@ impl Support { /// The main advantage of this is that it is encodable. pub type Supports = Vec<(A, Support)>; -#[cfg(feature = "mocks")] -pub fn supports_eq_unordered( - a: &Supports, - b: &Supports, -) -> bool { - let map: BTreeMap<_, _> = a.iter().cloned().collect(); - b.iter().all(|(id, b_support)| { - let a_support = match map.get(id) { - Some(support) => support, - None => return false, - }; - a_support.eq_unordered(b_support) - }) -} - /// Linkage from a winner to their [`Support`]. /// /// This is more helpful than a normal [`Supports`] as it allows faster error checking. From 16a5c87c6aa1d94c41f867d2f127983167b48add Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Mon, 6 Sep 2021 10:05:50 -0700 Subject: [PATCH 190/241] Update utils/frame/generate-bags/src/lib.rs Co-authored-by: Guillaume Thiolliere --- utils/frame/generate-bags/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/generate-bags/src/lib.rs b/utils/frame/generate-bags/src/lib.rs index b42867d5e5007..0eff1fd3bbe51 100644 --- a/utils/frame/generate-bags/src/lib.rs +++ b/utils/frame/generate-bags/src/lib.rs @@ -38,7 +38,7 @@ //! //! 2. Write a little program to generate the definitions. This program exists only to hook together //! the runtime definitions with the various calculations here. Take a look at -//! _utils/frame/node-runtime_ for an example. +//! _utils/frame/generate_bags/node-runtime_ for an example. //! //! 3. Run that program: //! From 2ee478cd6f0250994acfaf2dc0ab947a8926e91c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 7 Sep 2021 09:53:55 -0700 Subject: [PATCH 191/241] Make total-issuance & minimium-balance CLI args; Dont use emptry ext --- Cargo.lock | 2 -- frame/staking/src/benchmarking.rs | 14 ++++++---- .../generate-bags/node-runtime/Cargo.toml | 6 ----- .../generate-bags/node-runtime/src/main.rs | 12 ++++++--- utils/frame/generate-bags/src/lib.rs | 27 ++++++++++++++----- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d59f04de46d56..237e69116d243 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4600,10 +4600,8 @@ dependencies = [ name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ - "frame-support", "generate-bags", "node-runtime", - "sp-io", "structopt", ] diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 58a3a044a9256..61c33c15ee466 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -803,10 +803,12 @@ benchmarks! { // total number of slashing spans. Assigned to validators randomly. let s in 1 .. 20; - let validators = create_validators_with_nominators_for_era::(v, n, T::MAX_NOMINATIONS as usize, false, None)? - .into_iter() - .map(|v| T::Lookup::lookup(v).unwrap()) - .collect::>(); + let validators = create_validators_with_nominators_for_era::( + v, n, T::MAX_NOMINATIONS as usize, false, None + )? + .into_iter() + .map(|v| T::Lookup::lookup(v).unwrap()) + .collect::>(); (0..s).for_each(|index| { add_slashing_spans::(&validators[index as usize], 10); @@ -824,7 +826,9 @@ benchmarks! { // number of nominator intention. let n = MAX_NOMINATORS; - let _ = create_validators_with_nominators_for_era::(v, n, T::MAX_NOMINATIONS as usize, false, None)?; + let _ = create_validators_with_nominators_for_era::( + v, n, T::MAX_NOMINATIONS as usize, false, None + )?; }: { let targets = >::get_npos_targets(); assert_eq!(targets.len() as u32, v); diff --git a/utils/frame/generate-bags/node-runtime/Cargo.toml b/utils/frame/generate-bags/node-runtime/Cargo.toml index 5b517e4162b2a..7fcd981a6bbd6 100644 --- a/utils/frame/generate-bags/node-runtime/Cargo.toml +++ b/utils/frame/generate-bags/node-runtime/Cargo.toml @@ -13,11 +13,5 @@ readme = "README.md" node-runtime = { version = "3.0.0-dev", path = "../../../../bin/node/runtime" } generate-bags = { version = "3.0.0", path = "../" } -# FRAME -frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/support" } - -# primitives -sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } - # third-party structopt = "0.3.21" diff --git a/utils/frame/generate-bags/node-runtime/src/main.rs b/utils/frame/generate-bags/node-runtime/src/main.rs index f99acdbfcea15..9de06bcf7bf73 100644 --- a/utils/frame/generate-bags/node-runtime/src/main.rs +++ b/utils/frame/generate-bags/node-runtime/src/main.rs @@ -29,11 +29,15 @@ struct Opt { /// Where to write the output. output: PathBuf, + + #[structopt(short, long)] + total_issuance: u128, + + #[structopt(short, long)] + minimum_balance: u128, } fn main() -> Result<(), std::io::Error> { - let Opt { n_bags, output } = Opt::from_args(); - let mut ext = sp_io::TestExternalities::new_empty(); - - ext.execute_with(|| generate_thresholds::(n_bags, &output)) + let Opt { n_bags, output, total_issuance, minimum_balance } = Opt::from_args(); + generate_thresholds::(n_bags, &output, total_issuance, minimum_balance) } diff --git a/utils/frame/generate-bags/src/lib.rs b/utils/frame/generate-bags/src/lib.rs index 0eff1fd3bbe51..96e74825dcb68 100644 --- a/utils/frame/generate-bags/src/lib.rs +++ b/utils/frame/generate-bags/src/lib.rs @@ -66,12 +66,23 @@ use std::{ /// Note that this value depends on the current issuance, a quantity known to change over time. /// This makes the project of computing a static value suitable for inclusion in a static, /// generated file _excitingly unstable_. -fn existential_weight() -> VoteWeight { - use frame_support::traits::{Currency, CurrencyToVote}; - - let existential_deposit = >::minimum_balance(); - let issuance = >::total_issuance(); - T::CurrencyToVote::to_vote(existential_deposit, issuance) +fn existential_weight( + total_issuance: u128, + minimum_balance: u128, +) -> VoteWeight { + use frame_support::traits::CurrencyToVote; + use std::convert::TryInto; + + T::CurrencyToVote::to_vote( + minimum_balance + .try_into() + .map_err(|_| "failed to convert minimum_balance to type Balance") + .unwrap(), + total_issuance + .try_into() + .map_err(|_| "failed to convert total_issuance to type Balance") + .unwrap(), + ) } /// Return the path to a header file used in this repository if is exists. @@ -162,6 +173,8 @@ pub fn thresholds( pub fn generate_thresholds( n_bags: usize, output: &Path, + total_issuance: u128, + minimum_balance: u128, ) -> Result<(), std::io::Error> { // ensure the file is accessable if let Some(parent) = output.parent() { @@ -195,7 +208,7 @@ pub fn generate_thresholds( ::Version::get().spec_name, )?; - let existential_weight = existential_weight::(); + let existential_weight = existential_weight::(total_issuance, minimum_balance); num_buf.write_formatted(&existential_weight, &format); writeln!(buf)?; writeln!(buf, "/// Existential weight for this runtime.")?; From 2933daa39106f8df994654509f49014875b78205 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 7 Sep 2021 10:00:26 -0700 Subject: [PATCH 192/241] Improve docs for generate bags CLI args --- utils/frame/generate-bags/node-runtime/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/frame/generate-bags/node-runtime/src/main.rs b/utils/frame/generate-bags/node-runtime/src/main.rs index 9de06bcf7bf73..5d36b381a7d0c 100644 --- a/utils/frame/generate-bags/node-runtime/src/main.rs +++ b/utils/frame/generate-bags/node-runtime/src/main.rs @@ -30,9 +30,12 @@ struct Opt { /// Where to write the output. output: PathBuf, + /// The total issuance of the currency used to create `VoteWeight`. #[structopt(short, long)] total_issuance: u128, + /// The minimum account balance (i.e. existential deposit) for the currency used to create + /// `VoteWeight`. #[structopt(short, long)] minimum_balance: u128, } From c2daac50e730dba06731957b7d49301873bd2105 Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 09:07:20 -0700 Subject: [PATCH 193/241] Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/bags-list/src/lib.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index feb69a3d54cc7..ca7b578842c66 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -15,6 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! # Bags-List Pallet +//! //! A semi-sorted list, where items hold an `AccountId` based on some `VoteWeight`. The `AccountId` //! (`id` for short) might be synonym to a `voter` or `nominator` in some context, and `VoteWeight` //! signifies the chance of each id being included in the final [`VoteWeightProvider::iter`]. @@ -24,7 +26,7 @@ //! weights of accounts via [`sp_election_provider_support::VoteWeightProvider`]. //! //! This pallet is not configurable at genesis. Whoever uses it should call appropriate functions of -//! the `SortedListProvider` (i.e. `on_insert`) at their genesis. +//! the `SortedListProvider` (e.g. `on_insert`, or `regenerate`) at their genesis. //! //! # Goals //! @@ -129,8 +131,7 @@ pub mod pallet { /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * /// constant_ratio).max(threshold[k] + 1)` for all `k`. /// - /// The helpers in the `generate-bags` module can simplify this calculation. To use them, - /// the `generate-bags` feature must be enabled. + /// The helpers in the `/utils/frame/generate-bags` module can simplify this calculation. /// /// # Examples /// @@ -152,15 +153,19 @@ pub mod pallet { } /// How many ids are registered. + // NOTE: This is merely a counter for `ListNodes`. It should someday be replaced by the `CountedMaop` storage. #[pallet::storage] pub(crate) type CounterForListNodes = StorageValue<_, u32, ValueQuery>; - /// Nodes store links forward and back within their respective bags, id. + /// A single node, within some bag. + /// + /// Nodes store links forward and back within their respective bags. #[pallet::storage] pub(crate) type ListNodes = StorageMap<_, Twox64Concat, T::AccountId, list::Node>; - /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which - /// mainly exists to store head and tail pointers to the appropriate nodes. + /// A bag stored in storage. + /// + /// Stores a `Bag` struct, which stores head and tail pointers to itself. #[pallet::storage] pub(crate) type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; From 2cbedc046ac8155741bdd0d73b9da012d42712dd Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 09:11:40 -0700 Subject: [PATCH 194/241] Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/bags-list/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 75423a78b6654..aca230c9a1941 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -60,6 +60,6 @@ runtime-benchmarks = [ "sp-io", "pallet-balances", "sp-tracing", - "frame-election-provider-support/runtime-benchmarks", + "frame-election-provider-support/runtime-benchmarks", ] From 4c8c4eb1986a01dcc7499d2ea7c4a7dbde39c932 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 09:40:37 -0700 Subject: [PATCH 195/241] Don't use default bags weight in node runtime --- bin/node/runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 3e08dfe7e1927..266235d93503a 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -615,7 +615,7 @@ parameter_types! { impl pallet_bags_list::Config for Runtime { type Event = Event; type VoteWeightProvider = Staking; - type WeightInfo = (); + type WeightInfo = pallet_bags_list::weights::SubstrateWeight; type BagThresholds = BagThresholds; } From cf4990cfa4bfd84a75521c81a1b3792e3d219f1e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 10:31:22 -0700 Subject: [PATCH 196/241] Feature gating sanity_check not working --- frame/bags-list/src/lib.rs | 10 ++++++---- frame/bags-list/src/list/mod.rs | 1 + frame/election-provider-support/src/lib.rs | 1 + frame/staking/src/pallet/impls.rs | 1 + 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index ca7b578842c66..da9519a6f7dd0 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -16,7 +16,7 @@ // limitations under the License. //! # Bags-List Pallet -//! +//! //! A semi-sorted list, where items hold an `AccountId` based on some `VoteWeight`. The `AccountId` //! (`id` for short) might be synonym to a `voter` or `nominator` in some context, and `VoteWeight` //! signifies the chance of each id being included in the final [`VoteWeightProvider::iter`]. @@ -153,17 +153,18 @@ pub mod pallet { } /// How many ids are registered. - // NOTE: This is merely a counter for `ListNodes`. It should someday be replaced by the `CountedMaop` storage. + // NOTE: This is merely a counter for `ListNodes`. It should someday be replaced by the + // `CountedMaop` storage. #[pallet::storage] pub(crate) type CounterForListNodes = StorageValue<_, u32, ValueQuery>; - /// A single node, within some bag. + /// A single node, within some bag. /// /// Nodes store links forward and back within their respective bags. #[pallet::storage] pub(crate) type ListNodes = StorageMap<_, Twox64Concat, T::AccountId, list::Node>; - /// A bag stored in storage. + /// A bag stored in storage. /// /// Stores a `Bag` struct, which stores head and tail pointers to itself. #[pallet::storage] @@ -267,6 +268,7 @@ impl SortedListProvider for Pallet { List::::regenerate(all, weight_of) } + #[cfg(any(feature = "std", feature = "debug-assertions"))] fn sanity_check() -> Result<(), &'static str> { List::::sanity_check() } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 44a6b71715d62..d434d302481f3 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -372,6 +372,7 @@ impl List { /// * length of this list is in sync with `CounterForListNodes`, /// * and sanity-checks all bags. This will cascade down all the checks and makes sure all bags /// are checked per *any* update to `List`. + #[cfg(any(feature = "std", feature = "debug-assertions"))] pub(crate) fn sanity_check() -> Result<(), &'static str> { use frame_support::ensure; let mut seen_in_list = BTreeSet::new(); diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 83106dc551f40..d988229ddd4ee 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -341,6 +341,7 @@ pub trait SortedListProvider { fn clear(); /// Sanity check internal state of list. Only meant for debug compilation. + #[cfg(any(feature = "std", feature = "debug-assertions"))] fn sanity_check() -> Result<(), &'static str>; /// If `who` changes by the returned amount they are guaranteed to have a worst case change diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 83dff6a94e9ba..fe340e1d4f8f3 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1274,6 +1274,7 @@ impl SortedListProvider for UseNominatorsMap { // nothing to do upon regenerate. 0 } + #[cfg(any(feature = "std", feature = "debug-assertions"))] fn sanity_check() -> Result<(), &'static str> { Ok(()) } From 3594eab2231198209b0c45b66e2c761707655260 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 11:55:50 -0700 Subject: [PATCH 197/241] Feature gate sanity check by creating duplicate fns --- frame/bags-list/src/lib.rs | 7 ++++++- frame/bags-list/src/list/mod.rs | 15 ++++++++++++++- frame/election-provider-support/src/lib.rs | 1 - frame/staking/src/pallet/impls.rs | 1 - 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index da9519a6f7dd0..7cd86dcf6ea3e 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -268,11 +268,16 @@ impl SortedListProvider for Pallet { List::::regenerate(all, weight_of) } - #[cfg(any(feature = "std", feature = "debug-assertions"))] + #[cfg(feature = "std")] fn sanity_check() -> Result<(), &'static str> { List::::sanity_check() } + #[cfg(not(feature = "std"))] + fn sanity_check() -> Result<(), &'static str> { + Ok(()) + } + fn clear() { List::::clear() } diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index d434d302481f3..304eb6c608087 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -372,7 +372,7 @@ impl List { /// * length of this list is in sync with `CounterForListNodes`, /// * and sanity-checks all bags. This will cascade down all the checks and makes sure all bags /// are checked per *any* update to `List`. - #[cfg(any(feature = "std", feature = "debug-assertions"))] + #[cfg(feature = "std")] pub(crate) fn sanity_check() -> Result<(), &'static str> { use frame_support::ensure; let mut seen_in_list = BTreeSet::new(); @@ -393,6 +393,11 @@ impl List { Ok(()) } + + #[cfg(not(feature = "std"))] + pub(crate) fn sanity_check() -> Result<(), &'static str> { + Ok(()) + } } /// A Bag is a doubly-linked list of ids, where each id is mapped to a [`ListNode`]. @@ -552,6 +557,7 @@ impl Bag { /// * Ensures head has no prev. /// * Ensures tail has no next. /// * Ensures there are no loops, traversal from head to tail is correct. + #[cfg(feature = "std")] fn sanity_check(&self) -> Result<(), &'static str> { frame_support::ensure!( self.head() @@ -583,6 +589,11 @@ impl Bag { Ok(()) } + #[cfg(not(feature = "std"))] + fn sanity_check(&self) -> Result<(), &'static str> { + Ok(()) + } + /// Iterate over the nodes in this bag (public for tests). #[cfg(feature = "std")] #[allow(dead_code)] @@ -630,6 +641,8 @@ impl Node { } } + // fn remove_from_storage_unchecked + /// Get the previous node in the bag. fn prev(&self) -> Option> { self.prev.as_ref().and_then(|id| Node::get(id)) diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index d988229ddd4ee..83106dc551f40 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -341,7 +341,6 @@ pub trait SortedListProvider { fn clear(); /// Sanity check internal state of list. Only meant for debug compilation. - #[cfg(any(feature = "std", feature = "debug-assertions"))] fn sanity_check() -> Result<(), &'static str>; /// If `who` changes by the returned amount they are guaranteed to have a worst case change diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index fe340e1d4f8f3..83dff6a94e9ba 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1274,7 +1274,6 @@ impl SortedListProvider for UseNominatorsMap { // nothing to do upon regenerate. 0 } - #[cfg(any(feature = "std", feature = "debug-assertions"))] fn sanity_check() -> Result<(), &'static str> { Ok(()) } From fa674aa968c7927001be0d7c10fbeecc65f25a00 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 12:04:17 -0700 Subject: [PATCH 198/241] Fix line wrapping --- frame/bags-list/src/lib.rs | 2 +- frame/election-provider-support/src/lib.rs | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 7cd86dcf6ea3e..ff86df482288e 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -273,7 +273,7 @@ impl SortedListProvider for Pallet { List::::sanity_check() } - #[cfg(not(feature = "std"))] + #[écfg(not(feature = "std"))] fn sanity_check() -> Result<(), &'static str> { Ok(()) } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 83106dc551f40..ddca38239a8df 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -300,10 +300,9 @@ impl ElectionProvider for () { /// This is generic over `AccountId` and it can represent a validator, a nominator, or any other /// entity. /// -/// To simplify the trait, the `VoteWeight` is hardcoded as the weight of each -/// entity. The weights are ascending, the higher, the better. In the long term, if this trait ends -/// up having use cases outside of the election context, it is easy enough to make it generic over -/// the `VoteWeight`. +/// To simplify the trait, the `VoteWeight` is hardcoded as the weight of each entity. The weights +/// are ascending, the higher, the better. In the long term, if this trait ends up having use cases +/// outside of the election context, it is easy enough to make it generic over the `VoteWeight`. /// /// Something that implements this trait will do a best-effort sort over ids, and thus can be /// used on the implementing side of [`ElectionDataProvider`]. From 0eb7bf2e8dc79b02969111a7473b3bbc5c9e0703 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 12:07:56 -0700 Subject: [PATCH 199/241] Document VoteWeightProvider --- frame/election-provider-support/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index ddca38239a8df..f8dc76b25e0cb 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -354,8 +354,10 @@ pub trait SortedListProvider { /// [`ElectionDataProvider`], this should typically be implementing by whoever is supposed to *use* /// `SortedListProvider`. pub trait VoteWeightProvider { + /// Get the current `VoteWeight` of `who`. fn vote_weight(who: &AccountId) -> VoteWeight; + /// For tests and benchmarks, set the `VoteWeight`. #[cfg(any(feature = "runtime-benchmarks", test))] fn set_vote_weight_of(_: &AccountId, _: VoteWeight) {} } From 36dafaa8cd73688d41e03a60371a19d7345e5100 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 12:13:02 -0700 Subject: [PATCH 200/241] Make bags ext-builder not a module --- frame/bags-list/src/lib.rs | 2 +- frame/bags-list/src/list/tests.rs | 2 +- frame/bags-list/src/mock.rs | 69 +++++++++++++++---------------- frame/bags-list/src/tests.rs | 2 +- 4 files changed, 36 insertions(+), 39 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index ff86df482288e..7cd86dcf6ea3e 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -273,7 +273,7 @@ impl SortedListProvider for Pallet { List::::sanity_check() } - #[écfg(not(feature = "std"))] + #[cfg(not(feature = "std"))] fn sanity_check() -> Result<(), &'static str> { Ok(()) } diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 1cbd981b9cb79..54f7794a29bbb 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -17,7 +17,7 @@ use super::*; use crate::{ - mock::{ext_builder::*, test_utils::*, *}, + mock::{test_utils::*, *}, CounterForListNodes, ListBags, ListNodes, }; use frame_election_provider_support::SortedListProvider; diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index e7baf9886dbd2..c0238292bdde7 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -95,50 +95,47 @@ frame_support::construct_runtime!( } ); -pub(crate) mod ext_builder { - use super::*; +use super::*; - /// Default AccountIds and their weights. - pub(crate) const GENESIS_IDS: [(AccountId, VoteWeight); 4] = - [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; +/// Default AccountIds and their weights. +pub(crate) const GENESIS_IDS: [(AccountId, VoteWeight); 4] = + [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; - #[derive(Default)] - pub(crate) struct ExtBuilder { - ids: Vec<(AccountId, VoteWeight)>, - } +#[derive(Default)] +pub(crate) struct ExtBuilder { + ids: Vec<(AccountId, VoteWeight)>, +} - impl ExtBuilder { - /// Add some AccountIds to insert into `List`. - pub(crate) fn add_ids(mut self, ids: Vec<(AccountId, VoteWeight)>) -> Self { - self.ids = ids; - self - } +impl ExtBuilder { + /// Add some AccountIds to insert into `List`. + pub(crate) fn add_ids(mut self, ids: Vec<(AccountId, VoteWeight)>) -> Self { + self.ids = ids; + self + } - pub(crate) fn build(self) -> sp_io::TestExternalities { - sp_tracing::try_init_simple(); - let storage = - frame_system::GenesisConfig::default().build_storage::().unwrap(); + pub(crate) fn build(self) -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); + let storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); - let mut ext = sp_io::TestExternalities::from(storage); - ext.execute_with(|| { - for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { - frame_support::assert_ok!(List::::insert(*id, *weight)); - } - }); + let mut ext = sp_io::TestExternalities::from(storage); + ext.execute_with(|| { + for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { + frame_support::assert_ok!(List::::insert(*id, *weight)); + } + }); - ext - } + ext + } - pub(crate) fn build_and_execute(self, test: impl FnOnce() -> ()) { - self.build().execute_with(|| { - test(); - List::::sanity_check().expect("Sanity check post condition failed") - }) - } + pub(crate) fn build_and_execute(self, test: impl FnOnce() -> ()) { + self.build().execute_with(|| { + test(); + List::::sanity_check().expect("Sanity check post condition failed") + }) + } - pub(crate) fn build_and_execute_no_post_check(self, test: impl FnOnce() -> ()) { - self.build().execute_with(test) - } + pub(crate) fn build_and_execute_no_post_check(self, test: impl FnOnce() -> ()) { + self.build().execute_with(test) } } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index b74403b1051f7..1da779afffe4d 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -20,7 +20,7 @@ use frame_support::{assert_ok, assert_storage_noop, traits::IntegrityTest}; use super::*; use frame_election_provider_support::SortedListProvider; use list::Bag; -use mock::{ext_builder::*, test_utils::*, *}; +use mock::{test_utils::*, *}; mod pallet { use super::*; From 6f4043bbdf4f0aa03ab7e2afa57abc2bedeee4fb Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 12:16:34 -0700 Subject: [PATCH 201/241] Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/bags-list/src/mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index c0238292bdde7..0332d2216697e 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -91,7 +91,7 @@ frame_support::construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic, { System: frame_system::{Pallet, Call, Storage, Event, Config}, - BagsList: crate::{Pallet, Call, Storage, Event}, + BagsList: bags_list::{Pallet, Call, Storage, Event}, } ); From 1951b2a736162ddf6d0ae8cd4709d4d488f5c19a Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 12:20:46 -0700 Subject: [PATCH 202/241] use pallet_bags_list instead of crate in mock --- frame/bags-list/src/benchmarks.rs | 2 +- frame/bags-list/src/list/tests.rs | 2 +- frame/bags-list/src/mock.rs | 3 ++- frame/bags-list/src/tests.rs | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index b95eadf1d6659..d5c463952b951 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -102,6 +102,6 @@ frame_benchmarking::benchmarks! { use frame_benchmarking::impl_benchmark_test_suite; impl_benchmark_test_suite!( Pallet, - crate::mock::ext_builder::ExtBuilder::default().build(), + crate::mock::ExtBuilder::default().build(), crate::mock::Runtime, ); diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 54f7794a29bbb..491c27b887cbc 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -365,7 +365,7 @@ mod list { #[test] fn contains_works() { ExtBuilder::default().build_and_execute(|| { - assert!(ext_builder::GENESIS_IDS.iter().all(|(id, _)| List::::contains(id))); + assert!(GENESIS_IDS.iter().all(|(id, _)| List::::contains(id))); let non_existent_ids = vec![&42, &666, &13]; assert!(non_existent_ids.iter().all(|id| !List::::contains(id))); diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index 0332d2216697e..d111619a59141 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -18,6 +18,7 @@ //! Mock runtime for pallet-bags-lists tests. use super::*; +use crate::{self as bags_list}; use frame_election_provider_support::VoteWeight; use frame_support::parameter_types; @@ -75,7 +76,7 @@ parameter_types! { pub static BagThresholds: &'static [VoteWeight] = &[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; } -impl crate::Config for Runtime { +impl bags_list::Config for Runtime { type Event = Event; type WeightInfo = (); type BagThresholds = BagThresholds; diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 1da779afffe4d..791f877333d24 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -350,7 +350,7 @@ mod sorted_list_provider { #[test] fn contains_works() { ExtBuilder::default().build_and_execute(|| { - assert!(ext_builder::GENESIS_IDS.iter().all(|(id, _)| BagsList::contains(id))); + assert!(GENESIS_IDS.iter().all(|(id, _)| BagsList::contains(id))); let non_existent_ids = vec![&42, &666, &13]; assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); From 97459c50808bd097ff895e2002d36518c8ca1484 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 13:15:14 -0700 Subject: [PATCH 203/241] Make get_bags test helper fn live in List --- frame/bags-list/src/benchmarks.rs | 16 ++----- frame/bags-list/src/list/mod.rs | 21 +++++++++ frame/bags-list/src/list/tests.rs | 46 +++++++++++-------- frame/bags-list/src/mock.rs | 14 ------ frame/bags-list/src/tests.rs | 76 +++++++++++++++++++++---------- 5 files changed, 103 insertions(+), 70 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index d5c463952b951..7917ec2eb4618 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -18,22 +18,12 @@ //! Benchmarks for the bags list pallet. use super::*; -use crate::list::{Bag, List}; +use crate::list::List; use frame_benchmarking::{account, whitelisted_caller}; use frame_election_provider_support::VoteWeightProvider; use frame_support::{assert_ok, traits::Get}; use frame_system::RawOrigin as SystemOrigin; -fn get_bags() -> Vec<(VoteWeight, Vec)> { - T::BagThresholds::get() - .into_iter() - .filter_map(|t| { - Bag::::get(*t) - .map(|bag| (*t, bag.iter().map(|n| n.id().clone()).collect::>())) - }) - .collect::>() -} - frame_benchmarking::benchmarks! { rebag { // An expensive case for rebag-ing: @@ -71,7 +61,7 @@ frame_benchmarking::benchmarks! { // the bags are in the expected state after insertions. assert_eq!( - get_bags::(), + List::::get_bags(), vec![ (origin_bag_thresh, vec![origin_head.clone(), origin_middle.clone(), origin_tail.clone()]), (dest_bag_thresh, vec![dest_head.clone()]) @@ -84,7 +74,7 @@ frame_benchmarking::benchmarks! { verify { // check the bags have updated as expected. assert_eq!( - get_bags::(), + List::::get_bags(), vec![ ( origin_bag_thresh, diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 304eb6c608087..e8a22e58a36e8 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -398,6 +398,27 @@ impl List { pub(crate) fn sanity_check() -> Result<(), &'static str> { Ok(()) } + + /// Returns the nodes of all non-empty bags. For testing and benchmarks. + #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] + pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { + use frame_support::traits::Get as _; + + let thresholds = T::BagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter) + } else { + // otherwise, insert it here. + Box::new(iter.chain(sp_std::iter::once(VoteWeight::MAX))) + }; + + iter.filter_map(|t| { + Bag::::get(t).map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) + }) + .collect::>() + } } /// A Bag is a doubly-linked list of ids, where each id is mapped to a [`ListNode`]. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 491c27b887cbc..e2730cbf4e33d 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -33,7 +33,7 @@ fn basic_setup_works() { assert_eq!(ListNodes::::iter().count(), 4); assert_eq!(ListBags::::iter().count(), 2); - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // the state of the bags is as expected assert_eq!( @@ -89,18 +89,21 @@ fn notional_bag_for_works() { fn remove_last_node_in_bags_cleans_bag() { ExtBuilder::default().build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // bump 1 to a bigger bag List::::remove(&1); assert_ok!(List::::insert(1, 10_000)); // then the bag with bound 10 is wiped from storage. - assert_eq!(get_bags(), vec![(1_000, vec![2, 3, 4]), (10_000, vec![1])]); + assert_eq!(List::::get_bags(), vec![(1_000, vec![2, 3, 4]), (10_000, vec![1])]); // and can be recreated again as needed. assert_ok!(List::::insert(77, 10)); - assert_eq!(get_bags(), vec![(10, vec![77]), (1_000, vec![2, 3, 4]), (10_000, vec![1])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![77]), (1_000, vec![2, 3, 4]), (10_000, vec![1])] + ); }); } @@ -111,7 +114,7 @@ fn migrate_works() { .build_and_execute(|| { // given assert_eq!( - get_bags(), + List::::get_bags(), vec![ (10, vec![1]), (20, vec![710, 711]), @@ -131,7 +134,7 @@ fn migrate_works() { // then assert_eq!( - get_bags(), + List::::get_bags(), vec![ (10, vec![1]), (15, vec![710]), // nodes in range 11 ..= 15 move from bag 20 to bag 15 @@ -154,7 +157,7 @@ mod list { .build_and_execute(|| { // given assert_eq!( - get_bags(), + List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![5, 6])] ); assert_eq!( @@ -189,7 +192,7 @@ mod list { .build_and_execute(|| { // given assert_eq!( - get_bags(), + List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![5, 6])] ); @@ -215,7 +218,7 @@ mod list { assert_ok!(List::::insert(5, 1_000)); // then - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); assert_eq!(get_list_as_ids(), vec![2, 3, 4, 5, 1]); // when inserting into a non-existent bag @@ -223,7 +226,7 @@ mod list { // then assert_eq!( - get_bags(), + List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5]), (2_000, vec![6])] ); assert_eq!(get_list_as_ids(), vec![6, 2, 3, 4, 5, 1]); @@ -263,7 +266,7 @@ mod list { // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); ensure_left(2, 3); // when removing a node from a bag with only one node: @@ -271,7 +274,7 @@ mod list { // then assert_eq!(get_list_as_ids(), vec![3, 4]); - assert_eq!(get_bags(), vec![(1_000, vec![3, 4])]); + assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); ensure_left(1, 2); // bag 10 is removed assert!(!ListBags::::contains_key(10)); @@ -316,7 +319,7 @@ mod list { // move it to bag 20. assert_eq!(List::::update_position_for(node, 20), Some((10, 20))); - assert_eq!(get_bags(), vec![(20, vec![1]), (1_000, vec![2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(20, vec![1]), (1_000, vec![2, 3, 4])]); // get the new updated node; try and update the position with no change in weight. let node = Node::::get(&1).unwrap(); @@ -327,7 +330,7 @@ mod list { // then move it to bag 1_000 by giving it weight 500. assert_eq!(List::::update_position_for(node.clone(), 500), Some((20, 1_000))); - assert_eq!(get_bags(), vec![(1_000, vec![2, 3, 4, 1])]); + assert_eq!(List::::get_bags(), vec![(1_000, vec![2, 3, 4, 1])]); // moving within that bag again is a noop let node = Node::::get(&1).unwrap(); @@ -387,7 +390,7 @@ mod bags { assert_eq!(bag_ids, ids); }; - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // we can fetch them check_bag(10, Some(1), Some(1), vec![1]); @@ -416,7 +419,7 @@ mod bags { ExtBuilder::default().build_and_execute_no_post_check(|| { let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); let mut bag_10 = Bag::::get(10).unwrap(); bag_10.insert_node_unchecked(node(42, 5)); @@ -466,7 +469,7 @@ mod bags { // state of all bags is as expected bag_20.put(); // need to put this newly created bag so its in the storage map assert_eq!( - get_bags(), + List::::get_bags(), vec![(10, vec![1, 42]), (20, vec![62, 61]), (1_000, vec![2, 3, 4, 52])] ); }); @@ -540,7 +543,7 @@ mod bags { let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; // given - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])],); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])],); let mut bag_1000 = Bag::::get(1_000).unwrap(); // when inserting a duplicate id that is already the tail @@ -652,7 +655,10 @@ mod bags { assert_ok!(bag_2000.sanity_check()); // finally, when reading from storage, the state of all bags is as expected - assert_eq!(get_bags(), vec![(10, vec![12]), (2_000, vec![15, 17, 19])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![12]), (2_000, vec![15, 17, 19])] + ); }); } @@ -672,7 +678,7 @@ mod bags { bag_1000.put(); // then the node is no longer in any bags - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); // .. and the bag it was removed from let bag_1000 = Bag::::get(1_000).unwrap(); // is sane diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index d111619a59141..a6ab35896b1e7 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -96,8 +96,6 @@ frame_support::construct_runtime!( } ); -use super::*; - /// Default AccountIds and their weights. pub(crate) const GENESIS_IDS: [(AccountId, VoteWeight); 4] = [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; @@ -144,18 +142,6 @@ pub(crate) mod test_utils { use super::*; use list::Bag; - /// Returns the nodes of all non-empty bags. - pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { - BagThresholds::get() - .into_iter() - .chain(std::iter::once(&VoteWeight::MAX)) // assumes this is not an explicit threshold - .filter_map(|t| { - Bag::::get(*t) - .map(|bag| (*t, bag.iter().map(|n| n.id().clone()).collect::>())) - }) - .collect::>() - } - /// Returns the ordered ids within the given bag. pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { bag.iter().map(|n| *n.id()).collect::>() diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 791f877333d24..e94017730668b 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -29,35 +29,50 @@ mod pallet { fn rebag_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])] + ); // when increasing vote weight to the level of non-existent bag NextVoteWeight::set(2_000); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then a new bag is created and the id moves into it - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])] + ); // when decreasing weight within the range of the current bag NextVoteWeight::set(1001); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then the id does not move - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2_000, vec![42])] + ); // when reducing weight to the level of a non-existent bag NextVoteWeight::set(30); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then a new bag is created and the id moves into it - assert_eq!(get_bags(), vec![(10, vec![1]), (30, vec![42]), (1_000, vec![2, 3, 4])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (30, vec![42]), (1_000, vec![2, 3, 4])] + ); // when increasing weight to the level of a pre-existing bag NextVoteWeight::set(500); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then the id moves into that bag - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])] + ); }); } @@ -67,21 +82,21 @@ mod pallet { fn rebag_tail_works() { ExtBuilder::default().build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // when NextVoteWeight::set(10); assert_ok!(BagsList::rebag(Origin::signed(0), 4)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 4]), (1_000, vec![2, 3])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1, 4]), (1_000, vec![2, 3])]); assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(3), 1_000)); // when assert_ok!(BagsList::rebag(Origin::signed(0), 3)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 4, 3]), (1_000, vec![2])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1, 4, 3]), (1_000, vec![2])]); assert_eq!(Bag::::get(10).unwrap(), Bag::new(Some(1), Some(3), 10)); assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(2), 1_000)); @@ -91,7 +106,7 @@ mod pallet { assert_ok!(BagsList::rebag(Origin::signed(0), 2)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 4, 3, 2])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1, 4, 3, 2])]); assert_eq!(Bag::::get(1_000), None); }); } @@ -106,21 +121,21 @@ mod pallet { assert_ok!(BagsList::rebag(Origin::signed(0), 2)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 2]), (1_000, vec![3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1, 2]), (1_000, vec![3, 4])]); assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(3), Some(4), 1_000)); // when assert_ok!(BagsList::rebag(Origin::signed(0), 3)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 2, 3]), (1_000, vec![4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1, 2, 3]), (1_000, vec![4])]); assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(4), Some(4), 1_000)); // when assert_ok!(BagsList::rebag(Origin::signed(0), 4)); // then - assert_eq!(get_bags(), vec![(10, vec![1, 2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1, 2, 3, 4])]); assert_eq!(Bag::::get(1_000), None); }); } @@ -166,12 +181,15 @@ mod pallet { ExtBuilder::default().build_and_execute(|| { // everyone in the same bag. - assert_eq!(get_bags(), vec![(VoteWeight::MAX, vec![1, 2, 3, 4])]); + assert_eq!(List::::get_bags(), vec![(VoteWeight::MAX, vec![1, 2, 3, 4])]); // any insertion goes there as well. assert_ok!(List::::insert(5, 999)); assert_ok!(List::::insert(6, 0)); - assert_eq!(get_bags(), vec![(VoteWeight::MAX, vec![1, 2, 3, 4, 5, 6])]); + assert_eq!( + List::::get_bags(), + vec![(VoteWeight::MAX, vec![1, 2, 3, 4, 5, 6])] + ); // any rebag is noop. assert_storage_noop!(assert!(BagsList::rebag(Origin::signed(0), 1).is_ok())); @@ -223,7 +241,7 @@ mod sorted_list_provider { assert_ok!(BagsList::on_insert(6, 1_000)); // then the bags - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6])]); // and list correctly include the new id, assert_eq!(BagsList::iter().collect::>(), vec![2, 3, 4, 6, 1]); // and the count is incremented. @@ -234,7 +252,7 @@ mod sorted_list_provider { // then the bags assert_eq!( - get_bags(), + List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 6]), (2_000, vec![7])] ); // and list correctly include the new id, @@ -262,14 +280,20 @@ mod sorted_list_provider { fn on_update_works() { ExtBuilder::default().add_ids(vec![(42, 20)]).build_and_execute(|| { // given - assert_eq!(get_bags(), vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (20, vec![42]), (1_000, vec![2, 3, 4])] + ); assert_eq!(BagsList::count(), 5); // when increasing weight to the level of non-existent bag BagsList::on_update(&42, 2_000); // then the bag is created with the id in it, - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])] + ); // and the id position is updated in the list. assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); @@ -277,7 +301,10 @@ mod sorted_list_provider { BagsList::on_update(&42, 1_001); // then the id does not change bags, - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (2000, vec![42])] + ); // or change position in the list. assert_eq!(BagsList::iter().collect::>(), vec![42, 2, 3, 4, 1]); @@ -286,7 +313,7 @@ mod sorted_list_provider { // the the new bag is created with the id in it, assert_eq!( - get_bags(), + List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4]), (VoteWeight::MAX, vec![42])] ); // and the id position is updated in the list. @@ -296,7 +323,10 @@ mod sorted_list_provider { BagsList::on_update(&42, 1_000); // then id is moved to the correct bag (as the last member), - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])]); + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 42])] + ); // and the id position is updated in the list. assert_eq!(BagsList::iter().collect::>(), vec![2, 3, 4, 42, 1]); @@ -324,7 +354,7 @@ mod sorted_list_provider { // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); - assert_eq!(get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); ensure_left(2, 3); // when removing a node from a bag with only one node @@ -332,7 +362,7 @@ mod sorted_list_provider { // then assert_eq!(get_list_as_ids(), vec![3, 4]); - assert_eq!(get_bags(), vec![(1_000, vec![3, 4])]); + assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); ensure_left(1, 2); // when removing all remaining ids From 651148e6e2ee6e2bbf2679f220094b367278e449 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 13:31:11 -0700 Subject: [PATCH 204/241] use remove_from_storage_unchecked for node removal --- frame/bags-list/src/list/mod.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index e8a22e58a36e8..92f2a6aeb2048 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -305,7 +305,7 @@ impl List { } // now get rid of the node itself - crate::ListNodes::::remove(id); + node.remove_from_storage_unchecked() } for (_, bag) in bags { @@ -401,6 +401,7 @@ impl List { /// Returns the nodes of all non-empty bags. For testing and benchmarks. #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] + #[allow(dead_code)] pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { use frame_support::traits::Get as _; @@ -662,7 +663,12 @@ impl Node { } } - // fn remove_from_storage_unchecked + /// This is a naive function that removes a node from the `ListNodes` storage item. + /// + /// It is naive because it does not check if the node has first been removed from its bag. + fn remove_from_storage_unchecked(&self) { + crate::ListNodes::::remove(&self.id) + } /// Get the previous node in the bag. fn prev(&self) -> Option> { From 6d4b526368066fe7cc5f23a0339fd285c27a9cc5 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 13:35:52 -0700 Subject: [PATCH 205/241] Remove count of ids removed in remove_many --- frame/bags-list/src/list/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 92f2a6aeb2048..40f833c1da151 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -225,7 +225,7 @@ impl List { /// Insert several ids into the appropriate bags in the list. Continues with insertions /// if duplicates are detected. /// - /// Return the final count of number of ids inserted. + /// Returns the final count of number of ids inserted. fn insert_many( ids: impl IntoIterator, weight_of: impl Fn(&T::AccountId) -> VoteWeight, @@ -281,7 +281,9 @@ impl List { /// Remove many ids from the list. /// /// This is more efficient than repeated calls to `Self::remove`. - fn remove_many<'a>(ids: impl IntoIterator) { + /// + /// Returns the final count of number of ids removed. + fn remove_many<'a>(ids: impl IntoIterator) -> u32 { let mut bags = BTreeMap::new(); let mut count = 0; @@ -315,6 +317,8 @@ impl List { crate::CounterForListNodes::::mutate(|prev_count| { *prev_count = prev_count.saturating_sub(count) }); + + count } /// Update a node's position in the list. From b39e1c05e1f1a8cec49c5f04ca551c00aee0a713 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:01:13 -0700 Subject: [PATCH 206/241] Add node sanity check, improve list sanity check --- frame/bags-list/src/list/mod.rs | 69 ++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 40f833c1da151..dc16260bdd019 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -387,13 +387,38 @@ impl List { let iter_count = Self::iter().count() as u32; let stored_count = crate::CounterForListNodes::::get(); - ensure!(iter_count == stored_count, "iter_count != stored_count",); + let nodes_count = crate::ListNodes::::iter().count() as u32; + ensure!(iter_count == stored_count, "iter_count != stored_count"); + ensure!(stored_count == nodes_count, "stored_count != nodes_count"); + + crate::log!(debug, "count of nodes: {}", stored_count); + + let active_bags = { + let thresholds = T::BagThresholds::get().iter().copied(); + let thresholds: Vec = if thresholds.clone().last() == Some(VoteWeight::MAX) { + // in the event that they included it, we don't need to make any changes + // Box::new(thresholds.collect() + thresholds.collect() + } else { + // otherwise, insert it here. + thresholds.chain(iter::once(VoteWeight::MAX)).collect() + }; + thresholds.into_iter().filter_map(|t| Bag::::get(t)) + }; + + let _ = active_bags.clone().map(|b| b.sanity_check()).collect::>()?; + + let nodes_in_bags_count = + active_bags.clone().fold(0u32, |acc, cur| acc + cur.iter().count() as u32); + ensure!(nodes_count == nodes_in_bags_count, "stored_count != nodes_in_bags_count"); + + crate::log!(debug, "count of active bags {}", active_bags.count()); - let _ = T::BagThresholds::get() - .into_iter() - .map(|t| Bag::::get(*t).unwrap_or_default()) - .map(|b| b.sanity_check()) - .collect::>()?; + // check that all nodes are sane. We check the `ListNodes` storage item directly in case we + // have some "stale" nodes that are not in a bag. + for (_id, node) in crate::ListNodes::::iter() { + node.sanity_check()? + } Ok(()) } @@ -434,7 +459,7 @@ impl List { /// iteration so that there's no incentive to churn ids positioning to improve the chances of /// appearing within the ids set. #[derive(DefaultNoBound, Encode, Decode)] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone))] #[cfg_attr(test, derive(PartialEq))] pub struct Bag { head: Option, @@ -626,12 +651,17 @@ impl Bag { pub fn std_iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } + + /// Check if the bag contains a node with `id`. + #[cfg(feature = "std")] + fn contains(&self, id: &T::AccountId) -> bool { + self.iter().find(|n| n.id() == id).is_some() + } } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. #[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] -#[cfg_attr(test, derive(PartialEq, Clone))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] pub struct Node { id: T::AccountId, prev: Option, @@ -712,4 +742,25 @@ impl Node { pub fn bag_upper(&self) -> VoteWeight { self.bag_upper } + + #[cfg(feature = "std")] + fn sanity_check(&self) -> Result<(), &'static str> { + let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; + + let id = self.id(); + + frame_support::ensure!( + expected_bag.contains(id), + "node does not exist in the expected bag" + ); + + frame_support::ensure!( + !self.is_terminal() || + expected_bag.head.as_ref() == Some(id) || + expected_bag.tail.as_ref() == Some(id), + "a terminal node is neither its bag head or tail" + ); + + Ok(()) + } } From 161c6919d07b74f01428d82902821f4a99a63f46 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:02:39 -0700 Subject: [PATCH 207/241] Do a list sanity check after on_update --- frame/staking/src/pallet/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index a575e16745b5e..1f7c547319d88 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -801,6 +801,7 @@ pub mod pallet { // update this staker in the sorted list, if they exist in it. if T::SortedListProvider::contains(&stash) { T::SortedListProvider::on_update(&stash, Self::weight_of(&ledger.stash)); + debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); } Self::deposit_event(Event::::Bonded(stash.clone(), extra)); From 597e1865a7f8c509b56977428a60be8a9b2d4df1 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:19:10 -0700 Subject: [PATCH 208/241] List::migrate: clean up debug assert, exit early when no change in thresholds --- frame/bags-list/src/list/mod.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index dc16260bdd019..bfe4c09866a53 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -116,14 +116,23 @@ impl List { /// threshold set. #[allow(dead_code)] pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + let new_thresholds = T::BagThresholds::get(); + if new_thresholds == old_thresholds { + return 0 + } + // we can't check all preconditions, but we can check one debug_assert!( crate::ListBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), "not all `bag_upper` currently in storage are members of `old_thresholds`", ); + debug_assert!( + crate::ListNodes::::iter().all(|(_, node)| old_thresholds.contains(&node.bag_upper)), + "not all `node.bag_upper` currently in storage are members of `old_thresholds`", + ); let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = T::BagThresholds::get().iter().copied().collect(); + let new_set: BTreeSet<_> = new_thresholds.iter().copied().collect(); // accounts that need to be rebagged let mut affected_accounts = BTreeSet::new(); @@ -163,10 +172,12 @@ impl List { } // migrate the voters whose bag has changed - let weight_of = T::VoteWeightProvider::vote_weight; - Self::remove_many(&affected_accounts); let num_affected = affected_accounts.len() as u32; - let _ = Self::insert_many(affected_accounts.into_iter(), weight_of); + let weight_of = T::VoteWeightProvider::vote_weight; + let _removed = Self::remove_many(&affected_accounts); + debug_assert_eq!(_removed, num_affected); + let _inserted = Self::insert_many(affected_accounts.into_iter(), weight_of); + debug_assert_eq!(_inserted, num_affected); // we couldn't previously remove the old bags because both insertion and removal assume that // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid @@ -182,13 +193,7 @@ impl List { crate::ListBags::::remove(removed_bag); } - debug_assert!( - { - let thresholds = T::BagThresholds::get(); - crate::ListBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) - }, - "all `bag_upper` in storage must be members of the new thresholds", - ); + debug_assert_eq!(Self::sanity_check(), Ok(())); num_affected } From 06a05c498c3a26623d9d8dba492725c028de314c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:27:35 -0700 Subject: [PATCH 209/241] Improve public doc comments for pallet_bags_list::list::List --- frame/bags-list/src/list/mod.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index bfe4c09866a53..db351c2ba32c5 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -57,17 +57,20 @@ pub(crate) fn notional_bag_for(weight: VoteWeight) -> VoteWeight { thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } -/// Data structure providing efficient mostly-accurate selection of the top N id by `VoteWeight`. -/// -/// It's implemented as a set of linked lists. Each linked list comprises a bag of ids of -/// arbitrary and unbounded length, all having a vote weight within a particular constant range. -/// This structure means that ids can be added and removed in `O(1)` time. -/// -/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While -/// the users within any particular bag are sorted in an entirely arbitrary order, the overall vote -/// weight decreases as successive bags are reached. This means that it is valid to truncate -/// iteration at any desired point; only those ids in the lowest bag can be excluded. This -/// satisfies both the desire for fairness and the requirement for efficiency. +/// The **only** entry point of this module. All operations to the bags-list should happen through +/// this interface. No other module members should be directly accessed. +// +// Data structure providing efficient mostly-accurate selection of the top N id by `VoteWeight`. +// +// It's implemented as a set of linked lists. Each linked list comprises a bag of ids of +// arbitrary and unbounded length, all having a vote weight within a particular constant range. +// This structure means that ids can be added and removed in `O(1)` time. +// +// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While +// the users within any particular bag are sorted in an entirely arbitrary order, the overall vote +// weight decreases as successive bags are reached. This means that it is valid to truncate +// iteration at any desired point; only those ids in the lowest bag can be excluded. This +// satisfies both the desire for fairness and the requirement for efficiency. pub struct List(PhantomData); impl List { From 69ac8c14fadc1476927b6c43a7ad60f94c07e622 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:28:54 -0700 Subject: [PATCH 210/241] Improve public doc comments for pallet_bags_list::list::List --- frame/bags-list/src/list/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index db351c2ba32c5..9b21c29c45ec9 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -57,8 +57,8 @@ pub(crate) fn notional_bag_for(weight: VoteWeight) -> VoteWeight { thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } -/// The **only** entry point of this module. All operations to the bags-list should happen through -/// this interface. No other module members should be directly accessed. +/// The **ONLY** entry point of this module. All operations to the bags-list should happen through +/// this interface. It is forbidden to access other module members directly. // // Data structure providing efficient mostly-accurate selection of the top N id by `VoteWeight`. // From 35a94719e6ca16b747ef03cb01aa80f0e13fb679 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:44:46 -0700 Subject: [PATCH 211/241] Update generate bags docs --- utils/frame/generate-bags/src/lib.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/frame/generate-bags/src/lib.rs b/utils/frame/generate-bags/src/lib.rs index 96e74825dcb68..af9df4435bcab 100644 --- a/utils/frame/generate-bags/src/lib.rs +++ b/utils/frame/generate-bags/src/lib.rs @@ -43,8 +43,8 @@ //! 3. Run that program: //! //! ```sh,notrust -//! $ cargo run -p node-runtime-generate-bags -- output.rs -//! ``` +//! $ cargo run -p node-runtime-generate-bags -- --total-issuance 1234 --minimum-balance 1 +//! output.rs ``` //! //! 4. Update the runtime definition. //! @@ -162,7 +162,13 @@ pub fn thresholds( /// Write a thresholds module to the path specified. /// -/// The `output` path should terminate with a Rust module name, i.e. `foo/bar/thresholds.rs`. +/// Parameters: +/// - `n_bags` the number of bags to generate. +/// - `output` the path to write to; should terminate with a Rust module name, i.e. +/// `foo/bar/thresholds.rs`. +/// - `total_issuance` the total amount of the currency in the network. +/// - `minimum_balance` the minimum balance of the currency required for an account to exist (i.e. +/// existential deposit). /// /// This generated module contains, in order: /// From 02f607ae9bab45b7254384ae0e75fe12036b056d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 14:46:19 -0700 Subject: [PATCH 212/241] Fix grammar in bags-list benchmark --- frame/bags-list/src/benchmarks.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 7917ec2eb4618..09f6b61702f2f 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -25,12 +25,12 @@ use frame_support::{assert_ok, traits::Get}; use frame_system::RawOrigin as SystemOrigin; frame_benchmarking::benchmarks! { - rebag { + rebag_non_terminal { // An expensive case for rebag-ing: // - // - The node to be rebagged should exist as a non-terminal node in a bag with at least - // 2 other nodes so both its prev and next are nodes that will need be updated - // when it is removed. + // - The node to be rebagged, _R_, should exist as a non-terminal node in a bag with at + // least 2 other nodes. Thus _R_ will have both its `prev` and `next` nodes updated when + // it is removed. // - The destination bag is not empty, because then we need to update the `next` pointer // of the previous node in addition to the work we do otherwise. // From 2f18113f7fe6e577ab91cfe39b8fd45a86ec7890 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 15:13:00 -0700 Subject: [PATCH 213/241] Add benchmark case for `rebag` extrinsic --- frame/bags-list/src/benchmarks.rs | 69 ++++++++++++++++++++++++++----- frame/bags-list/src/lib.rs | 2 +- frame/bags-list/src/weights.rs | 17 ++++++-- 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 09f6b61702f2f..b303c72c191e8 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -26,17 +26,13 @@ use frame_system::RawOrigin as SystemOrigin; frame_benchmarking::benchmarks! { rebag_non_terminal { - // An expensive case for rebag-ing: + // An expensive case for rebag-ing (rebag a non-terminal node): // // - The node to be rebagged, _R_, should exist as a non-terminal node in a bag with at // least 2 other nodes. Thus _R_ will have both its `prev` and `next` nodes updated when - // it is removed. - // - The destination bag is not empty, because then we need to update the `next` pointer - // of the previous node in addition to the work we do otherwise. - // - // NOTE: another expensive case for rebag-ing is for a terminal node, because in that case - // the tail node of each back will need to be updated and both bags will need to be read and - // written to storage. + // it is removed. (3 W/R) + // - The destination bag is not empty, thus we need to update the `next` pointer of the last + // node in the destination in addition to the work we do otherwise. (2 W/R) // clear any pre-existing storage. List::::clear(); @@ -49,7 +45,7 @@ frame_benchmarking::benchmarks! { let origin_head: T::AccountId = account("origin_head", 0, 0); assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); - let origin_middle: T::AccountId = account("origin_middle", 0, 0); + let origin_middle: T::AccountId = account("origin_middle", 0, 0); // the node we rebag (_R_) assert_ok!(List::::insert(origin_middle.clone(), origin_bag_thresh)); let origin_tail: T::AccountId = account("origin_tail", 0, 0); @@ -59,7 +55,7 @@ frame_benchmarking::benchmarks! { let dest_head: T::AccountId = account("dest_head", 0, 0); assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); - // the bags are in the expected state after insertions. + // the bags are in the expected state after initial setup. assert_eq!( List::::get_bags(), vec![ @@ -69,8 +65,9 @@ frame_benchmarking::benchmarks! { ); let caller = whitelisted_caller(); + // update the weight of `origin_middle` to guarantee it will be rebagged into the destination. T::VoteWeightProvider::set_vote_weight_of(&origin_middle, dest_bag_thresh); - }: _(SystemOrigin::Signed(caller), origin_middle.clone()) + }: rebag(SystemOrigin::Signed(caller), origin_middle.clone()) verify { // check the bags have updated as expected. assert_eq!( @@ -87,6 +84,56 @@ frame_benchmarking::benchmarks! { ] ); } + + rebag_terminal { + // An expensive case for rebag-ing (rebag a terminal node): + // + // - The node to be rebagged, _R_, is a terminal node; so _R_, the node pointing to _R_ and + // the origin bag itself will need to be updated. (3 W/R) + // - The destination bag is not empty, thus we need to update the `next` pointer of the last + // node in the destination in addition to the work we do otherwise. (2 W/R) + + // clear any pre-existing storage. + List::::clear(); + + // define our origin and destination thresholds. + let origin_bag_thresh = T::BagThresholds::get()[0]; + let dest_bag_thresh = T::BagThresholds::get()[1]; + + // seed items in the origin bag. + let origin_head: T::AccountId = account("origin_head", 0, 0); + assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); + + let origin_tail: T::AccountId = account("origin_tail", 0, 0); // the node we rebag (_R_) + assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); + + // seed items in the destination bag. + let dest_head: T::AccountId = account("dest_head", 0, 0); + assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); + + // the bags are in the expected state after initial setup. + assert_eq!( + List::::get_bags(), + vec![ + (origin_bag_thresh, vec![origin_head.clone(), origin_tail.clone()]), + (dest_bag_thresh, vec![dest_head.clone()]) + ] + ); + + let caller = whitelisted_caller(); + // update the weight of `origin_tail` to guarantee it will be rebagged into the destination. + T::VoteWeightProvider::set_vote_weight_of(&origin_tail, dest_bag_thresh); + }: rebag(SystemOrigin::Signed(caller), origin_tail.clone()) + verify { + // check the bags have updated as expected. + assert_eq!( + List::::get_bags(), + vec![ + (origin_bag_thresh, vec![origin_head.clone()]), + (dest_bag_thresh, vec![dest_head.clone(), origin_tail.clone()]) + ] + ); + } } use frame_benchmarking::impl_benchmark_test_suite; diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 7cd86dcf6ea3e..308d8ed6c26c9 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -188,7 +188,7 @@ pub mod pallet { /// /// Will never return an error; if `dislocated` does not exist or doesn't need a rebag, then /// it is a noop and fees are still collected from `origin`. - #[pallet::weight(T::WeightInfo::rebag())] + #[pallet::weight(T::WeightInfo::rebag_non_terminal().max(T::WeightInfo::rebag_terminal()))] pub fn rebag(origin: OriginFor, dislocated: T::AccountId) -> DispatchResult { ensure_signed(origin)?; let current_weight = T::VoteWeightProvider::vote_weight(&dislocated); diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 456bed6538555..d663b01e5f3a8 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -45,17 +45,23 @@ use sp_std::marker::PhantomData; /// Weight functions needed for pallet_bags_list. pub trait WeightInfo { - fn rebag() -> Weight; + fn rebag_non_terminal() -> Weight; + fn rebag_terminal() -> Weight; } /// Weights for pallet_bags_list using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { + fn rebag_non_terminal() -> Weight { + (75_718_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) + } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:0) // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) - fn rebag() -> Weight { + fn rebag_terminal() -> Weight { (75_718_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) @@ -64,11 +70,16 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { + fn rebag_non_terminal() -> Weight { + (75_718_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:0) // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) - fn rebag() -> Weight { + fn rebag_terminal() -> Weight { (75_718_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) From d8f3c1c6fac4b774dc26413f252b7bf1f85903c4 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 16:18:57 -0700 Subject: [PATCH 214/241] Add count parameter to List::clear; WIP adding MaxEncodedLen to list' --- frame/bags-list/src/benchmarks.rs | 4 +- frame/bags-list/src/lib.rs | 8 ++- frame/bags-list/src/list/mod.rs | 106 ++++++++++++++++-------------- 3 files changed, 62 insertions(+), 56 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index b303c72c191e8..db084fd31bd24 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -35,7 +35,7 @@ frame_benchmarking::benchmarks! { // node in the destination in addition to the work we do otherwise. (2 W/R) // clear any pre-existing storage. - List::::clear(); + List::::clear(None); // define our origin and destination thresholds. let origin_bag_thresh = T::BagThresholds::get()[0]; @@ -94,7 +94,7 @@ frame_benchmarking::benchmarks! { // node in the destination in addition to the work we do otherwise. (2 W/R) // clear any pre-existing storage. - List::::clear(); + List::::clear(None); // define our origin and destination thresholds. let origin_bag_thresh = T::BagThresholds::get()[0]; diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 308d8ed6c26c9..f2f979d79ba44 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -92,6 +92,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] + // #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::config] @@ -219,7 +220,7 @@ impl Pallet { ) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = list::Node::::get(&account) + let maybe_movement = list::Node::::get::(&account) .and_then(|node| List::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { Self::deposit_event(Event::::Rebagged(account.clone(), from, to)); @@ -278,15 +279,16 @@ impl SortedListProvider for Pallet { Ok(()) } + #[cfg(feature = "runtime-benchmarks")] fn clear() { - List::::clear() + List::::clear(None) } #[cfg(feature = "runtime-benchmarks")] fn weight_update_worst_case(who: &T::AccountId, is_increase: bool) -> VoteWeight { use frame_support::traits::Get as _; let thresholds = T::BagThresholds::get(); - let node = list::Node::::get(who).unwrap(); + let node = list::Node::::get::(who).unwrap(); let current_bag_idx = thresholds .iter() .chain(sp_std::iter::once(&VoteWeight::MAX)) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 9b21c29c45ec9..0dfcceecd4d02 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -25,7 +25,7 @@ //! interface. use crate::Config; -use codec::{Decode, Encode}; +use codec::{Decode, Encode, MaxEncodedLen}; use frame_election_provider_support::{VoteWeight, VoteWeightProvider}; use frame_support::{traits::Get, DefaultNoBound}; use sp_std::{ @@ -74,17 +74,19 @@ pub(crate) fn notional_bag_for(weight: VoteWeight) -> VoteWeight { pub struct List(PhantomData); impl List { - /// Remove all data associated with the list from storage. - pub(crate) fn clear() { + /// Remove all data associated with the list from storage. Parameter `items` is the number of + /// items to clear from the list. WARNING: `None` will clear all items and should generally not + /// be used in production as it could lead to an infinite number of storage accesses. + pub(crate) fn clear(items: Option) { crate::CounterForListNodes::::kill(); - crate::ListBags::::remove_all(None); - crate::ListNodes::::remove_all(None); + crate::ListBags::::remove_all(items); + crate::ListNodes::::remove_all(items); } /// Regenerate all of the data from the given ids. /// - /// This is expensive and should only ever be performed during a migration, or when - /// the data needs to be generated from scratch again. + /// WARNING: this is expensive and should only ever be performed when the list needs to be + /// generated from scratch. Care needs to be taken to ensure /// /// This may or may not need to be called at genesis as well, based on the configuration of the /// pallet using this `List`. @@ -94,7 +96,7 @@ impl List { all: impl IntoIterator, weight_of: Box VoteWeight>, ) -> u32 { - Self::clear(); + Self::clear(None); Self::insert_many(all, weight_of) } @@ -210,7 +212,7 @@ impl List { /// /// Full iteration can be expensive; it's recommended to limit the number of items with /// `.take(n)`. - pub(crate) fn iter() -> impl Iterator> { + pub(crate) fn iter() -> impl Iterator> { // We need a touch of special handling here: because we permit `T::BagThresholds` to // omit the final bound, we need to ensure that we explicitly include that threshold in the // list. @@ -296,7 +298,7 @@ impl List { let mut count = 0; for id in ids.into_iter() { - let node = match Node::::get(id) { + let node = match Node::::get::(id) { Some(node) => node, None => continue, }; @@ -304,7 +306,7 @@ impl List { if !node.is_terminal() { // this node is not a head or a tail and thus the bag does not need to be updated - node.excise() + node.excise::() } else { // this node is a head or tail, so the bag needs to be updated let bag = bags @@ -340,16 +342,16 @@ impl List { /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient /// to call [`self.remove_many`] followed by [`self.insert_many`]. pub(crate) fn update_position_for( - node: Node, + node: Node, new_weight: VoteWeight, ) -> Option<(VoteWeight, VoteWeight)> { - node.is_misplaced(new_weight).then(move || { + node.is_misplaced::(new_weight).then(move || { let old_bag_upper = node.bag_upper; if !node.is_terminal() { // this node is not a head or a tail, so we can just cut it out of the list. update // and put the prev and next of this node, we do `node.put` inside `insert_note`. - node.excise(); + node.excise::(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { // this is a head or tail, so the bag must be updated. bag.remove_node_unchecked(&node); @@ -466,9 +468,8 @@ impl List { /// desirable to ensure that there is some element of first-come, first-serve to the list's /// iteration so that there's no incentive to churn ids positioning to improve the chances of /// appearing within the ids set. -#[derive(DefaultNoBound, Encode, Decode)] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone))] -#[cfg_attr(test, derive(PartialEq))] +#[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen)] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] pub struct Bag { head: Option, tail: Option, @@ -516,18 +517,18 @@ impl Bag { } /// Get the head node in this bag. - fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(id)) + fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get::(id)) } /// Get the tail node in this bag. - fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(id)) + fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get::(id)) } /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) + pub(crate) fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next::()) } /// Insert a new id into this bag. @@ -541,7 +542,7 @@ impl Bag { // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long // as this bag is the correct one, we're good. All calls to this must come after getting the // correct [`notional_bag_for`]. - self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); + self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); } /// Insert a node into this bag. @@ -551,7 +552,7 @@ impl Bag { /// /// Storage note: this modifies storage, but only for the node. You still need to call /// `self.put()` after use. - fn insert_node_unchecked(&mut self, mut node: Node) { + fn insert_node_unchecked(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { // this should never happen, but this check prevents one path to a worst case @@ -595,9 +596,9 @@ impl Bag { /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. - fn remove_node_unchecked(&mut self, node: &Node) { + fn remove_node_unchecked(&mut self, node: &Node) { // reassign neighboring nodes. - node.excise(); + node.excise::(); // clear the bag head/tail pointers as necessary. if self.tail.as_ref() == Some(&node.id) { @@ -620,7 +621,7 @@ impl Bag { fn sanity_check(&self) -> Result<(), &'static str> { frame_support::ensure!( self.head() - .map(|head| head.prev().is_none()) + .map(|head| head.prev::().is_none()) // if there is no head, then there must not be a tail, meaning that the bag is // empty. .unwrap_or_else(|| self.tail.is_none()), @@ -629,7 +630,7 @@ impl Bag { frame_support::ensure!( self.tail() - .map(|tail| tail.next().is_none()) + .map(|tail| tail.next::().is_none()) // if there is no tail, then there must not be a head, meaning that the bag is // empty. .unwrap_or_else(|| self.head.is_none()), @@ -656,8 +657,8 @@ impl Bag { /// Iterate over the nodes in this bag (public for tests). #[cfg(feature = "std")] #[allow(dead_code)] - pub fn std_iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) + pub fn std_iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next::()) } /// Check if the bag contains a node with `id`. @@ -668,23 +669,26 @@ impl Bag { } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. -#[derive(Encode, Decode)] +#[derive(Encode, Decode, MaxEncodedLen)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] -pub struct Node { - id: T::AccountId, - prev: Option, - next: Option, +pub struct Node { + id: AccountId, + prev: Option, + next: Option, bag_upper: VoteWeight, + + // #[codec(skip)] + // _phantom: PhantomData } -impl Node { +impl Node { /// Get a node by id. - pub(crate) fn get(id: &T::AccountId) -> Option> { + pub(crate) fn get(id: &AccountId) -> Option> { crate::ListNodes::::try_get(id).ok() } /// Put the node back into storage. - fn put(self) { + fn put(self) { crate::ListNodes::::insert(self.id.clone(), self); } @@ -692,14 +696,14 @@ impl Node { /// /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call /// `self.put`. - fn excise(&self) { + fn excise(&self) { // Update previous node. - if let Some(mut prev) = self.prev() { + if let Some(mut prev) = self.prev::() { prev.next = self.next.clone(); prev.put(); } // Update next self. - if let Some(mut next) = self.next() { + if let Some(mut next) = self.next::() { next.prev = self.prev.clone(); next.put(); } @@ -708,22 +712,22 @@ impl Node { /// This is a naive function that removes a node from the `ListNodes` storage item. /// /// It is naive because it does not check if the node has first been removed from its bag. - fn remove_from_storage_unchecked(&self) { + fn remove_from_storage_unchecked(&self) { crate::ListNodes::::remove(&self.id) } /// Get the previous node in the bag. - fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| Node::get(id)) + fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| Node::get::(id)) } /// Get the next node in the bag. - fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| Node::get(id)) + fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| Node::get::(id)) } /// `true` when this voter is in the wrong bag. - pub(crate) fn is_misplaced(&self, current_weight: VoteWeight) -> bool { + pub(crate) fn is_misplaced(&self, current_weight: VoteWeight) -> bool { notional_bag_for::(current_weight) != self.bag_upper } @@ -733,14 +737,14 @@ impl Node { } /// Get the underlying voter. - pub(crate) fn id(&self) -> &T::AccountId { + pub(crate) fn id(&self) -> &AccountId { &self.id } /// Get the underlying voter (public fo tests). #[cfg(feature = "std")] #[allow(dead_code)] - pub fn std_id(&self) -> &T::AccountId { + pub fn std_id(&self) -> &AccountId { &self.id } @@ -752,7 +756,7 @@ impl Node { } #[cfg(feature = "std")] - fn sanity_check(&self) -> Result<(), &'static str> { + fn sanity_check(&self) -> Result<(), &'static str> { let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; let id = self.id(); From 3af2543ad59cf1001efa0164f9461ad8e8a2dfae Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 16:46:33 -0700 Subject: [PATCH 215/241] MaxEncodeLen + generate_storage_info not working for Bag or Node --- frame/bags-list/src/lib.rs | 8 ++-- frame/bags-list/src/list/mod.rs | 83 ++++++++++++++++----------------- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index f2f979d79ba44..3c20a21290d21 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -92,7 +92,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] - // #[pallet::generate_storage_info] + #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::config] @@ -220,7 +220,7 @@ impl Pallet { ) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = list::Node::::get::(&account) + let maybe_movement = list::Node::::get(&account) .and_then(|node| List::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { Self::deposit_event(Event::::Rebagged(account.clone(), from, to)); @@ -279,7 +279,7 @@ impl SortedListProvider for Pallet { Ok(()) } - #[cfg(feature = "runtime-benchmarks")] + // #[cfg(feature = "runtime-benchmarks")] fn clear() { List::::clear(None) } @@ -288,7 +288,7 @@ impl SortedListProvider for Pallet { fn weight_update_worst_case(who: &T::AccountId, is_increase: bool) -> VoteWeight { use frame_support::traits::Get as _; let thresholds = T::BagThresholds::get(); - let node = list::Node::::get::(who).unwrap(); + let node = list::Node::::get(who).unwrap(); let current_bag_idx = thresholds .iter() .chain(sp_std::iter::once(&VoteWeight::MAX)) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 0dfcceecd4d02..89797a0955d1a 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -212,7 +212,7 @@ impl List { /// /// Full iteration can be expensive; it's recommended to limit the number of items with /// `.take(n)`. - pub(crate) fn iter() -> impl Iterator> { + pub(crate) fn iter() -> impl Iterator> { // We need a touch of special handling here: because we permit `T::BagThresholds` to // omit the final bound, we need to ensure that we explicitly include that threshold in the // list. @@ -298,7 +298,7 @@ impl List { let mut count = 0; for id in ids.into_iter() { - let node = match Node::::get::(id) { + let node = match Node::::get(id) { Some(node) => node, None => continue, }; @@ -306,7 +306,7 @@ impl List { if !node.is_terminal() { // this node is not a head or a tail and thus the bag does not need to be updated - node.excise::() + node.excise() } else { // this node is a head or tail, so the bag needs to be updated let bag = bags @@ -342,16 +342,16 @@ impl List { /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient /// to call [`self.remove_many`] followed by [`self.insert_many`]. pub(crate) fn update_position_for( - node: Node, + node: Node, new_weight: VoteWeight, ) -> Option<(VoteWeight, VoteWeight)> { - node.is_misplaced::(new_weight).then(move || { + node.is_misplaced(new_weight).then(move || { let old_bag_upper = node.bag_upper; if !node.is_terminal() { // this node is not a head or a tail, so we can just cut it out of the list. update // and put the prev and next of this node, we do `node.put` inside `insert_note`. - node.excise::(); + node.excise(); } else if let Some(mut bag) = Bag::::get(node.bag_upper) { // this is a head or tail, so the bag must be updated. bag.remove_node_unchecked(&node); @@ -517,18 +517,18 @@ impl Bag { } /// Get the head node in this bag. - fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get::(id)) + fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(id)) } /// Get the tail node in this bag. - fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get::(id)) + fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(id)) } /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next::()) + pub(crate) fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) } /// Insert a new id into this bag. @@ -542,7 +542,7 @@ impl Bag { // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long // as this bag is the correct one, we're good. All calls to this must come after getting the // correct [`notional_bag_for`]. - self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); + self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); } /// Insert a node into this bag. @@ -552,7 +552,7 @@ impl Bag { /// /// Storage note: this modifies storage, but only for the node. You still need to call /// `self.put()` after use. - fn insert_node_unchecked(&mut self, mut node: Node) { + fn insert_node_unchecked(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { // this should never happen, but this check prevents one path to a worst case @@ -596,9 +596,9 @@ impl Bag { /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. - fn remove_node_unchecked(&mut self, node: &Node) { + fn remove_node_unchecked(&mut self, node: &Node) { // reassign neighboring nodes. - node.excise::(); + node.excise(); // clear the bag head/tail pointers as necessary. if self.tail.as_ref() == Some(&node.id) { @@ -621,7 +621,7 @@ impl Bag { fn sanity_check(&self) -> Result<(), &'static str> { frame_support::ensure!( self.head() - .map(|head| head.prev::().is_none()) + .map(|head| head.prev().is_none()) // if there is no head, then there must not be a tail, meaning that the bag is // empty. .unwrap_or_else(|| self.tail.is_none()), @@ -630,7 +630,7 @@ impl Bag { frame_support::ensure!( self.tail() - .map(|tail| tail.next::().is_none()) + .map(|tail| tail.next().is_none()) // if there is no tail, then there must not be a head, meaning that the bag is // empty. .unwrap_or_else(|| self.head.is_none()), @@ -657,8 +657,8 @@ impl Bag { /// Iterate over the nodes in this bag (public for tests). #[cfg(feature = "std")] #[allow(dead_code)] - pub fn std_iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next::()) + pub fn std_iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) } /// Check if the bag contains a node with `id`. @@ -671,24 +671,23 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. #[derive(Encode, Decode, MaxEncodedLen)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] -pub struct Node { - id: AccountId, - prev: Option, - next: Option, +pub struct Node { + id: T::AccountId, + prev: Option, + next: Option, bag_upper: VoteWeight, - - // #[codec(skip)] - // _phantom: PhantomData + /* #[codec(skip)] + * _phantom: PhantomData */ } -impl Node { +impl Node { /// Get a node by id. - pub(crate) fn get(id: &AccountId) -> Option> { + pub(crate) fn get(id: &T::AccountId) -> Option> { crate::ListNodes::::try_get(id).ok() } /// Put the node back into storage. - fn put(self) { + fn put(self) { crate::ListNodes::::insert(self.id.clone(), self); } @@ -696,14 +695,14 @@ impl Node { /// /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call /// `self.put`. - fn excise(&self) { + fn excise(&self) { // Update previous node. - if let Some(mut prev) = self.prev::() { + if let Some(mut prev) = self.prev() { prev.next = self.next.clone(); prev.put(); } // Update next self. - if let Some(mut next) = self.next::() { + if let Some(mut next) = self.next() { next.prev = self.prev.clone(); next.put(); } @@ -712,22 +711,22 @@ impl Node { /// This is a naive function that removes a node from the `ListNodes` storage item. /// /// It is naive because it does not check if the node has first been removed from its bag. - fn remove_from_storage_unchecked(&self) { + fn remove_from_storage_unchecked(&self) { crate::ListNodes::::remove(&self.id) } /// Get the previous node in the bag. - fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| Node::get::(id)) + fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| Node::get(id)) } /// Get the next node in the bag. - fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| Node::get::(id)) + fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| Node::get(id)) } /// `true` when this voter is in the wrong bag. - pub(crate) fn is_misplaced(&self, current_weight: VoteWeight) -> bool { + pub(crate) fn is_misplaced(&self, current_weight: VoteWeight) -> bool { notional_bag_for::(current_weight) != self.bag_upper } @@ -737,14 +736,14 @@ impl Node { } /// Get the underlying voter. - pub(crate) fn id(&self) -> &AccountId { + pub(crate) fn id(&self) -> &T::AccountId { &self.id } /// Get the underlying voter (public fo tests). #[cfg(feature = "std")] #[allow(dead_code)] - pub fn std_id(&self) -> &AccountId { + pub fn std_id(&self) -> &T::AccountId { &self.id } @@ -756,7 +755,7 @@ impl Node { } #[cfg(feature = "std")] - fn sanity_check(&self) -> Result<(), &'static str> { + fn sanity_check(&self) -> Result<(), &'static str> { let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; let id = self.id(); From 104775681571805d9183c6440b4afe735eb8092b Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 18:01:27 -0700 Subject: [PATCH 216/241] Get MaxEncodeLen derive to work --- frame/bags-list/src/list/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 89797a0955d1a..24187a4d73181 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -469,6 +469,7 @@ impl List { /// iteration so that there's no incentive to churn ids positioning to improve the chances of /// appearing within the ids set. #[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen)] +#[codec(mel_bound(T: Config))] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] pub struct Bag { head: Option, @@ -670,14 +671,13 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. #[derive(Encode, Decode, MaxEncodedLen)] +#[codec(mel_bound(T: Config))] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] pub struct Node { id: T::AccountId, prev: Option, next: Option, bag_upper: VoteWeight, - /* #[codec(skip)] - * _phantom: PhantomData */ } impl Node { From be2038fa01624e2d67cefdbbf311f69acb277848 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 18:20:44 -0700 Subject: [PATCH 217/241] Try to correctly feature gate SortedListProvider::clear --- frame/bags-list/src/lib.rs | 2 +- frame/election-provider-support/src/lib.rs | 5 ++++- frame/staking/src/pallet/impls.rs | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 3c20a21290d21..5c0eba8f9d9e2 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -279,7 +279,7 @@ impl SortedListProvider for Pallet { Ok(()) } - // #[cfg(feature = "runtime-benchmarks")] + #[cfg(feature = "runtime-benchmarks")] fn clear() { List::::clear(None) } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 6fda658a1f79f..d4ee7bf1e4e15 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -339,7 +339,10 @@ pub trait SortedListProvider { ) -> u32; /// Remove the list and all its associated storage items. - fn clear(); + #[cfg(feature = "runtime-benchmarks")] + fn clear() { + todo!() + } /// Sanity check internal state of list. Only meant for debug compilation. fn sanity_check() -> Result<(), &'static str>; diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 83dff6a94e9ba..c6e5e3642298a 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1277,6 +1277,7 @@ impl SortedListProvider for UseNominatorsMap { fn sanity_check() -> Result<(), &'static str> { Ok(()) } + #[cfg(feature = "runtime-benchmarks")] fn clear() { Nominators::::remove_all(None); CounterForNominators::::kill(); From 080f1b6f21072b292f894bc77a7b28b853c518aa Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 18:46:50 -0700 Subject: [PATCH 218/241] Use u32::MAX, not u32::max_value --- bin/node/runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 5f66b5934939f..e6df9b468c123 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -549,7 +549,7 @@ parameter_types! { .max .get(DispatchClass::Normal); - pub const VoterSnapshotPerBlock: u32 = u32::max_value(); + pub const VoterSnapshotPerBlock: u32 = u32::MAX; } sp_npos_elections::generate_solution_type!( From 14b91205b0ea014b77340f712fd90b1817402e58 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 13 Sep 2021 19:14:07 -0700 Subject: [PATCH 219/241] Get up to nominators_quota noms --- frame/staking/src/pallet/impls.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index c6e5e3642298a..8a74c9d9c3b90 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -674,10 +674,24 @@ impl Pallet { } // .. and grab whatever we have left from nominators. - let nominators_quota = max_allowed_len.saturating_sub(validators_taken as usize); + let nominators_quota = (max_allowed_len as u32).saturating_sub(validators_taken); let slashing_spans = >::iter().collect::>(); + + // track the count of nominators added to `all_voters let mut nominators_taken = 0u32; - for nominator in T::SortedListProvider::iter().take(nominators_quota) { + // track every nominator iterated over, but not necessarily added to `all_voters` + let mut nominators_seen = 0u32; + + let mut nominators_iter = T::SortedListProvider::iter(); + while nominators_taken < nominators_quota && nominators_seen < nominators_quota * 2 { + let nominator = match nominators_iter.next() { + Some(nominator) => { + nominators_seen.saturating_inc(); + nominator + }, + None => break, + }; + if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = >::get(&nominator) { From 385138f6328c2f91ff60f53180315b27696b5e80 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 11:00:41 -0700 Subject: [PATCH 220/241] SortedListProvider::clear takes an Option --- frame/bags-list/src/lib.rs | 5 ++--- frame/bags-list/src/list/mod.rs | 15 ++++++++++----- frame/election-provider-support/src/lib.rs | 9 ++++----- frame/staking/src/pallet/impls.rs | 12 ++++++++---- frame/staking/src/testing_utils.rs | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 5c0eba8f9d9e2..87440bfac9028 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -279,9 +279,8 @@ impl SortedListProvider for Pallet { Ok(()) } - #[cfg(feature = "runtime-benchmarks")] - fn clear() { - List::::clear(None) + fn clear(maybe_count: Option) -> u32 { + List::::clear(maybe_count) } #[cfg(feature = "runtime-benchmarks")] diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 24187a4d73181..f0cfd2c31de81 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -77,13 +77,18 @@ impl List { /// Remove all data associated with the list from storage. Parameter `items` is the number of /// items to clear from the list. WARNING: `None` will clear all items and should generally not /// be used in production as it could lead to an infinite number of storage accesses. - pub(crate) fn clear(items: Option) { - crate::CounterForListNodes::::kill(); - crate::ListBags::::remove_all(items); - crate::ListNodes::::remove_all(items); + pub(crate) fn clear(maybe_count: Option) -> u32 { + crate::ListBags::::remove_all(maybe_count); + crate::ListNodes::::remove_all(maybe_count); + if let Some(count) = maybe_count { + crate::CounterForListNodes::::mutate(|items| *items - count); + count + } else { + crate::CounterForListNodes::::take() + } } - /// Regenerate all of the data from the given ids. + // Regenerate all of the data from the given ids. /// /// WARNING: this is expensive and should only ever be performed when the list needs to be /// generated from scratch. Care needs to be taken to ensure diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index d4ee7bf1e4e15..cb36e025c3bee 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -338,11 +338,10 @@ pub trait SortedListProvider { weight_of: Box VoteWeight>, ) -> u32; - /// Remove the list and all its associated storage items. - #[cfg(feature = "runtime-benchmarks")] - fn clear() { - todo!() - } + /// Remove `maybe_count` number of items from the list. Returns the number of items actually + /// removed. WARNING: removes all items if `maybe_count` is `None`, which should never be done + /// in production settings because it can lead to an unbounded amount of storage accesses. + fn clear(maybe_count: Option) -> u32; /// Sanity check internal state of list. Only meant for debug compilation. fn sanity_check() -> Result<(), &'static str>; diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 8a74c9d9c3b90..3d05f761f5cf2 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1291,9 +1291,13 @@ impl SortedListProvider for UseNominatorsMap { fn sanity_check() -> Result<(), &'static str> { Ok(()) } - #[cfg(feature = "runtime-benchmarks")] - fn clear() { - Nominators::::remove_all(None); - CounterForNominators::::kill(); + fn clear(maybe_count: Option) -> u32 { + Nominators::::remove_all(maybe_count); + if let Some(count) = maybe_count { + CounterForNominators::::mutate(|noms| *noms - count); + count + } else { + CounterForNominators::::take() + } } } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 27367048cc6a9..d3cc2221db44a 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -42,7 +42,7 @@ pub fn clear_validators_and_nominators() { // whenever we touch nominators counter we should update `T::SortedListProvider` as well. Nominators::::remove_all(None); CounterForNominators::::kill(); - T::SortedListProvider::clear(); + T::SortedListProvider::clear(None); } /// Grab a funded user. From adcf4bf5042103f3ba128f41160781162ce16aa8 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 11:03:11 -0700 Subject: [PATCH 221/241] Eplicitly ignore SortedListProvider return value --- frame/staking/src/pallet/impls.rs | 2 +- frame/staking/src/testing_utils.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 3d05f761f5cf2..85d5302270acb 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -953,7 +953,7 @@ impl ElectionDataProvider> for Pallet >::remove_all(None); >::kill(); >::kill(); - T::SortedListProvider::clear(); + let _ = T::SortedListProvider::clear(); } #[cfg(feature = "runtime-benchmarks")] diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index d3cc2221db44a..13762cf5886db 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -42,7 +42,7 @@ pub fn clear_validators_and_nominators() { // whenever we touch nominators counter we should update `T::SortedListProvider` as well. Nominators::::remove_all(None); CounterForNominators::::kill(); - T::SortedListProvider::clear(None); + let _ = T::SortedListProvider::clear(None); } /// Grab a funded user. From ce85f9118ff00ccc30d27b981f0a458c6329f241 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 11:04:40 -0700 Subject: [PATCH 222/241] Fix doc comment --- frame/bags-list/src/list/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index f0cfd2c31de81..8f923858c00ee 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -88,7 +88,7 @@ impl List { } } - // Regenerate all of the data from the given ids. + /// Regenerate all of the data from the given ids. /// /// WARNING: this is expensive and should only ever be performed when the list needs to be /// generated from scratch. Care needs to be taken to ensure From a2899e65a74ad9e137db3ea0da8448031d720662 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 11:19:27 -0700 Subject: [PATCH 223/241] Update node-runtime voter snapshot per block --- bin/node/runtime/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index e6df9b468c123..de458aa89c89d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -549,7 +549,10 @@ parameter_types! { .max .get(DispatchClass::Normal); - pub const VoterSnapshotPerBlock: u32 = u32::MAX; + // BagsList allows a practically unbounded count of nominators to participate in NPoS elections. + // To ensure we respect memory limits when using the BagsList this must be set to a number of + // voters we know can fit into a single vec allocation. + pub const VoterSnapshotPerBlock: u32 = 10_000; } sp_npos_elections::generate_solution_type!( From 2e9e31702bf751d2c009789552961d247b297f63 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 20:31:47 -0700 Subject: [PATCH 224/241] Add test get_max_len_voters_even_if_some_nominators_are_slashed --- frame/staking/src/tests.rs | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index c573dab4eec14..484e68f474415 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3905,6 +3905,77 @@ mod election_data_provider { }); } + #[test] + fn only_iterates_max_2_times_nominators_quota() { + ExtBuilder::default() + .nominate(true) // add nominator 101, who nominates [11, 21] + .add_staker(61, 60, 20, StakerStatus::::Nominator(vec![21])) + .add_staker(71, 70, 10, StakerStatus::::Nominator(vec![11, 21])) + .build_and_execute(|| {}); + } + + // Even if some of the higher staked nominators are slashed, we still get up to max len voters + // by adding more lower staked nominators. In other words, we assert that we keep on adding + // valid nominators until we reach max len voters; which is opposed to simply stopping after we + // have iterated max len voters, but not adding all of them to voters due to some nominators not + // having valid targets. + #[test] + fn get_max_len_voters_even_if_some_nominators_are_slashed() { + ExtBuilder::default() + .nominate(true) // add nominator 101, who nominates [11, 21] + .add_staker(61, 60, 20, StakerStatus::::Nominator(vec![21])) + // 61 only nominates validator 21 ^^ + .add_staker(71, 70, 10, StakerStatus::::Nominator(vec![11, 21])) + .build_and_execute(|| { + // our nominators ordered by stake + assert_eq!( + ::SortedListProvider::iter().collect::>(), + vec![101, 61, 71] + ); + + // total voters + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 6 + ); + + // we take 5 voters + assert_eq!( + Staking::voters(Some(5)) + .unwrap() + .iter() + .map(|(stash, _, _)| stash) + .copied() + .collect::>(), + vec![ + 31, 21, 11, // 3 nominators + 101, 61 // 2 validators, and 71 is excluded + ], + ); + + // roll to session 5 + run_to_block(25); + + // slash 21, the only validator nominated by 61 + add_slash(&21); + + // we take 4 voters + assert_eq!( + Staking::voters(Some(4)) + .unwrap() + .iter() + .map(|(stash, _, _)| stash) + .copied() + .collect::>(), + vec![ + 31, 11, // 2 validators (21 was slashed) + 101, 71 // 2 nominators, excluding 61 + ], + ); + }); + } + #[test] fn estimate_next_election_works() { ExtBuilder::default().session_per_era(5).period(5).build_and_execute(|| { From e6dd68bbeab047f90b270059da856f3298e6ade4 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 20:50:17 -0700 Subject: [PATCH 225/241] Add test only_iterates_max_2_times_nominators_quota --- frame/staking/src/tests.rs | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 484e68f474415..3db421f9dba73 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3909,9 +3909,41 @@ mod election_data_provider { fn only_iterates_max_2_times_nominators_quota() { ExtBuilder::default() .nominate(true) // add nominator 101, who nominates [11, 21] - .add_staker(61, 60, 20, StakerStatus::::Nominator(vec![21])) - .add_staker(71, 70, 10, StakerStatus::::Nominator(vec![11, 21])) - .build_and_execute(|| {}); + // the other nominators only nominate 21 + .add_staker(61, 60, 2_000, StakerStatus::::Nominator(vec![21])) + .add_staker(71, 70, 2_000, StakerStatus::::Nominator(vec![21])) + .add_staker(81, 80, 2_000, StakerStatus::::Nominator(vec![21])) + .build_and_execute(|| { + // given our nominators ordered by stake, + assert_eq!( + ::SortedListProvider::iter().collect::>(), + vec![61, 71, 81, 101] + ); + + // and total voters + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 7 + ); + + // roll to session 5 + run_to_block(25); + + // slash 21, the only validator nominated by our first 3 nominators + add_slash(&21); + + // we take 4 voters: 2 validators and 2 nominators (so nominators quota = 2) + assert_eq!( + Staking::voters(Some(3)) + .unwrap() + .iter() + .map(|(stash, _, _)| stash) + .copied() + .collect::>(), + vec![31, 11], // 2 validators, but no nominators because we hit the quota + ); + }); } // Even if some of the higher staked nominators are slashed, we still get up to max len voters @@ -3927,13 +3959,13 @@ mod election_data_provider { // 61 only nominates validator 21 ^^ .add_staker(71, 70, 10, StakerStatus::::Nominator(vec![11, 21])) .build_and_execute(|| { - // our nominators ordered by stake + // given our nominators ordered by stake, assert_eq!( ::SortedListProvider::iter().collect::>(), vec![101, 61, 71] ); - // total voters + // and total voters assert_eq!( ::SortedListProvider::count() + >::iter().count() as u32, @@ -3948,6 +3980,7 @@ mod election_data_provider { .map(|(stash, _, _)| stash) .copied() .collect::>(), + // then vec![ 31, 21, 11, // 3 nominators 101, 61 // 2 validators, and 71 is excluded From 76d01d84fd3b7205a131174e60a583a2a6388944 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 21:33:24 -0700 Subject: [PATCH 226/241] Fix generate bags cargo.toml --- Cargo.lock | 1 - utils/frame/generate-bags/Cargo.toml | 2 -- 2 files changed, 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e85d982eb3e8e..d8967f31a5fca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2310,7 +2310,6 @@ dependencies = [ "frame-support", "frame-system", "git2", - "node-runtime", "num-format", "pallet-staking", "sp-io", diff --git a/utils/frame/generate-bags/Cargo.toml b/utils/frame/generate-bags/Cargo.toml index ecbc09b05f7f7..384307fbec9e5 100644 --- a/utils/frame/generate-bags/Cargo.toml +++ b/utils/frame/generate-bags/Cargo.toml @@ -10,8 +10,6 @@ description = "Bag threshold generation script for pallet-bag-list" readme = "README.md" [dependencies] -node-runtime = { version = "3.0.0-dev", path = "../../../bin/node/runtime" } - # FRAME frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/support" } frame-election-provider-support = { version = "4.0.0-dev", path = "../../../frame/election-provider-support", features = ["runtime-benchmarks"] } From aa8c0cc8760dfdca753b3c7a077a417706789785 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 14 Sep 2021 22:24:15 -0700 Subject: [PATCH 227/241] use sp_std vec --- frame/bags-list/src/list/mod.rs | 1 + frame/staking/src/pallet/impls.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 8f923858c00ee..720e72e551e7a 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -33,6 +33,7 @@ use sp_std::{ collections::{btree_map::BTreeMap, btree_set::BTreeSet}, iter, marker::PhantomData, + vec::Vec }; #[derive(Debug, PartialEq, Eq)] diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 85d5302270acb..3ae520872f278 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -953,7 +953,7 @@ impl ElectionDataProvider> for Pallet >::remove_all(None); >::kill(); >::kill(); - let _ = T::SortedListProvider::clear(); + let _ = T::SortedListProvider::clear(None); } #[cfg(feature = "runtime-benchmarks")] From ca1b6dd71eebad17760a6c3d17522ae4e7f8f5ca Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 15 Sep 2021 10:44:53 -0700 Subject: [PATCH 228/241] Remove v8 migration hooks from pallet-staking --- frame/staking/src/pallet/mod.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index f9cd177d957e5..4538b83c1af38 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -633,8 +633,6 @@ pub mod pallet { fn on_runtime_upgrade() -> Weight { if StorageVersion::::get() == Releases::V6_0_0 { migrations::v7::migrate::() - } else if StorageVersion::::get() == Releases::V7_0_0 { - migrations::v8::migrate::() } else { T::DbWeight::get().reads(1) } @@ -644,17 +642,6 @@ pub mod pallet { fn pre_upgrade() -> Result<(), &'static str> { if StorageVersion::::get() == Releases::V6_0_0 { migrations::v7::pre_migrate::() - } else if StorageVersion::::get() == Releases::V7_0_0 { - migrations::v8::pre_migrate::() - } else { - Ok(()) - } - } - - #[cfg(feature = "try-runtime")] - fn post_upgrade() -> Result<(), &'static str> { - if StorageVersion::::get() == Releases::V7_0_0 { - migrations::v8::post_migrate::() } else { Ok(()) } From 0a5fa7b96ccc09a1cc2324660bfb35a6065338d7 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:33:02 -0700 Subject: [PATCH 229/241] Update npos trait --- primitives/npos-elections/src/traits.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/primitives/npos-elections/src/traits.rs b/primitives/npos-elections/src/traits.rs index 45b6fa368ae2a..597d7e648fd9b 100644 --- a/primitives/npos-elections/src/traits.rs +++ b/primitives/npos-elections/src/traits.rs @@ -10,8 +10,8 @@ // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. @@ -22,6 +22,7 @@ use crate::{ VoteWeight, }; use codec::Encode; +use scale_info::TypeInfo; use sp_arithmetic::{ traits::{Bounded, UniqueSaturatedInto}, PerThing, @@ -72,7 +73,8 @@ where + Copy + Clone + Bounded - + Encode; + + Encode + + TypeInfo; /// The target type. Needs to be an index (convert to usize). type TargetIndex: UniqueSaturatedInto @@ -82,7 +84,8 @@ where + Copy + Clone + Bounded - + Encode; + + Encode + + TypeInfo; /// The weight/accuracy type of each vote. type Accuracy: PerThing128; From 904e4a840204da2128de1bad501ee2b9b4bf83c4 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:56:46 -0700 Subject: [PATCH 230/241] Try respect line width --- frame/staking/src/benchmarking.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 61c33c15ee466..f3def7206320c 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -215,7 +215,8 @@ benchmarks! { bond { let stash = create_funded_user::("stash", USER_SEED, 100); let controller = create_funded_user::("controller", USER_SEED, 100); - let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); + let controller_lookup: ::Source + = T::Lookup::unlookup(controller.clone()); let reward_destination = RewardDestination::Staked; let amount = T::Currency::minimum_balance() * 10u32.into(); whitelist_account!(stash); @@ -240,7 +241,8 @@ benchmarks! { let stash = scenario.origin_stash1.clone(); let controller = scenario.origin_controller1.clone(); - let original_bonded: BalanceOf = Ledger::::get(&controller).map(|l| l.active).ok_or("ledger not created after")?; + let original_bonded: BalanceOf + = Ledger::::get(&controller).map(|l| l.active).ok_or("ledger not created after")?; T::Currency::deposit_into_existing(&stash, max_additional).unwrap(); From b91487d67a3f60a5536a40e5af47c731e59fdfd5 Mon Sep 17 00:00:00 2001 From: Parity Bot Date: Wed, 15 Sep 2021 21:18:07 +0000 Subject: [PATCH 231/241] cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bags_list --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bags-list/src/weights.rs --template=./.maintain/frame-weight-template.hbs --- frame/bags-list/src/weights.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index d663b01e5f3a8..95d3dfa6eb989 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_bags_list //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-09-02, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-09-15, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -52,17 +52,21 @@ pub trait WeightInfo { /// Weights for pallet_bags_list using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: BagsList ListBags (r:1 w:1) fn rebag_non_terminal() -> Weight { - (75_718_000 as Weight) + (74_175_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:0) - // Storage: BagsList ListNodes (r:4 w:4) - // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) fn rebag_terminal() -> Weight { - (75_718_000 as Weight) + (73_305_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -70,17 +74,21 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { + // Storage: Staking Bonded (r:1 w:0) + // Storage: Staking Ledger (r:1 w:0) + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: BagsList ListBags (r:1 w:1) fn rebag_non_terminal() -> Weight { - (75_718_000 as Weight) + (74_175_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Staking Bonded (r:1 w:0) // Storage: Staking Ledger (r:1 w:0) - // Storage: BagsList ListNodes (r:4 w:4) - // Storage: BagsList ListBags (r:1 w:1) + // Storage: BagsList ListNodes (r:3 w:3) + // Storage: BagsList ListBags (r:2 w:2) fn rebag_terminal() -> Weight { - (75_718_000 as Weight) + (73_305_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } From 59ed5c5ea74dff4311a951fddc08e3b00019a75b Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 15 Sep 2021 16:07:17 -0700 Subject: [PATCH 232/241] Update frame/bags-list/src/benchmarks.rs --- frame/bags-list/src/benchmarks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index db084fd31bd24..a820eeba13b12 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -32,7 +32,7 @@ frame_benchmarking::benchmarks! { // least 2 other nodes. Thus _R_ will have both its `prev` and `next` nodes updated when // it is removed. (3 W/R) // - The destination bag is not empty, thus we need to update the `next` pointer of the last - // node in the destination in addition to the work we do otherwise. (2 W/R) + // node in the destination in addition to the work we do otherwise. (2 W/R) // clear any pre-existing storage. List::::clear(None); From a78477cb59072f147f611b15e52af4e22d34d498 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 12:41:48 -0700 Subject: [PATCH 233/241] Unwrap try-runtime error; remove sortedlistprovider pre upgrade len check --- frame/executive/src/lib.rs | 4 ++-- frame/staking/src/migrations.rs | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 655a38fe1b540..9a0fce4d6b5b4 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -229,7 +229,7 @@ where (frame_system::Pallet::, COnRuntimeUpgrade, AllPallets) as OnRuntimeUpgrade - >::pre_upgrade()?; + >::pre_upgrade().unwrap(); let weight = Self::execute_on_runtime_upgrade(); @@ -237,7 +237,7 @@ where (frame_system::Pallet::, COnRuntimeUpgrade, AllPallets) as OnRuntimeUpgrade - >::post_upgrade()?; + >::post_upgrade().unwrap(); Ok(weight) } diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 3abfcebabcea3..7064f06dd12c7 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -30,10 +30,7 @@ pub mod v8 { StorageVersion::::get() == crate::Releases::V7_0_0, "must upgrade linearly" ); - frame_support::ensure!( - T::SortedListProvider::iter().count() == 0, - "voter list already exists" - ); + crate::log!(info, "👜 staking bags-list migration passes PRE migrate checks ✅",); Ok(()) } From ffa3a93dfe8c40fe3e3b2a8898fb5c3848a74aa5 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 14:33:42 -0700 Subject: [PATCH 234/241] trigger ci --- frame/staking/src/migrations.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 7064f06dd12c7..0c89a0a0bdc99 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -52,6 +52,7 @@ pub mod v8 { "👜 completed staking migration to Releases::V8_0_0 with {} voters migrated", migrated, ); + T::BlockWeights::get().max_block } else { From ff0f48ceeb5a80e2cfcadff0df4c7ee2ff4aae08 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 14:34:19 -0700 Subject: [PATCH 235/241] restore --- frame/staking/src/migrations.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 0c89a0a0bdc99..7064f06dd12c7 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -52,7 +52,6 @@ pub mod v8 { "👜 completed staking migration to Releases::V8_0_0 with {} voters migrated", migrated, ); - T::BlockWeights::get().max_block } else { From e162c0f33f246ba360e9a3583bd628dc329d0ff4 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 15:00:17 -0700 Subject: [PATCH 236/241] trigger ci --- frame/staking/src/migrations.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 7064f06dd12c7..f5af86acc4ee2 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -53,6 +53,7 @@ pub mod v8 { migrated, ); + T::BlockWeights::get().max_block } else { T::DbWeight::get().reads(1) From 43a57f96888fa79293c8be86431b4e9cda55b840 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 15:00:37 -0700 Subject: [PATCH 237/241] restore --- frame/staking/src/migrations.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index f5af86acc4ee2..7064f06dd12c7 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -53,7 +53,6 @@ pub mod v8 { migrated, ); - T::BlockWeights::get().max_block } else { T::DbWeight::get().reads(1) From d96f2c24c5efd24428b33dbdd0950e079b29be87 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 17:46:19 -0700 Subject: [PATCH 238/241] trigger ci --- frame/staking/src/migrations.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 7064f06dd12c7..f5af86acc4ee2 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -53,6 +53,7 @@ pub mod v8 { migrated, ); + T::BlockWeights::get().max_block } else { T::DbWeight::get().reads(1) From 9c2c6d0370d8ca31e3fb8d793901570a3ca596d1 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 17:47:17 -0700 Subject: [PATCH 239/241] revert --- frame/staking/src/migrations.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index f5af86acc4ee2..7064f06dd12c7 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -53,7 +53,6 @@ pub mod v8 { migrated, ); - T::BlockWeights::get().max_block } else { T::DbWeight::get().reads(1) From a645fd53214c259c84d2be75af0fe705ca89914d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 18:30:49 -0700 Subject: [PATCH 240/241] trigger ci --- frame/staking/src/migrations.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 7064f06dd12c7..562f54e90ae61 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -54,6 +54,7 @@ pub mod v8 { ); T::BlockWeights::get().max_block + } else { T::DbWeight::get().reads(1) } From 800223484a68a347e1ff9389dbf13018d82f004c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 16 Sep 2021 18:31:21 -0700 Subject: [PATCH 241/241] revert --- frame/staking/src/migrations.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 562f54e90ae61..7064f06dd12c7 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -54,7 +54,6 @@ pub mod v8 { ); T::BlockWeights::get().max_block - } else { T::DbWeight::get().reads(1) }