From 76207f5373e5353d6eda09fae00eb44dbbfb8b4b Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 26 May 2021 20:46:49 +0200 Subject: [PATCH 1/2] make remote ext use batch ws-client --- Cargo.lock | 2 + primitives/storage/src/lib.rs | 8 ++-- utils/frame/remote-externalities/Cargo.toml | 2 + utils/frame/remote-externalities/src/lib.rs | 51 ++++++++++++++++----- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5115bf4d42b87..51d296bea2f44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6669,6 +6669,8 @@ dependencies = [ "jsonrpsee-ws-client", "log", "parity-scale-codec", + "serde", + "serde_json", "sp-core", "sp-io", "sp-runtime", diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index ced8d8c02a80b..76557d64753b3 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -107,10 +107,12 @@ impl PrefixedStorageKey { /// Storage data associated to a [`StorageKey`]. #[derive(PartialEq, Eq, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode))] +#[cfg_attr( + feature = "std", + derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode, Default) +)] pub struct StorageData( - #[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))] - pub Vec, + #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, ); /// Map of data to use in a storage, it is a collection of diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 0d6336f60d88f..3eb037f94a53d 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -21,6 +21,8 @@ env_logger = "0.8.2" log = "0.4.11" codec = { package = "parity-scale-codec", version = "2.0.0" } +serde_json = "1.0" + sp-io = { version = "3.0.0", path = "../../../primitives/io" } sp-core = { version = "3.0.0", path = "../../../primitives/core" } sp-runtime = { version = "3.0.0", path = "../../../primitives/runtime" } diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 077892baabf71..a89bc28729dfd 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -114,12 +114,13 @@ use sp_core::{ }; use codec::{Encode, Decode}; use sp_runtime::traits::Block as BlockT; -use jsonrpsee_ws_client::{WsClientBuilder, WsClient}; +use jsonrpsee_ws_client::{WsClientBuilder, WsClient, v2::params::JsonRpcParams, traits::Client}; type KeyPair = (StorageKey, StorageData); const LOG_TARGET: &str = "remote-ext"; const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io"; +const BATCH_SIZE: usize = 512; jsonrpsee_proc_macros::rpc_client_api! { RpcApi { @@ -333,16 +334,28 @@ impl Builder { info!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); let mut key_values: Vec = vec![]; - for key in keys { - let value = - RpcApi::::get_storage(self.as_online().rpc_client(), key.clone(), Some(at)) - .await - .map_err(|e| { - error!(target: LOG_TARGET, "Error = {:?}", e); - "rpc get_storage failed" - })?; - key_values.push((key, value)); - if key_values.len() % 1000 == 0 { + let client = self.as_online().rpc_client(); + for chunk_keys in keys.chunks(BATCH_SIZE) { + let batch = chunk_keys + .iter() + .cloned() + .map(|key| { + assert!(key.0.len() == 32); + ( + "state_getStorage", + JsonRpcParams::Array(vec![serde_json::to_value(key).unwrap()]), + ) + }) + .collect::>(); + log::trace!(target: LOG_TARGET, "sending batch: {:?}", batch); + let values = client.batch_request::(batch).await.unwrap(); + assert_eq!(chunk_keys.len(), values.len()); + for (idx, key) in chunk_keys.into_iter().enumerate() { + let value = values[idx].clone(); + key_values.push((key.clone(), value)); + } + + if key_values.len() % (10 * BATCH_SIZE) == 0 { let ratio: f64 = key_values.len() as f64 / keys_count as f64; debug!( target: LOG_TARGET, @@ -529,7 +542,21 @@ mod remote_tests { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - modules: vec!["Proxy".to_owned()], + modules: vec!["System".to_owned()], + ..Default::default() + })) + .build() + .await + .expect("Can't reach the remote node. Is it running?") + .execute_with(|| {}); + } + + #[tokio::test] + async fn can_build_few_pallet() { + init_logger(); + Builder::::new() + .mode(Mode::Online(OnlineConfig { + modules: vec!["Proxy".to_owned(), "Multisig".to_owned(), "Balances".to_owned()], ..Default::default() })) .build() From 59b2fac1bbbeb5775f5c262c9f5327d43ca38894 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 26 May 2021 21:06:22 +0200 Subject: [PATCH 2/2] emergency mode: allow governance origin to provide solution to election in case of failure --- Cargo.lock | 1 - .../election-provider-multi-phase/src/lib.rs | 52 +++++++++++++++---- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51d296bea2f44..7dd94295cafc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6669,7 +6669,6 @@ dependencies = [ "jsonrpsee-ws-client", "log", "parity-scale-codec", - "serde", "serde_json", "sp-core", "sp-io", diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 9bec5cc4bd310..3220bbbefcfb3 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -311,9 +311,13 @@ pub enum Phase { /// advising validators not to bother running the unsigned offchain worker. /// /// As validator nodes are free to edit their OCW code, they could simply ignore this advisory - /// and always compute their own solution. However, by default, when the unsigned phase is passive, - /// the offchain workers will not bother running. + /// and always compute their own solution. However, by default, when the unsigned phase is + /// passive, the offchain workers will not bother running. Unsigned((bool, Bn)), + /// The emergency phase. This is enabled upon a failing call to `T::ElectionProvider::elect`. + /// After that, the only way to leave this phase is through a successful + /// `T::ElectionProvider::elect`. + Emergency, } impl Default for Phase { @@ -323,6 +327,11 @@ impl Default for Phase { } impl Phase { + /// Whether the phase is emergency or not. + pub fn is_emergency(&self) -> bool { + matches!(self, Phase::Emergency) + } + /// Whether the phase is signed or not. pub fn is_signed(&self) -> bool { matches!(self, Phase::Signed) @@ -581,7 +590,7 @@ pub mod pallet { /// Configuration for the fallback type Fallback: Get; - /// Origin that can set the minimum score. + /// Origin that can control this pallet. type ForceOrigin: EnsureOrigin; /// The configuration of benchmarking. @@ -793,6 +802,17 @@ pub mod pallet { >::set(maybe_next_score); Ok(()) } + + #[pallet::weight(T::DbWeight::get().reads_writes(1, 1))] + fn set_emergency_election_result( + origin: OriginFor, + solution: ReadySolution, + ) -> DispatchResult { + T::ForceOrigin::ensure_origin(origin)?; + ensure!(Self::current_phase().is_emergency(), >::CallNotAllowed); + >::put(solution); + Ok(()) + } } #[pallet::event] @@ -828,6 +848,8 @@ pub mod pallet { PreDispatchWeakSubmission, /// OCW submitted solution for wrong round OcwCallWrongEra, + /// The call is now allowed at this point. + CallNotAllowed, } #[pallet::origin] @@ -1162,14 +1184,14 @@ impl Pallet { /// 1. Increment round. /// 2. Change phase to [`Phase::Off`] /// 3. Clear all snapshot data. - fn post_elect() { - // inc round + fn rotate_round() { + // inc round. >::mutate(|r| *r = *r + 1); - // change phase + // phase is off now. >::put(Phase::Off); - // kill snapshots + // kill snapshots. Self::kill_snapshot(); } @@ -1219,10 +1241,18 @@ impl ElectionProvider for Pallet { type DataProvider = T::DataProvider; fn elect() -> Result<(Supports, Weight), Self::Error> { - let outcome_and_weight = Self::do_elect(); - // IMPORTANT: regardless of if election was `Ok` or `Err`, we shall do some cleanup. - Self::post_elect(); - outcome_and_weight + match Self::do_elect() { + Ok((supports, weight)) => { + // all went okay, put sign to be Off, clean snapshot, etc. + Self::rotate_round(); + Ok((supports, weight)) + }, + Err(why) => { + log!(error, "Entering emergency mode."); + >::put(Phase::Emergency); + Err(why) + } + } } }