From 76207f5373e5353d6eda09fae00eb44dbbfb8b4b Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 26 May 2021 20:46:49 +0200 Subject: [PATCH 01/10] make remote ext use batch ws-client --- Cargo.lock | 2 + primitives/storage/src/lib.rs | 8 ++-- utils/frame/remote-externalities/Cargo.toml | 2 + utils/frame/remote-externalities/src/lib.rs | 51 ++++++++++++++++----- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5115bf4d42b87..51d296bea2f44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6669,6 +6669,8 @@ dependencies = [ "jsonrpsee-ws-client", "log", "parity-scale-codec", + "serde", + "serde_json", "sp-core", "sp-io", "sp-runtime", diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index ced8d8c02a80b..76557d64753b3 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -107,10 +107,12 @@ impl PrefixedStorageKey { /// Storage data associated to a [`StorageKey`]. #[derive(PartialEq, Eq, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode))] +#[cfg_attr( + feature = "std", + derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone, Encode, Decode, Default) +)] pub struct StorageData( - #[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))] - pub Vec, + #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, ); /// Map of data to use in a storage, it is a collection of diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 0d6336f60d88f..3eb037f94a53d 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -21,6 +21,8 @@ env_logger = "0.8.2" log = "0.4.11" codec = { package = "parity-scale-codec", version = "2.0.0" } +serde_json = "1.0" + sp-io = { version = "3.0.0", path = "../../../primitives/io" } sp-core = { version = "3.0.0", path = "../../../primitives/core" } sp-runtime = { version = "3.0.0", path = "../../../primitives/runtime" } diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 077892baabf71..a89bc28729dfd 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -114,12 +114,13 @@ use sp_core::{ }; use codec::{Encode, Decode}; use sp_runtime::traits::Block as BlockT; -use jsonrpsee_ws_client::{WsClientBuilder, WsClient}; +use jsonrpsee_ws_client::{WsClientBuilder, WsClient, v2::params::JsonRpcParams, traits::Client}; type KeyPair = (StorageKey, StorageData); const LOG_TARGET: &str = "remote-ext"; const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io"; +const BATCH_SIZE: usize = 512; jsonrpsee_proc_macros::rpc_client_api! { RpcApi { @@ -333,16 +334,28 @@ impl Builder { info!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); let mut key_values: Vec = vec![]; - for key in keys { - let value = - RpcApi::::get_storage(self.as_online().rpc_client(), key.clone(), Some(at)) - .await - .map_err(|e| { - error!(target: LOG_TARGET, "Error = {:?}", e); - "rpc get_storage failed" - })?; - key_values.push((key, value)); - if key_values.len() % 1000 == 0 { + let client = self.as_online().rpc_client(); + for chunk_keys in keys.chunks(BATCH_SIZE) { + let batch = chunk_keys + .iter() + .cloned() + .map(|key| { + assert!(key.0.len() == 32); + ( + "state_getStorage", + JsonRpcParams::Array(vec![serde_json::to_value(key).unwrap()]), + ) + }) + .collect::>(); + log::trace!(target: LOG_TARGET, "sending batch: {:?}", batch); + let values = client.batch_request::(batch).await.unwrap(); + assert_eq!(chunk_keys.len(), values.len()); + for (idx, key) in chunk_keys.into_iter().enumerate() { + let value = values[idx].clone(); + key_values.push((key.clone(), value)); + } + + if key_values.len() % (10 * BATCH_SIZE) == 0 { let ratio: f64 = key_values.len() as f64 / keys_count as f64; debug!( target: LOG_TARGET, @@ -529,7 +542,21 @@ mod remote_tests { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - modules: vec!["Proxy".to_owned()], + modules: vec!["System".to_owned()], + ..Default::default() + })) + .build() + .await + .expect("Can't reach the remote node. Is it running?") + .execute_with(|| {}); + } + + #[tokio::test] + async fn can_build_few_pallet() { + init_logger(); + Builder::::new() + .mode(Mode::Online(OnlineConfig { + modules: vec!["Proxy".to_owned(), "Multisig".to_owned(), "Balances".to_owned()], ..Default::default() })) .build() From 874d0a6c69008ba8c60d32f8b60a326e48f18642 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 27 May 2021 10:52:28 -0700 Subject: [PATCH 02/10] Add debug log for key length --- Cargo.lock | 1 - utils/frame/remote-externalities/src/lib.rs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 51d296bea2f44..7dd94295cafc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6669,7 +6669,6 @@ dependencies = [ "jsonrpsee-ws-client", "log", "parity-scale-codec", - "serde", "serde_json", "sp-core", "sp-io", diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index a89bc28729dfd..ca7bb346d247a 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -340,6 +340,7 @@ impl Builder { .iter() .cloned() .map(|key| { + log::debug!("key.0.len() == {}", key.0.len()); assert!(key.0.len() == 32); ( "state_getStorage", From 9463c4777452a177b54056fd89fdb0200d59affa Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 09:02:42 +0200 Subject: [PATCH 03/10] better assertions --- Cargo.lock | 1 - utils/frame/remote-externalities/src/lib.rs | 14 ++++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51d296bea2f44..7dd94295cafc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6669,7 +6669,6 @@ dependencies = [ "jsonrpsee-ws-client", "log", "parity-scale-codec", - "serde", "serde_json", "sp-core", "sp-io", diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index a89bc28729dfd..a49b904e590b5 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -329,6 +329,7 @@ impl Builder { prefix: StorageKey, at: B::Hash, ) -> Result, &'static str> { + use serde_json::to_value; let keys = self.get_keys_paged(prefix, at).await?; let keys_count = keys.len(); info!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); @@ -340,15 +341,20 @@ impl Builder { .iter() .cloned() .map(|key| { - assert!(key.0.len() == 32); ( "state_getStorage", - JsonRpcParams::Array(vec![serde_json::to_value(key).unwrap()]), + JsonRpcParams::Array( + vec![to_value(key).expect("json serialization will work; qed.")] + ), ) }) .collect::>(); - log::trace!(target: LOG_TARGET, "sending batch: {:?}", batch); - let values = client.batch_request::(batch).await.unwrap(); + let values = client.batch_request::(batch) + .await + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to execute batch {:?} due to {:?}", chunk_keys, e); + "batch failed." + })?; assert_eq!(chunk_keys.len(), values.len()); for (idx, key) in chunk_keys.into_iter().enumerate() { let value = values[idx].clone(); From 48dad42e422b916fc8b002f32f16c5eb47bd103f Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 09:47:53 +0200 Subject: [PATCH 04/10] new sanity_checl --- Cargo.lock | 3 ++ utils/frame/remote-externalities/Cargo.toml | 2 ++ utils/frame/remote-externalities/src/lib.rs | 35 +++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 7dd94295cafc0..9b9a87b3fa07c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6664,10 +6664,13 @@ name = "remote-externalities" version = "0.9.0" dependencies = [ "env_logger 0.8.3", + "frame-support", + "frame-system", "hex-literal", "jsonrpsee-proc-macros", "jsonrpsee-ws-client", "log", + "pallet-elections-phragmen", "parity-scale-codec", "serde_json", "sp-core", diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 3eb037f94a53d..53acab715939b 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -29,6 +29,8 @@ sp-runtime = { version = "3.0.0", path = "../../../primitives/runtime" } [dev-dependencies] tokio = { version = "1.6.0", features = ["macros", "rt"] } +pallet-elections-phragmen = { path = "../../../frame/elections-phragmen", version = "4.0.0" } +frame-support = { path = "../../../frame/support", version = "3.0.0" } [features] remote-test = [] diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index a49b904e590b5..96728d0f872c4 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -571,6 +571,41 @@ mod remote_tests { .execute_with(|| {}); } + #[tokio::test] + async fn sanity_check_decoding() { + use pallet_elections_phragmen::SeatHolder; + use sp_core::crypto::Ss58Codec; + type AccountId = sp_runtime::AccountId32; + type Balance = u128; + frame_support::generate_storage_alias!( + PhragmenElection, + Members => + Value>> + ); + + init_logger(); + Builder::::new() + .mode(Mode::Online(OnlineConfig { + modules: vec!["PhragmenElection".to_owned()], + ..Default::default() + })) + .build() + .await + .expect("Can't reach the remote node. Is it running?") + .execute_with(|| { + // Gav's polkadot account. 99% this will be in the council. + let gav_polkadot = + AccountId::from_ss58check("13RDY9nrJpyTDBSUdBw12dGwhk19sGwsrVZ2bxkzYHBSagP2") + .unwrap(); + let members = Members::get().unwrap(); + assert!(members + .iter() + .map(|s| s.who.clone()) + .find(|a| a == &gav_polkadot) + .is_some()); + }); + } + #[tokio::test] async fn can_create_state_snapshot() { init_logger(); From c129e00228d7db6a7af88febe160dc2e1baeace9 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 10:22:56 +0200 Subject: [PATCH 05/10] try and make it work with batch --- Cargo.lock | 3 +-- utils/frame/remote-externalities/Cargo.toml | 2 +- utils/frame/remote-externalities/src/lib.rs | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b9a87b3fa07c..cde36c7d4003f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6665,8 +6665,7 @@ version = "0.9.0" dependencies = [ "env_logger 0.8.3", "frame-support", - "frame-system", - "hex-literal", + "hex", "jsonrpsee-proc-macros", "jsonrpsee-ws-client", "log", diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 53acab715939b..4fe0cf979c1b4 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] jsonrpsee-ws-client = { version = "=0.2.0-alpha.6", default-features = false } jsonrpsee-proc-macros = "=0.2.0-alpha.6" -hex-literal = "0.3.1" +hex = "0.4.0" env_logger = "0.8.2" log = "0.4.11" codec = { package = "parity-scale-codec", version = "2.0.0" } diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 96728d0f872c4..77d9e4333877a 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -114,12 +114,15 @@ use sp_core::{ }; use codec::{Encode, Decode}; use sp_runtime::traits::Block as BlockT; -use jsonrpsee_ws_client::{WsClientBuilder, WsClient, v2::params::JsonRpcParams, traits::Client}; +use jsonrpsee_ws_client::{ + WsClientBuilder, WsClient, JsonValue, v2::params::JsonRpcParams, traits::Client, +}; type KeyPair = (StorageKey, StorageData); const LOG_TARGET: &str = "remote-ext"; const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io"; +// const DEFAULT_TARGET: &str = "ws://127.0.0.1:9999"; const BATCH_SIZE: usize = 512; jsonrpsee_proc_macros::rpc_client_api! { @@ -349,7 +352,7 @@ impl Builder { ) }) .collect::>(); - let values = client.batch_request::(batch) + let values = client.batch_request::(batch) .await .map_err(|e| { log::error!(target: LOG_TARGET, "failed to execute batch {:?} due to {:?}", chunk_keys, e); @@ -358,6 +361,19 @@ impl Builder { assert_eq!(chunk_keys.len(), values.len()); for (idx, key) in chunk_keys.into_iter().enumerate() { let value = values[idx].clone(); + let value = match value { + JsonValue::String(s) => { + StorageData(hex::decode(&s[2..]).map_err(|e| { + println!("Error in {:?} => {:?}", s, e); + e + }).expect("failed to decode non-empty hex.")) + }, + JsonValue::Null => { + log::warn!(target: LOG_TARGET, "key {:?} had null corresponding value.", key); + StorageData(vec![]) + } + _ => panic!("expected response {:?}", value), + }; key_values.push((key.clone(), value)); } From 07cd7b42ff6f27bf7c25f87bdc0742dd25e0daa8 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 10:26:37 +0200 Subject: [PATCH 06/10] update test --- utils/frame/remote-externalities/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 77d9e4333877a..6c932531e722c 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -578,7 +578,7 @@ mod remote_tests { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - modules: vec!["Proxy".to_owned(), "Multisig".to_owned(), "Balances".to_owned()], + modules: vec!["Proxy".to_owned(), "Multisig".to_owned(), "PhragmenElection".to_owned()], ..Default::default() })) .build() From 6ee383ac4399d243e07583bd3a66286be62c984e Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 10:33:58 +0200 Subject: [PATCH 07/10] remove exctra uri --- utils/frame/remote-externalities/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 6c932531e722c..6522573842bd2 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -122,7 +122,6 @@ type KeyPair = (StorageKey, StorageData); const LOG_TARGET: &str = "remote-ext"; const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io"; -// const DEFAULT_TARGET: &str = "ws://127.0.0.1:9999"; const BATCH_SIZE: usize = 512; jsonrpsee_proc_macros::rpc_client_api! { From a02425dd89d973042e396d4a7844053b039a53ff Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 10:47:15 +0200 Subject: [PATCH 08/10] add missing at --- utils/frame/remote-externalities/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 6522573842bd2..c1a65ad5cf4ae 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -282,7 +282,7 @@ impl Builder { async fn get_keys_paged( &self, prefix: StorageKey, - hash: B::Hash, + at: B::Hash, ) -> Result, &'static str> { const PAGE: u32 = 512; let mut last_key: Option = None; @@ -293,7 +293,7 @@ impl Builder { Some(prefix.clone()), PAGE, last_key.clone(), - Some(hash), + Some(at), ) .await .map_err(|e| { @@ -346,7 +346,10 @@ impl Builder { ( "state_getStorage", JsonRpcParams::Array( - vec![to_value(key).expect("json serialization will work; qed.")] + vec![ + to_value(key).expect("json serialization will work; qed."), + to_value(at).expect("json serialization will work; qed."), + ] ), ) }) From 305bcc463f213e6f74b596702dca963909907a05 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 11:07:02 +0200 Subject: [PATCH 09/10] remove unused rpc stuff --- utils/frame/remote-externalities/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index c1a65ad5cf4ae..aa59fce510cb0 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -126,8 +126,6 @@ const BATCH_SIZE: usize = 512; jsonrpsee_proc_macros::rpc_client_api! { RpcApi { - #[rpc(method = "state_getStorage", positional_params)] - fn get_storage(prefix: StorageKey, hash: Option) -> StorageData; #[rpc(method = "state_getKeysPaged", positional_params)] fn get_keys_paged( prefix: Option, From d6f66ac74a3af4d7ec8f2b6cb802abf7b6c2ce90 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 28 May 2021 15:08:39 +0200 Subject: [PATCH 10/10] improve --- utils/frame/remote-externalities/src/lib.rs | 44 ++++++++------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index aa59fce510cb0..46aa583b9b2a5 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -115,7 +115,7 @@ use sp_core::{ use codec::{Encode, Decode}; use sp_runtime::traits::Block as BlockT; use jsonrpsee_ws_client::{ - WsClientBuilder, WsClient, JsonValue, v2::params::JsonRpcParams, traits::Client, + WsClientBuilder, WsClient, v2::params::JsonRpcParams, traits::Client, }; type KeyPair = (StorageKey, StorageData); @@ -352,7 +352,7 @@ impl Builder { ) }) .collect::>(); - let values = client.batch_request::(batch) + let values = client.batch_request::>(batch) .await .map_err(|e| { log::error!(target: LOG_TARGET, "failed to execute batch {:?} due to {:?}", chunk_keys, e); @@ -360,32 +360,22 @@ impl Builder { })?; assert_eq!(chunk_keys.len(), values.len()); for (idx, key) in chunk_keys.into_iter().enumerate() { - let value = values[idx].clone(); - let value = match value { - JsonValue::String(s) => { - StorageData(hex::decode(&s[2..]).map_err(|e| { - println!("Error in {:?} => {:?}", s, e); - e - }).expect("failed to decode non-empty hex.")) - }, - JsonValue::Null => { - log::warn!(target: LOG_TARGET, "key {:?} had null corresponding value.", key); - StorageData(vec![]) - } - _ => panic!("expected response {:?}", value), - }; + let maybe_value = values[idx].clone(); + let value = maybe_value.unwrap_or_else(|| { + log::warn!(target: LOG_TARGET, "key {:?} had none corresponding value.", &key); + StorageData(vec![]) + }); key_values.push((key.clone(), value)); - } - - if key_values.len() % (10 * BATCH_SIZE) == 0 { - let ratio: f64 = key_values.len() as f64 / keys_count as f64; - debug!( - target: LOG_TARGET, - "progress = {:.2} [{} / {}]", - ratio, - key_values.len(), - keys_count, - ); + if key_values.len() % (10 * BATCH_SIZE) == 0 { + let ratio: f64 = key_values.len() as f64 / keys_count as f64; + debug!( + target: LOG_TARGET, + "progress = {:.2} [{} / {}]", + ratio, + key_values.len(), + keys_count, + ); + } } }