From 15851801d681f1be7356cd069ead44ab611cb957 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 22 Oct 2019 13:25:44 +0200 Subject: [PATCH 1/7] drafting a proof extraction at the hashdb level (to include everything for the 'call' proof case). --- core/client/src/call_executor.rs | 8 +- core/client/src/client.rs | 4 +- core/client/src/light/call_executor.rs | 6 +- core/client/src/runtime_api.rs | 8 +- core/sr-api-macros/src/decl_runtime_apis.rs | 2 +- core/sr-api-macros/src/impl_runtime_apis.rs | 12 +- core/state-machine/src/lib.rs | 9 + .../state-machine/src/proving_backend_full.rs | 324 ++++++++++++++++++ 8 files changed, 356 insertions(+), 17 deletions(-) create mode 100644 core/state-machine/src/proving_backend_full.rs diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index 05a2a8eba1c3e..0270e787926b9 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -30,7 +30,7 @@ use primitives::{ traits::{CodeExecutor, KeystoreExt}, }; -use crate::runtime_api::{ProofRecorder, InitializeBlock}; +use crate::runtime_api::{FullProofRecorder, InitializeBlock}; use crate::backend; use crate::error; @@ -81,7 +81,7 @@ where execution_manager: ExecutionManager, native_call: Option, side_effects_handler: Option, - proof_recorder: &Option>>>, + proof_recorder: &Option>, enable_keystore: bool, ) -> error::Result> where ExecutionManager: Clone; @@ -241,7 +241,7 @@ where execution_manager: ExecutionManager, native_call: Option, side_effects_handler: Option, - recorder: &Option>>>, + recorder: &Option>, enable_keystore: bool, ) -> Result, error::Error> where ExecutionManager: Clone { match initialize_block { @@ -269,7 +269,7 @@ where as Box )?; - let backend = state_machine::ProvingBackend::new_with_recorder( + let backend = state_machine::full_proving_backend::ProvingBackend::new_with_recorder( trie_state, recorder.clone() ); diff --git a/core/client/src/client.rs b/core/client/src/client.rs index d853d851c546c..faddff6e7b455 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -55,7 +55,7 @@ use header_metadata::{HeaderMetadata, CachedHeaderMetadata}; use crate::{ runtime_api::{ - CallRuntimeAt, ConstructRuntimeApi, Core as CoreApi, ProofRecorder, + CallRuntimeAt, ConstructRuntimeApi, Core as CoreApi, FullProofRecorder, InitializeBlock, }, backend::{ @@ -1453,7 +1453,7 @@ impl CallRuntimeAt for Client where initialize_block: InitializeBlock<'a, Block>, native_call: Option, context: ExecutionContext, - recorder: &Option>>>, + recorder: &Option>, ) -> error::Result> { let manager = match context { ExecutionContext::BlockConstruction => diff --git a/core/client/src/light/call_executor.rs b/core/client/src/light/call_executor.rs index 65776bcfe08f3..da492f5be1e43 100644 --- a/core/client/src/light/call_executor.rs +++ b/core/client/src/light/call_executor.rs @@ -35,7 +35,7 @@ use state_machine::{ }; use hash_db::Hasher; -use crate::runtime_api::{ProofRecorder, InitializeBlock}; +use crate::runtime_api::{FullProofRecorder, InitializeBlock}; use crate::backend::RemoteBackend; use crate::call_executor::CallExecutor; use crate::error::{Error as ClientError, Result as ClientResult}; @@ -109,7 +109,7 @@ impl CallExecutor for _manager: ExecutionManager, native_call: Option, side_effects_handler: Option, - recorder: &Option>>>, + recorder: &Option>, enable_keystore: bool, ) -> ClientResult> where ExecutionManager: Clone { // there's no actual way/need to specify native/wasm execution strategy on light node @@ -339,7 +339,7 @@ mod tests { _execution_manager: ExecutionManager, _native_call: Option, _side_effects_handler: Option, - _proof_recorder: &Option>>>, + _proof_recorder: &Option>, _enable_keystore: bool, ) -> ClientResult> where ExecutionManager: Clone { unreachable!() diff --git a/core/client/src/runtime_api.rs b/core/client/src/runtime_api.rs index a5700951e9c4c..48dec99479ffe 100644 --- a/core/client/src/runtime_api.rs +++ b/core/client/src/runtime_api.rs @@ -55,6 +55,12 @@ pub type ProofRecorder = state_machine::ProofRecorder< <<<::Header as HeaderT>::Hashing as HashT>::Hasher as HasherT>::Out >; +#[cfg(feature = "std")] +/// A type that records all accessed trie nodes and generates a proof out of it. +pub type FullProofRecorder = state_machine::full_proving_backend::ProofRecorder< + <<::Header as HeaderT>::Hashing as HashT>::Hasher +>; + /// Something that can be constructed to a runtime api. #[cfg(feature = "std")] pub trait ConstructRuntimeApi> { @@ -148,7 +154,7 @@ pub trait CallRuntimeAt { initialize_block: InitializeBlock<'a, Block>, native_call: Option, context: ExecutionContext, - recorder: &Option>>>, + recorder: &Option>, ) -> error::Result>; /// Returns the runtime version at the given block. diff --git a/core/sr-api-macros/src/decl_runtime_apis.rs b/core/sr-api-macros/src/decl_runtime_apis.rs index 12639bd1c1f17..a847f75b110e8 100644 --- a/core/sr-api-macros/src/decl_runtime_apis.rs +++ b/core/sr-api-macros/src/decl_runtime_apis.rs @@ -411,7 +411,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { initialized_block: &std::cell::RefCell>>, native_call: Option, context: #crate_::runtime_api::ExecutionContext, - recorder: &Option>>>, + recorder: &Option<#crate_::runtime_api::FullProofRecorder>, ) -> #crate_::error::Result<#crate_::runtime_api::NativeOrEncoded> { let version = call_runtime_at.runtime_version_at(at)?; use #crate_::runtime_api::InitializeBlock; diff --git a/core/sr-api-macros/src/impl_runtime_apis.rs b/core/sr-api-macros/src/impl_runtime_apis.rs index fb154aa112358..40a3acbafb30e 100644 --- a/core/sr-api-macros/src/impl_runtime_apis.rs +++ b/core/sr-api-macros/src/impl_runtime_apis.rs @@ -267,7 +267,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result, initialized_block: std::cell::RefCell>, changes: std::cell::RefCell<#crate_::runtime_api::OverlayedChanges>, - recorder: Option>>>, + recorder: Option<#crate_::runtime_api::FullProofRecorder<#block>>, } // `RuntimeApi` itself is not threadsafe. However, an instance is only available in a @@ -310,10 +310,10 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result Result, &std::cell::RefCell>>, - &Option>>>, + &Option<#crate_::runtime_api::FullProofRecorder<#block>>, ) -> #crate_::error::Result<#crate_::runtime_api::NativeOrEncoded>, >( &self, diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index c3092367f0646..1bc2adeae7311 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -37,6 +37,7 @@ mod testing; mod basic; mod overlayed_changes; mod proving_backend; +mod proving_backend_full; mod trie_backend; mod trie_backend_essence; @@ -62,6 +63,14 @@ pub use proving_backend::{ create_proof_check_backend, create_proof_check_backend_storage, Recorder as ProofRecorder, ProvingBackend, }; +pub mod full_proving_backend { + pub use crate::proving_backend_full::{ + ProofRecorder, ProvingBackend, + }; + pub use crate::proving_backend::{ + create_proof_check_backend, create_proof_check_backend_storage, + }; +} pub use trie_backend_essence::{TrieBackendStorage, Storage}; pub use trie_backend::TrieBackend; pub use error::{Error, ExecutionError}; diff --git a/core/state-machine/src/proving_backend_full.rs b/core/state-machine/src/proving_backend_full.rs new file mode 100644 index 0000000000000..a029cbdcf8fd9 --- /dev/null +++ b/core/state-machine/src/proving_backend_full.rs @@ -0,0 +1,324 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Proving state machine backend. + +use std::sync::Arc; +use parking_lot::RwLock; +use std::{cell::RefCell, rc::Rc}; +use log::debug; +use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix, HashDB}; +use trie::{ + MemoryDB, PrefixedMemoryDB, default_child_trie_root, + read_trie_value_with, read_child_trie_value_with, record_all_keys +}; +pub use trie::trie_types::{Layout, TrieError}; +use crate::trie_backend::TrieBackend; +use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; +use crate::{Error, ExecutionError, Backend}; +use std::collections::HashMap; +use crate::DBValue; + +pub type ProofRecorder = Arc>>; + +/// Patricia trie-based backend which also tracks all touched storage trie values. +/// These can be sent to remote node and used as a proof of execution. +pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { + backend: &'a TrieBackend, + proof_recorder: ProofRecorder, +} + +pub struct ProofRecorderBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { + backend: &'a S, + proof_recorder: ProofRecorder, +} + +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> { + /// Create new proving backend. + pub fn new(backend: &'a TrieBackend) -> Self { + ProvingBackend { + backend, + proof_recorder: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Create new proving backend with the given recorder. + pub fn new_with_recorder( + backend: &'a TrieBackend, + proof_recorder: ProofRecorder, + ) -> Self { + ProvingBackend { + backend, + proof_recorder, + } + } + + /// Consume the backend, extracting the gathered proof in lexicographical order by value. + pub fn extract_proof(&self) -> Vec> { + // TODO mem replace hashmap and into_iter instead of clone. + // and store as vec in the first place?. + self.proof_recorder + .read() + //.borrow() + .iter() + .map(|(k, v)| v.to_vec()) + .collect() + } +} + +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage for ProofRecorderBackend<'a, S, H> { + type Overlay = S::Overlay; + + fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + // crazy bad mutex handle : mutex should not be needed, and get is occuring a lot so even + // if we use mutex we should use the mut handle of write TODO only for testing now + + if let Some(v) = self.proof_recorder.read().get(key) { + // TODO Notice that proof recorder could also stored missing value and be use as a better cache. + return Ok(Some(v.clone())); + } + Ok(match self.backend.get(key, prefix)? { + Some(v) => { + self.proof_recorder.write().insert(key.clone(), v.clone()); + Some(v) + }, + None => None, + }) + } +} +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ProvingBackend") + } +} + +impl<'a, S, H> ProvingBackend<'a, S, H> + where + S: 'a + TrieBackendStorage,// + Clone, + H: 'a + Hasher, + H::Out: Ord, +{ + + fn backend(&self) -> TrieBackend, H> { + let essence = self.backend.essence(); + let root = essence.root().clone(); +// // TODO bad code: TODO specialize for explicit type (one that +// // is an Arc some backend). + let recorder = ProofRecorderBackend { + backend: essence.backend_storage(), + proof_recorder: self.proof_recorder.clone(), + }; + TrieBackend::new(recorder, root) + } +} + +impl<'a, S, H> Backend for ProvingBackend<'a, S, H> + where + S: 'a + TrieBackendStorage, + H: 'a + Hasher, + H::Out: Ord, +{ + type Error = String; + type Transaction = S::Overlay; + type TrieBackendStorage = PrefixedMemoryDB; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.backend().storage(key) + } + + fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { + self.backend().child_storage(storage_key, key) + } + + fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { + self.backend().for_keys_in_child_storage(storage_key, f) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + self.backend().for_keys_with_prefix(prefix, f) + } + + fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { + self.backend().for_key_values_with_prefix(prefix, f) + } + + fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { + self.backend().for_child_keys_with_prefix(storage_key, prefix, f) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.backend().pairs() + } + + fn keys(&self, prefix: &[u8]) -> Vec> { + self.backend().keys(prefix) + } + + fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { + self.backend().child_keys(child_storage_key, prefix) + } + + fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) + where I: IntoIterator, Option>)> + { + self.backend().storage_root(delta) + } + + fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) + where + I: IntoIterator, Option>)>, + H::Out: Ord + { + self.backend().child_storage_root(storage_key, delta) + } +} + +#[cfg(test)] +mod tests { + use crate::backend::{InMemory}; + use crate::trie_backend::tests::test_trie; + use super::*; + use primitives::{Blake2Hasher, storage::ChildStorageKey}; + use crate::proving_backend::create_proof_check_backend; + + fn test_proving<'a>( + trie_backend: &'a TrieBackend,Blake2Hasher>, + ) -> ProvingBackend<'a, PrefixedMemoryDB, Blake2Hasher> { + ProvingBackend::new(trie_backend) + } + + #[test] + fn proof_is_empty_until_value_is_read() { + let trie_backend = test_trie(); + assert!(test_proving(&trie_backend).extract_proof().is_empty()); + } + + #[test] + fn proof_is_non_empty_after_value_is_read() { + let trie_backend = test_trie(); + let backend = test_proving(&trie_backend); + assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec())); + assert!(!backend.extract_proof().is_empty()); + } + + #[test] + fn proof_is_invalid_when_does_not_contains_root() { + use primitives::H256; + assert!(create_proof_check_backend::(H256::from_low_u64_be(1), vec![]).is_err()); + } + + #[test] + fn passes_throgh_backend_calls() { + let trie_backend = test_trie(); + let proving_backend = test_proving(&trie_backend); + assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); + assert_eq!(trie_backend.pairs(), proving_backend.pairs()); + + let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty()); + let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty()); + assert_eq!(trie_root, proving_root); + assert_eq!(trie_mdb.drain(), proving_mdb.drain()); + } + + #[test] + fn proof_recorded_and_checked() { + let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))).collect::>(); + let in_memory = InMemory::::default(); + let mut in_memory = in_memory.update(contents); + let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; + (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); + + let trie = in_memory.as_trie_backend().unwrap(); + let trie_root = trie.storage_root(::std::iter::empty()).0; + assert_eq!(in_memory_root, trie_root); + (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); + + let proving = ProvingBackend::new(trie); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); + + let proof = proving.extract_proof(); + + let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); + } + + #[test] + fn proof_recorded_and_checked_with_child() { + let subtrie1 = ChildStorageKey::from_slice(b":child_storage:default:sub1").unwrap(); + let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub2").unwrap(); + let own1 = subtrie1.into_owned(); + let own2 = subtrie2.into_owned(); + let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))) + .chain((28..65).map(|i| (Some(own1.clone()), vec![i], Some(vec![i])))) + .chain((10..15).map(|i| (Some(own2.clone()), vec![i], Some(vec![i])))) + .collect::>(); + let in_memory = InMemory::::default(); + let mut in_memory = in_memory.update(contents); + let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>( + ::std::iter::empty(), + in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new())) + ).0; + (0..64).for_each(|i| assert_eq!( + in_memory.storage(&[i]).unwrap().unwrap(), + vec![i] + )); + (28..65).for_each(|i| assert_eq!( + in_memory.child_storage(&own1[..], &[i]).unwrap().unwrap(), + vec![i] + )); + (10..15).for_each(|i| assert_eq!( + in_memory.child_storage(&own2[..], &[i]).unwrap().unwrap(), + vec![i] + )); + + let trie = in_memory.as_trie_backend().unwrap(); + let trie_root = trie.storage_root(::std::iter::empty()).0; + assert_eq!(in_memory_root, trie_root); + (0..64).for_each(|i| assert_eq!( + trie.storage(&[i]).unwrap().unwrap(), + vec![i] + )); + + let proving = ProvingBackend::new(trie); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); + + let proof = proving.extract_proof(); + + let proof_check = create_proof_check_backend::( + in_memory_root.into(), + proof + ).unwrap(); + assert!(proof_check.storage(&[0]).is_err()); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); + // note that it is include in root because proof close + assert_eq!(proof_check.storage(&[41]).unwrap().unwrap(), vec![41]); + assert_eq!(proof_check.storage(&[64]).unwrap(), None); + + let proving = ProvingBackend::new(trie); + assert_eq!(proving.child_storage(&own1[..], &[64]), Ok(Some(vec![64]))); + + let proof = proving.extract_proof(); + let proof_check = create_proof_check_backend::( + in_memory_root.into(), + proof + ).unwrap(); + assert_eq!( + proof_check.child_storage(&own1[..], &[64]).unwrap().unwrap(), + vec![64] + ); + } + +} From 889ec963ed1ccc4a9cb492af4ad2923d550f834b Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 25 Oct 2019 11:52:47 +0200 Subject: [PATCH 2/7] use full proof by default (replace previous proof recorder). --- core/client/src/call_executor.rs | 10 +- core/client/src/client.rs | 6 +- core/client/src/light/call_executor.rs | 8 +- core/client/src/runtime_api.rs | 12 +- core/sr-api-macros/src/decl_runtime_apis.rs | 2 +- core/sr-api-macros/src/impl_runtime_apis.rs | 4 +- core/state-machine/src/lib.rs | 11 +- core/state-machine/src/proving_backend.rs | 100 ++++-- .../state-machine/src/proving_backend_full.rs | 324 ------------------ 9 files changed, 91 insertions(+), 386 deletions(-) delete mode 100644 core/state-machine/src/proving_backend_full.rs diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index 0270e787926b9..919e4a4dde45f 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::{sync::Arc, cmp::Ord, panic::UnwindSafe, result, cell::RefCell, rc::Rc}; +use std::{sync::Arc, cmp::Ord, panic::UnwindSafe, result, cell::RefCell}; use codec::{Encode, Decode}; use sr_primitives::{ generic::BlockId, traits::Block as BlockT, traits::NumberFor, @@ -30,7 +30,7 @@ use primitives::{ traits::{CodeExecutor, KeystoreExt}, }; -use crate::runtime_api::{FullProofRecorder, InitializeBlock}; +use crate::runtime_api::{ProofRecorder, InitializeBlock}; use crate::backend; use crate::error; @@ -81,7 +81,7 @@ where execution_manager: ExecutionManager, native_call: Option, side_effects_handler: Option, - proof_recorder: &Option>, + proof_recorder: &Option>, enable_keystore: bool, ) -> error::Result> where ExecutionManager: Clone; @@ -241,7 +241,7 @@ where execution_manager: ExecutionManager, native_call: Option, side_effects_handler: Option, - recorder: &Option>, + recorder: &Option>, enable_keystore: bool, ) -> Result, error::Error> where ExecutionManager: Clone { match initialize_block { @@ -269,7 +269,7 @@ where as Box )?; - let backend = state_machine::full_proving_backend::ProvingBackend::new_with_recorder( + let backend = state_machine::ProvingBackend::new_with_recorder( trie_state, recorder.clone() ); diff --git a/core/client/src/client.rs b/core/client/src/client.rs index faddff6e7b455..7cbfae9e4adf6 100644 --- a/core/client/src/client.rs +++ b/core/client/src/client.rs @@ -18,7 +18,7 @@ use std::{ marker::PhantomData, collections::{HashSet, BTreeMap, HashMap}, sync::Arc, - panic::UnwindSafe, result, cell::RefCell, rc::Rc, + panic::UnwindSafe, result, cell::RefCell, }; use log::{info, trace, warn}; use futures03::channel::mpsc; @@ -55,7 +55,7 @@ use header_metadata::{HeaderMetadata, CachedHeaderMetadata}; use crate::{ runtime_api::{ - CallRuntimeAt, ConstructRuntimeApi, Core as CoreApi, FullProofRecorder, + CallRuntimeAt, ConstructRuntimeApi, Core as CoreApi, ProofRecorder, InitializeBlock, }, backend::{ @@ -1453,7 +1453,7 @@ impl CallRuntimeAt for Client where initialize_block: InitializeBlock<'a, Block>, native_call: Option, context: ExecutionContext, - recorder: &Option>, + recorder: &Option>, ) -> error::Result> { let manager = match context { ExecutionContext::BlockConstruction => diff --git a/core/client/src/light/call_executor.rs b/core/client/src/light/call_executor.rs index da492f5be1e43..8b9dea80d613e 100644 --- a/core/client/src/light/call_executor.rs +++ b/core/client/src/light/call_executor.rs @@ -18,7 +18,7 @@ use std::{ collections::HashSet, sync::Arc, panic::UnwindSafe, result, - cell::RefCell, rc::Rc, + cell::RefCell, }; use codec::{Encode, Decode}; @@ -35,7 +35,7 @@ use state_machine::{ }; use hash_db::Hasher; -use crate::runtime_api::{FullProofRecorder, InitializeBlock}; +use crate::runtime_api::{ProofRecorder, InitializeBlock}; use crate::backend::RemoteBackend; use crate::call_executor::CallExecutor; use crate::error::{Error as ClientError, Result as ClientResult}; @@ -109,7 +109,7 @@ impl CallExecutor for _manager: ExecutionManager, native_call: Option, side_effects_handler: Option, - recorder: &Option>, + recorder: &Option>, enable_keystore: bool, ) -> ClientResult> where ExecutionManager: Clone { // there's no actual way/need to specify native/wasm execution strategy on light node @@ -339,7 +339,7 @@ mod tests { _execution_manager: ExecutionManager, _native_call: Option, _side_effects_handler: Option, - _proof_recorder: &Option>, + _proof_recorder: &Option>, _enable_keystore: bool, ) -> ClientResult> where ExecutionManager: Clone { unreachable!() diff --git a/core/client/src/runtime_api.rs b/core/client/src/runtime_api.rs index 48dec99479ffe..69a62ff8dbf52 100644 --- a/core/client/src/runtime_api.rs +++ b/core/client/src/runtime_api.rs @@ -45,19 +45,11 @@ use crate::error; use sr_api_macros::decl_runtime_apis; use primitives::OpaqueMetadata; #[cfg(feature = "std")] -use std::{panic::UnwindSafe, cell::RefCell, rc::Rc}; -#[cfg(feature = "std")] -use primitives::Hasher as HasherT; +use std::{panic::UnwindSafe, cell::RefCell}; #[cfg(feature = "std")] /// A type that records all accessed trie nodes and generates a proof out of it. pub type ProofRecorder = state_machine::ProofRecorder< - <<<::Header as HeaderT>::Hashing as HashT>::Hasher as HasherT>::Out ->; - -#[cfg(feature = "std")] -/// A type that records all accessed trie nodes and generates a proof out of it. -pub type FullProofRecorder = state_machine::full_proving_backend::ProofRecorder< <<::Header as HeaderT>::Hashing as HashT>::Hasher >; @@ -154,7 +146,7 @@ pub trait CallRuntimeAt { initialize_block: InitializeBlock<'a, Block>, native_call: Option, context: ExecutionContext, - recorder: &Option>, + recorder: &Option>, ) -> error::Result>; /// Returns the runtime version at the given block. diff --git a/core/sr-api-macros/src/decl_runtime_apis.rs b/core/sr-api-macros/src/decl_runtime_apis.rs index a847f75b110e8..d50b6acfca5f8 100644 --- a/core/sr-api-macros/src/decl_runtime_apis.rs +++ b/core/sr-api-macros/src/decl_runtime_apis.rs @@ -411,7 +411,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { initialized_block: &std::cell::RefCell>>, native_call: Option, context: #crate_::runtime_api::ExecutionContext, - recorder: &Option<#crate_::runtime_api::FullProofRecorder>, + recorder: &Option<#crate_::runtime_api::ProofRecorder>, ) -> #crate_::error::Result<#crate_::runtime_api::NativeOrEncoded> { let version = call_runtime_at.runtime_version_at(at)?; use #crate_::runtime_api::InitializeBlock; diff --git a/core/sr-api-macros/src/impl_runtime_apis.rs b/core/sr-api-macros/src/impl_runtime_apis.rs index 40a3acbafb30e..2a5897a496186 100644 --- a/core/sr-api-macros/src/impl_runtime_apis.rs +++ b/core/sr-api-macros/src/impl_runtime_apis.rs @@ -267,7 +267,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result, initialized_block: std::cell::RefCell>, changes: std::cell::RefCell<#crate_::runtime_api::OverlayedChanges>, - recorder: Option<#crate_::runtime_api::FullProofRecorder<#block>>, + recorder: Option<#crate_::runtime_api::ProofRecorder<#block>>, } // `RuntimeApi` itself is not threadsafe. However, an instance is only available in a @@ -347,7 +347,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result, &std::cell::RefCell>>, - &Option<#crate_::runtime_api::FullProofRecorder<#block>>, + &Option<#crate_::runtime_api::ProofRecorder<#block>>, ) -> #crate_::error::Result<#crate_::runtime_api::NativeOrEncoded>, >( &self, diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 1bc2adeae7311..21acbb322b65f 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -37,7 +37,6 @@ mod testing; mod basic; mod overlayed_changes; mod proving_backend; -mod proving_backend_full; mod trie_backend; mod trie_backend_essence; @@ -61,16 +60,8 @@ pub use changes_trie::{ pub use overlayed_changes::OverlayedChanges; pub use proving_backend::{ create_proof_check_backend, create_proof_check_backend_storage, - Recorder as ProofRecorder, ProvingBackend, + ProofRecorder, ProvingBackend, }; -pub mod full_proving_backend { - pub use crate::proving_backend_full::{ - ProofRecorder, ProvingBackend, - }; - pub use crate::proving_backend::{ - create_proof_check_backend, create_proof_check_backend_storage, - }; -} pub use trie_backend_essence::{TrieBackendStorage, Storage}; pub use trie_backend::TrieBackend; pub use error::{Error, ExecutionError}; diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index c6c4ef1ec8c26..3978eca5148eb 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -16,9 +16,11 @@ //! Proving state machine backend. +use std::sync::Arc; +use parking_lot::RwLock; use std::{cell::RefCell, rc::Rc}; use log::debug; -use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; +use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix}; use trie::{ MemoryDB, PrefixedMemoryDB, default_child_trie_root, read_trie_value_with, read_child_trie_value_with, record_all_keys @@ -28,6 +30,8 @@ pub use trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; +use std::collections::HashMap; +use crate::DBValue; /// Patricia trie-based backend essence which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -101,11 +105,18 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> } } +pub type ProofRecorder = Arc>>; + /// Patricia trie-based backend which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { backend: &'a TrieBackend, - proof_recorder: Rc>>, + proof_recorder: ProofRecorder, +} + +pub struct ProofRecorderBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { + backend: &'a S, + proof_recorder: ProofRecorder, } impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> { @@ -113,14 +124,14 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> pub fn new(backend: &'a TrieBackend) -> Self { ProvingBackend { backend, - proof_recorder: Rc::new(RefCell::new(Recorder::new())), + proof_recorder: Arc::new(RwLock::new(HashMap::new())), } } /// Create new proving backend with the given recorder. pub fn new_with_recorder( backend: &'a TrieBackend, - proof_recorder: Rc>>, + proof_recorder: ProofRecorder, ) -> Self { ProvingBackend { backend, @@ -130,21 +141,63 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> /// Consume the backend, extracting the gathered proof in lexicographical order by value. pub fn extract_proof(&self) -> Vec> { + // TODO mem replace hashmap and into_iter instead of clone. + // and store as vec in the first place?. self.proof_recorder - .borrow_mut() - .drain() - .into_iter() - .map(|n| n.data.to_vec()) + .read() + //.borrow() + .iter() + .map(|(k, v)| v.to_vec()) .collect() } } +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage for ProofRecorderBackend<'a, S, H> { + type Overlay = S::Overlay; + + fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + // crazy bad mutex handle : mutex should not be needed, and get is occuring a lot so even + // if we use mutex we should use the mut handle of write TODO only for testing now + + if let Some(v) = self.proof_recorder.read().get(key) { + // TODO Notice that proof recorder could also stored missing value and be use as a better cache. + return Ok(Some(v.clone())); + } + Ok(match self.backend.get(key, prefix)? { + Some(v) => { + self.proof_recorder.write().insert(key.clone(), v.clone()); + Some(v) + }, + None => None, + }) + } +} impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "ProvingBackend") } } +impl<'a, S, H> ProvingBackend<'a, S, H> + where + S: 'a + TrieBackendStorage,// + Clone, + H: 'a + Hasher, + H::Out: Ord, +{ + + fn backend(&self) -> TrieBackend, H> { + let essence = self.backend.essence(); + let root = essence.root().clone(); +// // TODO bad code: TODO specialize for explicit type (one that +// // is an Arc some backend). + let recorder = ProofRecorderBackend { + backend: essence.backend_storage(), + proof_recorder: self.proof_recorder.clone(), + }; + TrieBackend::new(recorder, root) + } +} + impl<'a, S, H> Backend for ProvingBackend<'a, S, H> where S: 'a + TrieBackendStorage, @@ -156,53 +209,45 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> type TrieBackendStorage = PrefixedMemoryDB; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { - ProvingBackendEssence { - backend: self.backend.essence(), - proof_recorder: &mut *self.proof_recorder.try_borrow_mut() - .expect("only fails when already borrowed; storage() is non-reentrant; qed"), - }.storage(key) + self.backend().storage(key) } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - ProvingBackendEssence { - backend: self.backend.essence(), - proof_recorder: &mut *self.proof_recorder.try_borrow_mut() - .expect("only fails when already borrowed; child_storage() is non-reentrant; qed"), - }.child_storage(storage_key, key) + self.backend().child_storage(storage_key, key) } fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - self.backend.for_keys_in_child_storage(storage_key, f) + self.backend().for_keys_in_child_storage(storage_key, f) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { - self.backend.for_keys_with_prefix(prefix, f) + self.backend().for_keys_with_prefix(prefix, f) } fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { - self.backend.for_key_values_with_prefix(prefix, f) + self.backend().for_key_values_with_prefix(prefix, f) } fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - self.backend.for_child_keys_with_prefix(storage_key, prefix, f) + self.backend().for_child_keys_with_prefix(storage_key, prefix, f) } fn pairs(&self) -> Vec<(Vec, Vec)> { - self.backend.pairs() + self.backend().pairs() } fn keys(&self, prefix: &[u8]) -> Vec> { - self.backend.keys(prefix) + self.backend().keys(prefix) } fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { - self.backend.child_keys(child_storage_key, prefix) + self.backend().child_keys(child_storage_key, prefix) } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) where I: IntoIterator, Option>)> { - self.backend.storage_root(delta) + self.backend().storage_root(delta) } fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) @@ -210,7 +255,7 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> I: IntoIterator, Option>)>, H::Out: Ord { - self.backend.child_storage_root(storage_key, delta) + self.backend().child_storage_root(storage_key, delta) } } @@ -251,6 +296,7 @@ mod tests { use crate::trie_backend::tests::test_trie; use super::*; use primitives::{Blake2Hasher, storage::ChildStorageKey}; + use crate::proving_backend::create_proof_check_backend; fn test_proving<'a>( trie_backend: &'a TrieBackend,Blake2Hasher>, diff --git a/core/state-machine/src/proving_backend_full.rs b/core/state-machine/src/proving_backend_full.rs deleted file mode 100644 index a029cbdcf8fd9..0000000000000 --- a/core/state-machine/src/proving_backend_full.rs +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Proving state machine backend. - -use std::sync::Arc; -use parking_lot::RwLock; -use std::{cell::RefCell, rc::Rc}; -use log::debug; -use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix, HashDB}; -use trie::{ - MemoryDB, PrefixedMemoryDB, default_child_trie_root, - read_trie_value_with, read_child_trie_value_with, record_all_keys -}; -pub use trie::trie_types::{Layout, TrieError}; -use crate::trie_backend::TrieBackend; -use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; -use crate::{Error, ExecutionError, Backend}; -use std::collections::HashMap; -use crate::DBValue; - -pub type ProofRecorder = Arc>>; - -/// Patricia trie-based backend which also tracks all touched storage trie values. -/// These can be sent to remote node and used as a proof of execution. -pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { - backend: &'a TrieBackend, - proof_recorder: ProofRecorder, -} - -pub struct ProofRecorderBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { - backend: &'a S, - proof_recorder: ProofRecorder, -} - -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> { - /// Create new proving backend. - pub fn new(backend: &'a TrieBackend) -> Self { - ProvingBackend { - backend, - proof_recorder: Arc::new(RwLock::new(HashMap::new())), - } - } - - /// Create new proving backend with the given recorder. - pub fn new_with_recorder( - backend: &'a TrieBackend, - proof_recorder: ProofRecorder, - ) -> Self { - ProvingBackend { - backend, - proof_recorder, - } - } - - /// Consume the backend, extracting the gathered proof in lexicographical order by value. - pub fn extract_proof(&self) -> Vec> { - // TODO mem replace hashmap and into_iter instead of clone. - // and store as vec in the first place?. - self.proof_recorder - .read() - //.borrow() - .iter() - .map(|(k, v)| v.to_vec()) - .collect() - } -} - -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage for ProofRecorderBackend<'a, S, H> { - type Overlay = S::Overlay; - - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { - // crazy bad mutex handle : mutex should not be needed, and get is occuring a lot so even - // if we use mutex we should use the mut handle of write TODO only for testing now - - if let Some(v) = self.proof_recorder.read().get(key) { - // TODO Notice that proof recorder could also stored missing value and be use as a better cache. - return Ok(Some(v.clone())); - } - Ok(match self.backend.get(key, prefix)? { - Some(v) => { - self.proof_recorder.write().insert(key.clone(), v.clone()); - Some(v) - }, - None => None, - }) - } -} -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "ProvingBackend") - } -} - -impl<'a, S, H> ProvingBackend<'a, S, H> - where - S: 'a + TrieBackendStorage,// + Clone, - H: 'a + Hasher, - H::Out: Ord, -{ - - fn backend(&self) -> TrieBackend, H> { - let essence = self.backend.essence(); - let root = essence.root().clone(); -// // TODO bad code: TODO specialize for explicit type (one that -// // is an Arc some backend). - let recorder = ProofRecorderBackend { - backend: essence.backend_storage(), - proof_recorder: self.proof_recorder.clone(), - }; - TrieBackend::new(recorder, root) - } -} - -impl<'a, S, H> Backend for ProvingBackend<'a, S, H> - where - S: 'a + TrieBackendStorage, - H: 'a + Hasher, - H::Out: Ord, -{ - type Error = String; - type Transaction = S::Overlay; - type TrieBackendStorage = PrefixedMemoryDB; - - fn storage(&self, key: &[u8]) -> Result>, Self::Error> { - self.backend().storage(key) - } - - fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - self.backend().child_storage(storage_key, key) - } - - fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - self.backend().for_keys_in_child_storage(storage_key, f) - } - - fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { - self.backend().for_keys_with_prefix(prefix, f) - } - - fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { - self.backend().for_key_values_with_prefix(prefix, f) - } - - fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - self.backend().for_child_keys_with_prefix(storage_key, prefix, f) - } - - fn pairs(&self) -> Vec<(Vec, Vec)> { - self.backend().pairs() - } - - fn keys(&self, prefix: &[u8]) -> Vec> { - self.backend().keys(prefix) - } - - fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { - self.backend().child_keys(child_storage_key, prefix) - } - - fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) - where I: IntoIterator, Option>)> - { - self.backend().storage_root(delta) - } - - fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) - where - I: IntoIterator, Option>)>, - H::Out: Ord - { - self.backend().child_storage_root(storage_key, delta) - } -} - -#[cfg(test)] -mod tests { - use crate::backend::{InMemory}; - use crate::trie_backend::tests::test_trie; - use super::*; - use primitives::{Blake2Hasher, storage::ChildStorageKey}; - use crate::proving_backend::create_proof_check_backend; - - fn test_proving<'a>( - trie_backend: &'a TrieBackend,Blake2Hasher>, - ) -> ProvingBackend<'a, PrefixedMemoryDB, Blake2Hasher> { - ProvingBackend::new(trie_backend) - } - - #[test] - fn proof_is_empty_until_value_is_read() { - let trie_backend = test_trie(); - assert!(test_proving(&trie_backend).extract_proof().is_empty()); - } - - #[test] - fn proof_is_non_empty_after_value_is_read() { - let trie_backend = test_trie(); - let backend = test_proving(&trie_backend); - assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec())); - assert!(!backend.extract_proof().is_empty()); - } - - #[test] - fn proof_is_invalid_when_does_not_contains_root() { - use primitives::H256; - assert!(create_proof_check_backend::(H256::from_low_u64_be(1), vec![]).is_err()); - } - - #[test] - fn passes_throgh_backend_calls() { - let trie_backend = test_trie(); - let proving_backend = test_proving(&trie_backend); - assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); - assert_eq!(trie_backend.pairs(), proving_backend.pairs()); - - let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty()); - let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty()); - assert_eq!(trie_root, proving_root); - assert_eq!(trie_mdb.drain(), proving_mdb.drain()); - } - - #[test] - fn proof_recorded_and_checked() { - let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))).collect::>(); - let in_memory = InMemory::::default(); - let mut in_memory = in_memory.update(contents); - let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; - (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); - - let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(::std::iter::empty()).0; - assert_eq!(in_memory_root, trie_root); - (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); - - let proving = ProvingBackend::new(trie); - assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); - - let proof = proving.extract_proof(); - - let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); - assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); - } - - #[test] - fn proof_recorded_and_checked_with_child() { - let subtrie1 = ChildStorageKey::from_slice(b":child_storage:default:sub1").unwrap(); - let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub2").unwrap(); - let own1 = subtrie1.into_owned(); - let own2 = subtrie2.into_owned(); - let contents = (0..64).map(|i| (None, vec![i], Some(vec![i]))) - .chain((28..65).map(|i| (Some(own1.clone()), vec![i], Some(vec![i])))) - .chain((10..15).map(|i| (Some(own2.clone()), vec![i], Some(vec![i])))) - .collect::>(); - let in_memory = InMemory::::default(); - let mut in_memory = in_memory.update(contents); - let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>( - ::std::iter::empty(), - in_memory.child_storage_keys().map(|k|(k.to_vec(), Vec::new())) - ).0; - (0..64).for_each(|i| assert_eq!( - in_memory.storage(&[i]).unwrap().unwrap(), - vec![i] - )); - (28..65).for_each(|i| assert_eq!( - in_memory.child_storage(&own1[..], &[i]).unwrap().unwrap(), - vec![i] - )); - (10..15).for_each(|i| assert_eq!( - in_memory.child_storage(&own2[..], &[i]).unwrap().unwrap(), - vec![i] - )); - - let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(::std::iter::empty()).0; - assert_eq!(in_memory_root, trie_root); - (0..64).for_each(|i| assert_eq!( - trie.storage(&[i]).unwrap().unwrap(), - vec![i] - )); - - let proving = ProvingBackend::new(trie); - assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); - - let proof = proving.extract_proof(); - - let proof_check = create_proof_check_backend::( - in_memory_root.into(), - proof - ).unwrap(); - assert!(proof_check.storage(&[0]).is_err()); - assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); - // note that it is include in root because proof close - assert_eq!(proof_check.storage(&[41]).unwrap().unwrap(), vec![41]); - assert_eq!(proof_check.storage(&[64]).unwrap(), None); - - let proving = ProvingBackend::new(trie); - assert_eq!(proving.child_storage(&own1[..], &[64]), Ok(Some(vec![64]))); - - let proof = proving.extract_proof(); - let proof_check = create_proof_check_backend::( - in_memory_root.into(), - proof - ).unwrap(); - assert_eq!( - proof_check.child_storage(&own1[..], &[64]).unwrap().unwrap(), - vec![64] - ); - } - -} From 34cc3a4c80fefcfe2a2354fd282f65e773286ffc Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 25 Oct 2019 12:13:35 +0200 Subject: [PATCH 3/7] fix warnings. --- .../src/changes_trie/changes_iterator.rs | 4 ++-- core/state-machine/src/changes_trie/prune.rs | 4 ++-- core/state-machine/src/lib.rs | 2 +- core/state-machine/src/proving_backend.rs | 17 ++++++++++------- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/core/state-machine/src/changes_trie/changes_iterator.rs b/core/state-machine/src/changes_trie/changes_iterator.rs index e8730a1bddedb..2f61b87fe2199 100644 --- a/core/state-machine/src/changes_trie/changes_iterator.rs +++ b/core/state-machine/src/changes_trie/changes_iterator.rs @@ -28,7 +28,7 @@ use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage}; use crate::changes_trie::input::ChildIndex; use crate::changes_trie::surface_iterator::{surface_iterator, SurfaceIterator}; -use crate::proving_backend::ProvingBackendEssence; +use crate::proving_backend::ProvingBackendRecorder; use crate::trie_backend_essence::{TrieBackendEssence}; /// Return changes of given key at given blocks range. @@ -366,7 +366,7 @@ impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number> let proof_recorder = &mut *self.proof_recorder.try_borrow_mut() .expect("only fails when already borrowed; storage() is non-reentrant; qed"); self.essence.next(|storage, root, key| - ProvingBackendEssence::<_, H> { + ProvingBackendRecorder::<_, H> { backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root), proof_recorder, }.storage(key)) diff --git a/core/state-machine/src/changes_trie/prune.rs b/core/state-machine/src/changes_trie/prune.rs index 1dc7c3e6c07b2..531318e234c8d 100644 --- a/core/state-machine/src/changes_trie/prune.rs +++ b/core/state-machine/src/changes_trie/prune.rs @@ -20,7 +20,7 @@ use hash_db::Hasher; use trie::Recorder; use log::warn; use num_traits::{One, Zero}; -use crate::proving_backend::ProvingBackendEssence; +use crate::proving_backend::ProvingBackendRecorder; use crate::trie_backend_essence::TrieBackendEssence; use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber}; use crate::changes_trie::storage::TrieBackendAdapter; @@ -122,7 +122,7 @@ fn prune_trie, H: Hasher, Number: BlockNumber, F: FnMut(H: // (effectively - all changes trie nodes) let mut proof_recorder: Recorder = Default::default(); { - let mut trie = ProvingBackendEssence::<_, H> { + let mut trie = ProvingBackendRecorder::<_, H> { backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root), proof_recorder: &mut proof_recorder, }; diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 21acbb322b65f..c0158ca185005 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -60,7 +60,7 @@ pub use changes_trie::{ pub use overlayed_changes::OverlayedChanges; pub use proving_backend::{ create_proof_check_backend, create_proof_check_backend_storage, - ProofRecorder, ProvingBackend, + ProofRecorder, ProvingBackend,ProvingBackendRecorder, }; pub use trie_backend_essence::{TrieBackendStorage, Storage}; pub use trie_backend::TrieBackend; diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 3978eca5148eb..e23e745e9c9e7 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use parking_lot::RwLock; -use std::{cell::RefCell, rc::Rc}; use log::debug; use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix}; use trie::{ @@ -33,18 +32,18 @@ use crate::{Error, ExecutionError, Backend}; use std::collections::HashMap; use crate::DBValue; -/// Patricia trie-based backend essence which also tracks all touched storage trie values. -/// These can be sent to remote node and used as a proof of execution. -pub struct ProvingBackendEssence<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { +/// Patricia trie-based backend specialized in get value proofs. +pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { pub(crate) backend: &'a TrieBackendEssence, pub(crate) proof_recorder: &'a mut Recorder, } -impl<'a, S, H> ProvingBackendEssence<'a, S, H> +impl<'a, S, H> ProvingBackendRecorder<'a, S, H> where S: TrieBackendStorage, H: Hasher, { + /// Produce proof for a key query. pub fn storage(&mut self, key: &[u8]) -> Result>, String> { let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new( @@ -62,6 +61,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> ).map_err(map_e) } + /// Produce proof for a child key query. pub fn child_storage( &mut self, storage_key: &[u8], @@ -87,6 +87,7 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> ).map_err(map_e) } + /// Produce proof for tho whole backend. pub fn record_all_keys(&mut self) { let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new( @@ -105,7 +106,9 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H> } } -pub type ProofRecorder = Arc>>; +/// Global proof recorder, act as a layer over a hash db for recording queried +/// data. +pub type ProofRecorder = Arc::Out, DBValue>>>; /// Patricia trie-based backend which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -147,7 +150,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> .read() //.borrow() .iter() - .map(|(k, v)| v.to_vec()) + .map(|(_k, v)| v.to_vec()) .collect() } } From c6b4631549e6fbd291a2f58334c678d7842481a1 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 25 Oct 2019 12:49:26 +0200 Subject: [PATCH 4/7] Cache value not found in proof recorder. --- core/sr-api-macros/src/impl_runtime_apis.rs | 4 ++-- core/state-machine/src/proving_backend.rs | 18 +++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/core/sr-api-macros/src/impl_runtime_apis.rs b/core/sr-api-macros/src/impl_runtime_apis.rs index 2a5897a496186..3b2756805b9ae 100644 --- a/core/sr-api-macros/src/impl_runtime_apis.rs +++ b/core/sr-api-macros/src/impl_runtime_apis.rs @@ -311,9 +311,9 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result ProvingBackendRecorder<'a, S, H> /// Global proof recorder, act as a layer over a hash db for recording queried /// data. -pub type ProofRecorder = Arc::Out, DBValue>>>; +pub type ProofRecorder = Arc::Out, Option>>>; /// Patricia trie-based backend which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. @@ -150,7 +150,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> .read() //.borrow() .iter() - .map(|(_k, v)| v.to_vec()) + .filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec())) .collect() } } @@ -163,18 +163,14 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage fo // if we use mutex we should use the mut handle of write TODO only for testing now if let Some(v) = self.proof_recorder.read().get(key) { - // TODO Notice that proof recorder could also stored missing value and be use as a better cache. - return Ok(Some(v.clone())); + return Ok(v.clone()); } - Ok(match self.backend.get(key, prefix)? { - Some(v) => { - self.proof_recorder.write().insert(key.clone(), v.clone()); - Some(v) - }, - None => None, - }) + let backend_value = self.backend.get(key, prefix)?; + self.proof_recorder.write().insert(key.clone(), backend_value.clone()); + Ok(backend_value) } } + impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "ProvingBackend") From e967dc1a1ea9bf9915454b292ac04a35635c98f1 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 28 Oct 2019 17:48:36 +0100 Subject: [PATCH 5/7] Remove need or internal backend struct. --- core/sr-api-macros/src/impl_runtime_apis.rs | 1 - core/state-machine/src/lib.rs | 2 +- core/state-machine/src/proving_backend.rs | 77 +++++++-------------- 3 files changed, 27 insertions(+), 53 deletions(-) diff --git a/core/sr-api-macros/src/impl_runtime_apis.rs b/core/sr-api-macros/src/impl_runtime_apis.rs index 5d704473cbea1..84a0822238996 100644 --- a/core/sr-api-macros/src/impl_runtime_apis.rs +++ b/core/sr-api-macros/src/impl_runtime_apis.rs @@ -305,7 +305,6 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result ProvingBackendRecorder<'a, S, H> ).map_err(map_e) } - /// Produce proof for tho whole backend. + /// Produce proof for the whole backend. pub fn record_all_keys(&mut self) { let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new( @@ -112,11 +112,11 @@ pub type ProofRecorder = Arc::Out, Option, H: 'a + Hasher> { - backend: &'a TrieBackend, - proof_recorder: ProofRecorder, -} +pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ( + TrieBackend, H>, +); +/// Trie backend storage with its proof recorder. pub struct ProofRecorderBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { backend: &'a S, proof_recorder: ProofRecorder, @@ -125,10 +125,8 @@ pub struct ProofRecorderBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hashe impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> { /// Create new proving backend. pub fn new(backend: &'a TrieBackend) -> Self { - ProvingBackend { - backend, - proof_recorder: Arc::new(RwLock::new(HashMap::new())), - } + let proof_recorder = Default::default(); + Self::new_with_recorder(backend, proof_recorder) } /// Create new proving backend with the given recorder. @@ -136,19 +134,19 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> backend: &'a TrieBackend, proof_recorder: ProofRecorder, ) -> Self { - ProvingBackend { - backend, - proof_recorder, - } + let essence = backend.essence(); + let root = essence.root().clone(); + let recorder = ProofRecorderBackend { + backend: essence.backend_storage(), + proof_recorder: proof_recorder, + }; + ProvingBackend(TrieBackend::new(recorder, root)) } /// Consume the backend, extracting the gathered proof in lexicographical order by value. pub fn extract_proof(&self) -> Vec> { - // TODO mem replace hashmap and into_iter instead of clone. - // and store as vec in the first place?. - self.proof_recorder + self.0.essence().backend_storage().proof_recorder .read() - //.borrow() .iter() .filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec())) .collect() @@ -159,9 +157,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage fo type Overlay = S::Overlay; fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { - // crazy bad mutex handle : mutex should not be needed, and get is occuring a lot so even - // if we use mutex we should use the mut handle of write TODO only for testing now - if let Some(v) = self.proof_recorder.read().get(key) { return Ok(v.clone()); } @@ -177,26 +172,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug for Prov } } -impl<'a, S, H> ProvingBackend<'a, S, H> - where - S: 'a + TrieBackendStorage,// + Clone, - H: 'a + Hasher, - H::Out: Ord, -{ - - fn backend(&self) -> TrieBackend, H> { - let essence = self.backend.essence(); - let root = essence.root().clone(); -// // TODO bad code: TODO specialize for explicit type (one that -// // is an Arc some backend). - let recorder = ProofRecorderBackend { - backend: essence.backend_storage(), - proof_recorder: self.proof_recorder.clone(), - }; - TrieBackend::new(recorder, root) - } -} - impl<'a, S, H> Backend for ProvingBackend<'a, S, H> where S: 'a + TrieBackendStorage, @@ -208,45 +183,45 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> type TrieBackendStorage = PrefixedMemoryDB; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { - self.backend().storage(key) + self.0.storage(key) } fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Result>, Self::Error> { - self.backend().child_storage(storage_key, key) + self.0.child_storage(storage_key, key) } fn for_keys_in_child_storage(&self, storage_key: &[u8], f: F) { - self.backend().for_keys_in_child_storage(storage_key, f) + self.0.for_keys_in_child_storage(storage_key, f) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { - self.backend().for_keys_with_prefix(prefix, f) + self.0.for_keys_with_prefix(prefix, f) } fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { - self.backend().for_key_values_with_prefix(prefix, f) + self.0.for_key_values_with_prefix(prefix, f) } fn for_child_keys_with_prefix(&self, storage_key: &[u8], prefix: &[u8], f: F) { - self.backend().for_child_keys_with_prefix(storage_key, prefix, f) + self.0.for_child_keys_with_prefix(storage_key, prefix, f) } fn pairs(&self) -> Vec<(Vec, Vec)> { - self.backend().pairs() + self.0.pairs() } fn keys(&self, prefix: &[u8]) -> Vec> { - self.backend().keys(prefix) + self.0.keys(prefix) } fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { - self.backend().child_keys(child_storage_key, prefix) + self.0.child_keys(child_storage_key, prefix) } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) where I: IntoIterator, Option>)> { - self.backend().storage_root(delta) + self.0.storage_root(delta) } fn child_storage_root(&self, storage_key: &[u8], delta: I) -> (Vec, bool, Self::Transaction) @@ -254,7 +229,7 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> I: IntoIterator, Option>)>, H::Out: Ord { - self.backend().child_storage_root(storage_key, delta) + self.0.child_storage_root(storage_key, delta) } } From b817a6f53de7809b1fd1324b8457e5dd0ff33b8b Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 11 Nov 2019 17:23:48 +0100 Subject: [PATCH 6/7] fix type. --- core/sr-api/src/lib.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/sr-api/src/lib.rs b/core/sr-api/src/lib.rs index 2a149ffbc9c1a..4a5cbf513f13e 100644 --- a/core/sr-api/src/lib.rs +++ b/core/sr-api/src/lib.rs @@ -59,16 +59,14 @@ use rstd::result; pub use codec::{Encode, Decode}; use primitives::OpaqueMetadata; #[cfg(feature = "std")] -use std::{panic::UnwindSafe, cell::RefCell, rc::Rc}; -#[cfg(feature = "std")] -use primitives::Hasher as HasherT; +use std::{panic::UnwindSafe, cell::RefCell}; pub use sr_api_proc_macro::{decl_runtime_apis, impl_runtime_apis}; #[cfg(feature = "std")] /// A type that records all accessed trie nodes and generates a proof out of it. pub type ProofRecorder = state_machine::ProofRecorder< - <<<::Header as HeaderT>::Hashing as HashT>::Hasher as HasherT>::Out + <<::Header as HeaderT>::Hashing as HashT>::Hasher >; /// Something that can be constructed to a runtime api. @@ -168,7 +166,7 @@ pub trait CallRuntimeAt { initialize_block: InitializeBlock<'a, Block>, native_call: Option, context: ExecutionContext, - recorder: &Option>>>, + recorder: &Option>, ) -> Result, Self::Error>; /// Returns the runtime version at the given block. From 1517c97ceb707affe8c86f82a96d1b296368aca3 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 11 Nov 2019 18:07:47 +0100 Subject: [PATCH 7/7] doc --- core/state-machine/src/proving_backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state-machine/src/proving_backend.rs b/core/state-machine/src/proving_backend.rs index 39c2d92f666ab..2ebd82f0c43c0 100644 --- a/core/state-machine/src/proving_backend.rs +++ b/core/state-machine/src/proving_backend.rs @@ -220,7 +220,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> ProvingBackend(TrieBackend::new(recorder, root)) } - /// Extracting the gathered proof in undefined order. + /// Extracting the gathered unordered proof. pub fn extract_proof(&self) -> StorageProof { let trie_nodes = self.0.essence().backend_storage().proof_recorder .read()