From 16535d37adb1bcc7302f8bc72a687ca1ccc9c6e0 Mon Sep 17 00:00:00 2001 From: Svyatoslav Nikolsky Date: Mon, 28 May 2018 11:47:05 +0300 Subject: [PATCH 1/4] TrieBasedBackend --- Cargo.lock | 7 + substrate/client/db/src/lib.rs | 155 +---------- substrate/client/src/call_executor.rs | 105 +++++--- substrate/client/src/client.rs | 7 +- substrate/client/src/light.rs | 10 +- substrate/state-machine/Cargo.toml | 13 +- substrate/state-machine/src/backend.rs | 24 +- substrate/state-machine/src/lib.rs | 60 ++++- .../state-machine/src/proving_backend.rs | 149 +++++++++++ substrate/state-machine/src/trie_backend.rs | 252 ++++++++++++++++++ 10 files changed, 582 insertions(+), 200 deletions(-) create mode 100644 substrate/state-machine/src/proving_backend.rs create mode 100644 substrate/state-machine/src/trie_backend.rs diff --git a/Cargo.lock b/Cargo.lock index 763bbbf3db65a..486d1b2ea3c59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2304,7 +2304,14 @@ name = "substrate-state-machine" version = "0.1.0" dependencies = [ "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.1.1 (git+https://github.com/paritytech/parity.git)", "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.1.0 (git+https://github.com/paritytech/parity.git)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)", + "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)", "substrate-primitives 0.1.0", "triehash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 36968222915f1..b6ca6911cf0ec 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -17,13 +17,11 @@ //! Client backend that uses RocksDB database as storage. State is still kept in memory. extern crate substrate_client as client; -extern crate ethereum_types; extern crate kvdb_rocksdb; extern crate kvdb; extern crate hashdb; extern crate memorydb; extern crate parking_lot; -extern crate patricia_trie; extern crate substrate_state_machine as state_machine; extern crate substrate_primitives as primitives; extern crate substrate_runtime_support as runtime_support; @@ -38,22 +36,22 @@ extern crate kvdb_memorydb; use std::sync::Arc; use std::path::PathBuf; -use std::collections::HashMap; use codec::Slicable; -use ethereum_types::H256 as TrieH256; -use hashdb::{DBValue, HashDB}; +use hashdb::DBValue; use kvdb_rocksdb::{Database, DatabaseConfig}; use kvdb::{KeyValueDB, DBTransaction}; use memorydb::MemoryDB; use parking_lot::RwLock; -use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut}; use runtime_primitives::generic::BlockId; use runtime_primitives::bft::Justification; use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, Hashing, HashingFor, Zero}; use state_machine::backend::Backend as StateBackend; use state_machine::CodeExecutor; +/// DB-backed patricia trie state, transaction type is an overlay of changes to commit. +pub type DbState = state_machine::TrieBackend; + /// Database settings. pub struct DatabaseSettings { /// Cache size in bytes. If `None` default is used. @@ -300,133 +298,6 @@ impl client::backend::BlockImportOperation for BlockImport } } -struct Ephemeral<'a> { - backing: &'a KeyValueDB, - overlay: &'a mut MemoryDB, -} - -impl<'a> HashDB for Ephemeral<'a> { - fn keys(&self) -> HashMap { - self.overlay.keys() // TODO: iterate backing - } - - fn get(&self, key: &TrieH256) -> Option { - match self.overlay.raw(key) { - Some((val, i)) => { - if i <= 0 { - None - } else { - Some(val) - } - } - None => { - match self.backing.get(::columns::STATE, &key.0[..]) { - Ok(x) => x, - Err(e) => { - warn!("Failed to read from DB: {}", e); - None - } - } - } - } - } - - fn contains(&self, key: &TrieH256) -> bool { - self.get(key).is_some() - } - - fn insert(&mut self, value: &[u8]) -> TrieH256 { - self.overlay.insert(value) - } - - fn emplace(&mut self, key: TrieH256, value: DBValue) { - self.overlay.emplace(key, value) - } - - fn remove(&mut self, key: &TrieH256) { - self.overlay.remove(key) - } -} - -/// DB-backed patricia trie state, transaction type is an overlay of changes to commit. -#[derive(Clone)] -pub struct DbState { - db: Arc, - root: TrieH256, -} - -impl state_machine::Backend for DbState { - type Error = client::error::Error; - type Transaction = MemoryDB; - - fn storage(&self, key: &[u8]) -> Result>, Self::Error> { - let mut read_overlay = MemoryDB::default(); - let eph = Ephemeral { - backing: &*self.db, - overlay: &mut read_overlay, - }; - - let map_e = |e: Box| ::client::error::Error::from(format!("Trie lookup error: {}", e)); - - TrieDB::new(&eph, &self.root).map_err(map_e)? - .get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e) - } - - fn pairs(&self) -> Vec<(Vec, Vec)> { - let mut read_overlay = MemoryDB::default(); - let eph = Ephemeral { - backing: &*self.db, - overlay: &mut read_overlay, - }; - - let collect_all = || -> Result<_, Box> { - let trie = TrieDB::new(&eph, &self.root)?; - let mut v = Vec::new(); - for x in trie.iter()? { - let (key, value) = x?; - v.push((key.to_vec(), value.to_vec())); - } - - Ok(v) - }; - - match collect_all() { - Ok(v) => v, - Err(e) => { - debug!("Error extracting trie values: {}", e); - Vec::new() - } - } - } - - fn storage_root(&self, delta: I) -> ([u8; 32], MemoryDB) - where I: IntoIterator, Option>)> - { - let mut write_overlay = MemoryDB::default(); - let mut root = self.root; - { - let mut eph = Ephemeral { - backing: &*self.db, - overlay: &mut write_overlay, - }; - - let mut trie = TrieDBMut::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully - for (key, change) in delta { - let result = match change { - Some(val) => trie.insert(&key, &val), - None => trie.remove(&key), // TODO: archive mode - }; - - if let Err(e) = result { - warn!("Failed to write to trie: {}", e); - } - } - } - - (root.0.into(), write_overlay) - } -} - /// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks. /// Otherwise, trie nodes are kept only from the most recent block. pub struct Backend { @@ -522,25 +393,14 @@ impl client::backend::Backend for Backend where // special case for genesis initialization match block { - BlockId::Hash(h) if h == Default::default() => { - let mut root = TrieH256::default(); - let mut db = MemoryDB::default(); - TrieDBMut::new(&mut db, &mut root); - - return Ok(DbState { - db: self.db.clone(), - root, - }) - } + BlockId::Hash(h) if h == Default::default() => + return Ok(DbState::with_kvdb_for_genesis(self.db.clone(), ::columns::STATE)), _ => {} } self.blockchain.header(block).and_then(|maybe_hdr| maybe_hdr.map(|hdr| { let root: [u8; 32] = hdr.state_root().clone().into(); - DbState { - db: self.db.clone(), - root: root.into(), - } + DbState::with_kvdb(self.db.clone(), ::columns::STATE, root.into()) }).ok_or_else(|| client::error::ErrorKind::UnknownBlock(format!("{:?}", block)).into())) } } @@ -552,6 +412,7 @@ impl client::backend::LocalBackend for Backend wher #[cfg(test)] mod tests { + use hashdb::HashDB; use super::*; use client::backend::Backend as BTrait; use client::backend::BlockImportOperation as Op; diff --git a/substrate/client/src/call_executor.rs b/substrate/client/src/call_executor.rs index 27ae7d9af9edc..302b78d9a6078 100644 --- a/substrate/client/src/call_executor.rs +++ b/substrate/client/src/call_executor.rs @@ -17,9 +17,8 @@ use std::sync::Arc; use futures::{IntoFuture, Future}; use runtime_primitives::generic::BlockId; -use runtime_primitives::traits::Block as BlockT; +use runtime_primitives::traits::{Block as BlockT, Header as HeaderT}; use state_machine::{self, OverlayedChanges, Backend as StateBackend, CodeExecutor}; -use state_machine::backend::InMemory as InMemoryStateBackend; use backend; use blockchain::Backend as ChainBackend; @@ -49,6 +48,11 @@ pub trait CallExecutor { /// /// No changes are made. fn call_at_state(&self, state: &S, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec, S::Transaction), error::Error>; + + /// Execute a call to a contract on top of given state, gathering execution proof. + /// + /// No changes are made. + fn prove_at_state(&self, state: S, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec, Vec>), error::Error>; } /// Call executor that executes methods locally, querying all required @@ -105,6 +109,18 @@ impl CallExecutor for LocalCallExecutor call_data, ).map_err(Into::into) } + + fn prove_at_state(&self, state: S, changes: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec, Vec>), error::Error> { + state_machine::prove( + state, + changes, + &self.executor, + method, + call_data, + ) + .map(|(result, proof, _)| (result, proof)) + .map_err(Into::into) + } } impl RemoteCallExecutor { @@ -140,39 +156,43 @@ impl CallExecutor for RemoteCallExecutor fn call_at_state(&self, _state: &S, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8]) -> error::Result<(Vec, S::Transaction)> { Err(error::ErrorKind::NotAvailableOnLightClient.into()) } + + fn prove_at_state(&self, _state: S, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8]) -> Result<(Vec, Vec>), error::Error> { + Err(error::ErrorKind::NotAvailableOnLightClient.into()) + } } -/// Check remote execution proof. +/// Check remote execution proof using given backend. pub fn check_execution_proof(backend: &B, executor: &E, request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> Result where B: backend::RemoteBackend, E: CodeExecutor, Block: BlockT, + <::Header as HeaderT>::Hash: Into<[u8; 32]>, // TODO: remove when patricia_trie generic. error::Error: From<<>::State as StateBackend>::Error>, { - use runtime_primitives::traits::{Header, Hashing, HashingFor}; - - let (remote_result, remote_proof) = remote_proof; - - let remote_state = state_from_execution_proof(remote_proof); - let remote_state_root = HashingFor::::trie_root(remote_state.pairs().into_iter()); - let local_header = backend.blockchain().header(BlockId::Hash(request.block))?; - let local_header = local_header.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", request.block)))?; + let local_header = local_header.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", request.block)))?; let local_state_root = local_header.state_root().clone(); + do_check_execution_proof(local_state_root, executor, request, remote_proof) +} - if remote_state_root != local_state_root { - return Err(error::ErrorKind::InvalidExecutionProof.into()); - } +/// Check remote execution proof using given state root. +fn do_check_execution_proof(local_state_root: H, executor: &E, request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> Result + where + E: CodeExecutor, + H: Into<[u8; 32]>, // TODO: remove when patricia_trie generic. +{ + let (remote_result, remote_proof) = remote_proof; let mut changes = OverlayedChanges::default(); - let (local_result, _) = state_machine::execute( - &remote_state, + let (local_result, _) = state_machine::proof_check( + local_state_root.into(), + remote_proof, &mut changes, executor, &request.method, - &request.call_data, - )?; + &request.call_data)?; if local_result != remote_result { return Err(error::ErrorKind::InvalidExecutionProof.into()); @@ -181,28 +201,31 @@ pub fn check_execution_proof(backend: &B, executor: &E, request: &R Ok(CallResult { return_data: local_result, changes }) } -/// Convert state to execution proof. Proof is simple the whole state (temporary). -// TODO [light]: this method must be removed after trie-based proofs are landed. -pub fn state_to_execution_proof(state: &B) -> Vec> { - state.pairs().into_iter() - .flat_map(|(k, v)| ::std::iter::once(k).chain(::std::iter::once(v))) - .collect() -} - -/// Convert execution proof to in-memory state for check. Reverse function for state_to_execution_proof. -// TODO [light]: this method must be removed after trie-based proofs are landed. -fn state_from_execution_proof(proof: Vec>) -> InMemoryStateBackend { - let mut changes = Vec::new(); - let mut proof_iter = proof.into_iter(); - loop { - let key = proof_iter.next(); - let value = proof_iter.next(); - if let (Some(key), Some(value)) = (key, value) { - changes.push((key, Some(value))); - } else { - break; - } +#[cfg(test)] +mod tests { + use runtime_primitives::generic::BlockId; + use state_machine::Backend; + use test_client; + use light::RemoteCallRequest; + use super::do_check_execution_proof; + + #[test] + fn execution_proof_is_generated_and_checked() { + // prepare remote client + let remote_client = test_client::new(); + let remote_block_id = BlockId::Number(0); + let remote_block_storage_root = remote_client.state_at(&remote_block_id) + .unwrap().storage_root(::std::iter::empty()).0; + + // 'fetch' execution proof from remote node + let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap(); + + // check remote execution proof locally + let local_executor = test_client::NativeExecutor::new(); + do_check_execution_proof(remote_block_storage_root, &local_executor, &RemoteCallRequest { + block: Default::default(), + method: "authorities".into(), + call_data: vec![], + }, remote_execution_proof).unwrap(); } - - InMemoryStateBackend::default().update(changes) } diff --git a/substrate/client/src/client.rs b/substrate/client/src/client.rs index 27fb80eef58cb..4818849148ba7 100644 --- a/substrate/client/src/client.rs +++ b/substrate/client/src/client.rs @@ -232,12 +232,7 @@ impl Client where /// /// No changes are made. pub fn execution_proof(&self, id: &BlockId, method: &str, call_data: &[u8]) -> error::Result<(Vec, Vec>)> { - use call_executor::state_to_execution_proof; - - let result = self.executor.call(id, method, call_data); - let result = result?.return_data; - let proof = self.backend.state_at(*id).map(|state| state_to_execution_proof(&state))?; - Ok((result, proof)) + self.state_at(id).and_then(|state| self.executor.prove_at_state(state, &mut Default::default(), method, call_data)) } /// Set up the native execution environment to call into a native runtime code. diff --git a/substrate/client/src/light.rs b/substrate/client/src/light.rs index 6bd6210facff3..77308dbdcec7c 100644 --- a/substrate/client/src/light.rs +++ b/substrate/client/src/light.rs @@ -19,7 +19,8 @@ use std::sync::Arc; use futures::future::IntoFuture; -use state_machine::CodeExecutor; +use state_machine::{CodeExecutor, TryIntoTrieBackend as TryIntoStateTrieBackend, + TrieBackend as StateTrieBackend}; use state_machine::backend::Backend as StateBackend; use runtime_primitives::generic::BlockId; use runtime_primitives::bft::Justification; @@ -202,10 +203,17 @@ impl StateBackend for OnDemandState { } } +impl TryIntoStateTrieBackend for OnDemandState { + fn try_into_trie_backend(self) -> Option { + None + } +} + impl FetchChecker for LightDataChecker where E: CodeExecutor, B: BlockT, + <::Header as HeaderT>::Hash: Into<[u8; 32]>, // TODO: remove when patricia_trie generic. { fn check_execution_proof(&self, request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> error::Result { check_execution_proof(&*self.backend, &self.executor, request, remote_proof) diff --git a/substrate/state-machine/Cargo.toml b/substrate/state-machine/Cargo.toml index 40bb922f5ba5f..74a46764cf753 100644 --- a/substrate/state-machine/Cargo.toml +++ b/substrate/state-machine/Cargo.toml @@ -5,7 +5,16 @@ authors = ["Parity Technologies "] description = "Substrate State Machine" [dependencies] -substrate-primitives = { path = "../primitives", version = "0.1.0" } -triehash = "0.1.2" byteorder = "1.1" +ethereum-types = "0.3" hex-literal = "0.1.0" +log = "0.3" +parking_lot = "0.4" +triehash = "0.1" + +substrate-primitives = { path = "../primitives", version = "0.1.0" } + +hashdb = { git = "https://github.com/paritytech/parity.git" } +kvdb = { git = "https://github.com/paritytech/parity.git" } +memorydb = { git = "https://github.com/paritytech/parity.git" } +patricia-trie = { git = "https://github.com/paritytech/parity.git" } diff --git a/substrate/state-machine/src/backend.rs b/substrate/state-machine/src/backend.rs index da68b932feff3..fe547c5841648 100644 --- a/substrate/state-machine/src/backend.rs +++ b/substrate/state-machine/src/backend.rs @@ -19,12 +19,13 @@ use std::{error, fmt}; use std::collections::HashMap; use std::sync::Arc; +use trie_backend::{TryIntoTrieBackend, TrieBackend}; /// A state backend is used to read state data and can have changes committed /// to it. /// /// The clone operation should be cheap. -pub trait Backend: Clone { +pub trait Backend: TryIntoTrieBackend + Clone { /// An error type when fetching data is not possible. type Error: super::Error; @@ -124,3 +125,24 @@ impl Backend for InMemory { } } +impl TryIntoTrieBackend for InMemory { + fn try_into_trie_backend(self) -> Option { + use ethereum_types::H256 as TrieH256; + use memorydb::MemoryDB; + use patricia_trie::{TrieDBMut, TrieMut}; + + let mut root = TrieH256::default(); + let mut mdb = MemoryDB::default(); + { + let mut trie = TrieDBMut::new(&mut mdb, &mut root); + for (key, value) in self.inner.iter() { + if let Err(e) = trie.insert(&key, &value) { + warn!(target: "trie", "Failed to write to trie: {}", e); + return None; + } + } + } + + Some(TrieBackend::with_memorydb(mdb, root)) + } +} diff --git a/substrate/state-machine/src/lib.rs b/substrate/state-machine/src/lib.rs index 750b56a8565ac..6b3b9c702da9c 100644 --- a/substrate/state-machine/src/lib.rs +++ b/substrate/state-machine/src/lib.rs @@ -20,8 +20,19 @@ #[cfg_attr(test, macro_use)] extern crate hex_literal; + +#[macro_use] +extern crate log; + +extern crate ethereum_types; +extern crate kvdb; +extern crate hashdb; +extern crate memorydb; extern crate triehash; +extern crate patricia_trie; + extern crate byteorder; +extern crate parking_lot; use std::collections::HashMap; use std::collections::hash_map::Drain; @@ -30,10 +41,13 @@ use std::fmt; pub mod backend; mod ext; mod testing; +mod proving_backend; +mod trie_backend; pub use testing::TestExternalities; pub use ext::Ext; pub use backend::Backend; +pub use trie_backend::{TryIntoTrieBackend, TrieBackend}; /// The overlayed changes to state to be queried on top of the backend. /// @@ -93,7 +107,11 @@ impl Error for E where E: 'static + fmt::Debug + fmt::Display + Send {} #[derive(Debug, Eq, PartialEq)] pub enum ExecutionError { /// The entry `:code` doesn't exist in storage so there's no way we can execute anything. - CodeEntryDoesNotExist + CodeEntryDoesNotExist, + /// Backend is incompatible with execution proof generation process. + UnableToGenerateProof, + /// Invalid execution proof. + InvalidProof, } impl fmt::Display for ExecutionError { @@ -156,7 +174,6 @@ pub fn execute( call_data: &[u8], ) -> Result<(Vec, B::Transaction), Box> { - let result = { let mut externalities = ext::Ext::new(overlay, backend); // make a copy. @@ -184,6 +201,45 @@ pub fn execute( } } +/// Prove execution using the given state backend, overlayed changes, and call executor. +/// Produces a state-backend-specific "transaction" which can be used to apply the changes +/// to the backing store, such as the disk. +/// Execution proof is the set of all 'touched' storage DBValues from the backend. +/// +/// On an error, no prospective changes are written to the overlay. +/// +/// Note: changes to code will be in place if this call is made again. For running partial +/// blocks (e.g. a transaction at a time), ensure a different method is used. +pub fn prove( + backend: B, + overlay: &mut OverlayedChanges, + exec: &Exec, + method: &str, + call_data: &[u8], +) -> Result<(Vec, Vec>, ::Transaction), Box> +{ + let trie_backend = backend.try_into_trie_backend() + .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; + let proving_backend = proving_backend::ProvingBackend::new(trie_backend); + let (result, transaction) = execute(&proving_backend, overlay, exec, method, call_data)?; + let proof = proving_backend.extract_proof(); + Ok((result, proof, transaction)) +} + +/// Check execution proof, generated by `prove` call. +pub fn proof_check( + root: [u8; 32], + proof: Vec>, + overlay: &mut OverlayedChanges, + exec: &Exec, + method: &str, + call_data: &[u8], +) -> Result<(Vec, memorydb::MemoryDB), Box> +{ + let backend = proving_backend::create_proof_check_backend(root.into(), proof)?; + execute(&backend, overlay, exec, method, call_data) +} + #[cfg(test)] mod tests { use super::*; diff --git a/substrate/state-machine/src/proving_backend.rs b/substrate/state-machine/src/proving_backend.rs new file mode 100644 index 0000000000000..1f0bf69d27269 --- /dev/null +++ b/substrate/state-machine/src/proving_backend.rs @@ -0,0 +1,149 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Proving state machine backend. + +use std::collections::HashSet; +use parking_lot::Mutex; +use ethereum_types::H256 as TrieH256; +use hashdb::{HashDB, DBValue}; +use memorydb::MemoryDB; +use patricia_trie::{TrieDB, TrieError, Trie}; +use trie_backend::{TrieBackend, TrieLookupRecorder, Ephemeral}; +use {Error, ExecutionError, Backend, TryIntoTrieBackend}; + +/// Patricia trie-based backend which also tracks all touched storage trie values. +/// These can be sent to remote node and used as a proof of execution. +#[derive(Clone)] +pub struct ProvingBackend { + backend: TrieBackend, + proof_recorder: ProofRecorder, +} + +impl ProvingBackend { + /// Create new proving backend. + pub fn new(backend: TrieBackend) -> Self { + ProvingBackend { + backend, + proof_recorder: ProofRecorder::default(), + } + } + + /// Consume the backend, extracting the gathered proof in lexicographical order + /// by value. + pub fn extract_proof(self) -> Vec> { + self.proof_recorder.proof.into_inner().into_iter() + .map(DBValue::into_vec) + .collect() + } +} + +impl Backend for ProvingBackend { + type Error = String; + type Transaction = MemoryDB; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + let mut read_overlay = MemoryDB::default(); + let eph = Ephemeral::new( + self.backend.backend_storage(), + &mut read_overlay, + &self.proof_recorder, + ); + + let map_e = |e: Box| format!("Trie lookup error: {}", e); + + TrieDB::new(&eph, &self.backend.root()).map_err(map_e)? + .get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.backend.pairs() + } + + fn storage_root(&self, delta: I) -> ([u8; 32], MemoryDB) + where I: IntoIterator, Option>)> + { + self.backend.storage_root(delta) + } +} + +impl TryIntoTrieBackend for ProvingBackend { + fn try_into_trie_backend(self) -> Option { + None + } +} + +#[derive(Debug, Default)] +struct ProofRecorder { + proof: Mutex>, +} + +impl TrieLookupRecorder for ProofRecorder { + fn record(&self, value: &DBValue) { + self.proof.lock().insert(value.clone()); + } +} + +impl Clone for ProofRecorder { + fn clone(&self) -> Self { + ProofRecorder { + proof: Mutex::new(self.proof.lock().clone()), + } + } +} + +/// Create proof check backend. +pub fn create_proof_check_backend(root: TrieH256, proof: Vec>) -> Result> { + let mut db = MemoryDB::new(); + for item in proof { + db.insert(&item); + } + + if !db.contains(&root) { + return Err(Box::new(ExecutionError::InvalidProof) as Box); + } + + + Ok(TrieBackend::with_memorydb(db, root)) +} + +#[cfg(test)] +mod tests { + use backend::{InMemory}; + use super::*; + + #[test] + fn proof_recorded_and_checked() { + let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); + let in_memory = InMemory::default(); + let in_memory = in_memory.update(contents); + let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; + (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); + + let trie = in_memory.try_into_trie_backend().unwrap(); + let trie_root = trie.storage_root(::std::iter::empty()).0; + assert_eq!(in_memory_root, trie_root); + (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); + + let proving = ProvingBackend::new(trie); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); + + let proof = proving.extract_proof(); + + let proof_check = create_proof_check_backend(in_memory_root.into(), proof).unwrap(); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); + } +} diff --git a/substrate/state-machine/src/trie_backend.rs b/substrate/state-machine/src/trie_backend.rs new file mode 100644 index 0000000000000..e31bff2f3ede5 --- /dev/null +++ b/substrate/state-machine/src/trie_backend.rs @@ -0,0 +1,252 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Trie-based state machine backend. + +use std::collections::HashMap; +use std::sync::Arc; +use ethereum_types::H256 as TrieH256; +use hashdb::{DBValue, HashDB}; +use kvdb::KeyValueDB; +use memorydb::MemoryDB; +use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut}; +use {Backend}; + +/// Try convert into trie-based backend. +pub trait TryIntoTrieBackend { + /// Try to convert self into trie backend. + fn try_into_trie_backend(self) -> Option; +} + +/// Patricia trie-based backend. Transaction type is an overlay of changes to commit. +#[derive(Clone)] +pub struct TrieBackend { + storage: TrieBackendStorage, + root: TrieH256, +} + +impl TrieBackend { + /// Create new trie-based backend. + pub fn with_kvdb(db: Arc, storage_column: Option, root: TrieH256) -> Self { + TrieBackend { + storage: TrieBackendStorage::KeyValueDb(db, storage_column), + root, + } + } + + /// Create new trie-based backend for genesis block. + pub fn with_kvdb_for_genesis(db: Arc, storage_column: Option) -> Self { + let mut root = TrieH256::default(); + let mut mdb = MemoryDB::default(); + TrieDBMut::new(&mut mdb, &mut root); + + Self::with_kvdb(db, storage_column, root) + } + + /// Create new trie-based backend backed by MemoryDb storage. + pub fn with_memorydb(db: MemoryDB, root: TrieH256) -> Self { + // TODO: check that root is a part of db??? + TrieBackend { + storage: TrieBackendStorage::MemoryDb(db), + root, + } + } + + /// Get backend storage reference. + pub fn backend_storage(&self) -> &TrieBackendStorage { + &self.storage + } + + /// Get trie root. + pub fn root(&self) -> &TrieH256 { + &self.root + } +} + +impl Backend for TrieBackend { + type Error = String; + type Transaction = MemoryDB; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + let mut read_overlay = MemoryDB::default(); + let eph = Ephemeral { + storage: &self.storage, + overlay: &mut read_overlay, + recorder: &DummyRecorder, + }; + + let map_e = |e: Box| format!("Trie lookup error: {}", e); + + TrieDB::new(&eph, &self.root).map_err(map_e)? + .get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + let mut read_overlay = MemoryDB::default(); + let eph = Ephemeral { + storage: &self.storage, + overlay: &mut read_overlay, + recorder: &DummyRecorder, + }; + + let collect_all = || -> Result<_, Box> { + let trie = TrieDB::new(&eph, &self.root)?; + let mut v = Vec::new(); + for x in trie.iter()? { + let (key, value) = x?; + v.push((key.to_vec(), value.to_vec())); + } + + Ok(v) + }; + + match collect_all() { + Ok(v) => v, + Err(e) => { + debug!(target: "trie", "Error extracting trie values: {}", e); + Vec::new() + } + } + } + + fn storage_root(&self, delta: I) -> ([u8; 32], MemoryDB) + where I: IntoIterator, Option>)> + { + let mut write_overlay = MemoryDB::default(); + let mut root = self.root; + { + let mut eph = Ephemeral { + storage: &self.storage, + overlay: &mut write_overlay, + recorder: &DummyRecorder, + }; + + let mut trie = TrieDBMut::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully + for (key, change) in delta { + let result = match change { + Some(val) => trie.insert(&key, &val), + None => trie.remove(&key), // TODO: archive mode + }; + + if let Err(e) = result { + warn!(target: "trie", "Failed to write to trie: {}", e); + } + } + } + + (root.0.into(), write_overlay) + } +} + +impl TryIntoTrieBackend for TrieBackend { + fn try_into_trie_backend(self) -> Option { + Some(self) + } +} + +pub struct Ephemeral<'a, R: 'a> { + storage: &'a TrieBackendStorage, + overlay: &'a mut MemoryDB, + recorder: &'a R, +} + +pub trait TrieLookupRecorder: Clone { + fn record(&self, value: &DBValue); +} + +#[derive(Clone)] +struct DummyRecorder; + +impl TrieLookupRecorder for DummyRecorder { + fn record(&self, _value: &DBValue) {} +} + +impl<'a, R> Ephemeral<'a, R> { + pub fn new(storage: &'a TrieBackendStorage, overlay: &'a mut MemoryDB, recorder: &'a R) -> Self { + Ephemeral { + storage, + overlay, + recorder, + } + } +} + +impl<'a, R> HashDB for Ephemeral<'a, R> where R: 'a + TrieLookupRecorder + Send + Sync { + fn keys(&self) -> HashMap { + self.overlay.keys() // TODO: iterate backing + } + + fn get(&self, key: &TrieH256) -> Option { + match self.overlay.raw(key) { + Some((val, i)) => { + if i <= 0 { + None + } else { + Some(val) + } + } + None => { + match self.storage.get(&key.0[..]) { + Ok(Some(x)) => { + self.recorder.record(&x); + Some(x) + }, + Ok(None) => None, + Err(e) => { + warn!(target: "trie", "Failed to read from DB: {}", e); + None + } + } + } + } + } + + fn contains(&self, key: &TrieH256) -> bool { + self.get(key).is_some() + } + + fn insert(&mut self, value: &[u8]) -> TrieH256 { + self.overlay.insert(value) + } + + fn emplace(&mut self, key: TrieH256, value: DBValue) { + self.overlay.emplace(key, value) + } + + fn remove(&mut self, key: &TrieH256) { + self.overlay.remove(key) + } +} + +#[derive(Clone)] +pub enum TrieBackendStorage { + /// Key value db + storage column. + KeyValueDb(Arc, Option), + /// Hash db. + MemoryDb(MemoryDB), +} + +impl TrieBackendStorage { + pub fn get(&self, key: &[u8]) -> Result, String> { + match *self { + TrieBackendStorage::KeyValueDb(ref db, storage_column) => + db.get(storage_column, key) + .map_err(|e| format!("Trie lookup error: {}", e)), + TrieBackendStorage::MemoryDb(ref db) => + Ok(db.get(&TrieH256::from_slice(key))), + } + } +} From 94fb5db48a4379190d9e97ae35c4374a64f9d7be Mon Sep 17 00:00:00 2001 From: Svyatoslav Nikolsky Date: Fri, 1 Jun 2018 14:27:21 +0300 Subject: [PATCH 2/4] trie tests --- substrate/state-machine/src/lib.rs | 39 ++++++++++++ .../state-machine/src/proving_backend.rs | 35 +++++++++++ substrate/state-machine/src/trie_backend.rs | 60 +++++++++++++++++++ 3 files changed, 134 insertions(+) diff --git a/substrate/state-machine/src/lib.rs b/substrate/state-machine/src/lib.rs index 6b3b9c702da9c..af643d78971cc 100644 --- a/substrate/state-machine/src/lib.rs +++ b/substrate/state-machine/src/lib.rs @@ -246,6 +246,22 @@ mod tests { use super::backend::InMemory; use super::ext::Ext; + struct DummyCodeExecutor; + + impl CodeExecutor for DummyCodeExecutor { + type Error = u8; + + fn call( + &self, + ext: &mut E, + _code: &[u8], + _method: &str, + _data: &[u8], + ) -> Result, Self::Error> { + Ok(vec![ext.storage(b"value1").unwrap()[0] + ext.storage(b"value2").unwrap()[0]]) + } + } + #[test] fn overlayed_storage_works() { let mut overlayed = OverlayedChanges::default(); @@ -304,4 +320,27 @@ mod tests { const ROOT: [u8; 32] = hex!("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3"); assert_eq!(ext.storage_root(), ROOT); } + + #[test] + fn execute_works() { + assert_eq!(execute(&trie_backend::tests::test_trie(), + &mut Default::default(), &DummyCodeExecutor, "test", &[]).unwrap().0, vec![66]); + } + + #[test] + fn prove_and_proof_check_works() { + // fetch execution proof from 'remote' full node + let remote_backend = trie_backend::tests::test_trie(); + let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let (remote_result, remote_proof, _) = prove(remote_backend, + &mut Default::default(), &DummyCodeExecutor, "test", &[]).unwrap(); + + // check proof locally + let (local_result, _) = proof_check(remote_root, remote_proof, + &mut Default::default(), &DummyCodeExecutor, "test", &[]).unwrap(); + + // check that both results are correct + assert_eq!(remote_result, vec![66]); + assert_eq!(remote_result, local_result); + } } diff --git a/substrate/state-machine/src/proving_backend.rs b/substrate/state-machine/src/proving_backend.rs index 1f0bf69d27269..37fb52fc40be3 100644 --- a/substrate/state-machine/src/proving_backend.rs +++ b/substrate/state-machine/src/proving_backend.rs @@ -123,8 +123,43 @@ pub fn create_proof_check_backend(root: TrieH256, proof: Vec>) -> Result #[cfg(test)] mod tests { use backend::{InMemory}; + use trie_backend::tests::test_trie; use super::*; + fn test_proving() -> ProvingBackend { + ProvingBackend::new(test_trie()) + } + + #[test] + fn proof_is_empty_until_value_is_read() { + assert!(test_proving().extract_proof().is_empty()); + } + + #[test] + fn proof_is_non_empty_after_value_is_read() { + let backend = test_proving(); + assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec())); + assert!(!backend.extract_proof().is_empty()); + } + + #[test] + fn proof_is_invalid_when_does_not_contains_root() { + assert!(create_proof_check_backend(1.into(), vec![]).is_err()); + } + + #[test] + fn passes_throgh_backend_calls() { + let trie_backend = test_trie(); + let proving_backend = test_proving(); + assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); + assert_eq!(trie_backend.pairs(), proving_backend.pairs()); + + let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty()); + let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty()); + assert_eq!(trie_root, proving_root); + assert_eq!(trie_mdb.drain(), proving_mdb.drain()); + } + #[test] fn proof_recorded_and_checked() { let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); diff --git a/substrate/state-machine/src/trie_backend.rs b/substrate/state-machine/src/trie_backend.rs index e31bff2f3ede5..f973debc21fbc 100644 --- a/substrate/state-machine/src/trie_backend.rs +++ b/substrate/state-machine/src/trie_backend.rs @@ -250,3 +250,63 @@ impl TrieBackendStorage { } } } + +#[cfg(test)] +pub mod tests { + use super::*; + + fn test_db() -> (MemoryDB, TrieH256) { + let mut root = TrieH256::default(); + let mut mdb = MemoryDB::default(); + { + let mut trie = TrieDBMut::new(&mut mdb, &mut root); + trie.insert(b"key", b"value").unwrap(); + trie.insert(b"value1", &[42]).unwrap(); + trie.insert(b"value2", &[24]).unwrap(); + trie.insert(b":code", b"return 42").unwrap(); + } + (mdb, root) + } + + pub fn test_trie() -> TrieBackend { + let (mdb, root) = test_db(); + TrieBackend::with_memorydb(mdb, root) + } + + #[test] + fn read_from_storage_returns_some() { + assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec())); + } + + #[test] + fn read_from_storage_returns_none() { + assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None); + } + + #[test] + fn pairs_are_not_empty_on_non_empty_storage() { + assert!(!test_trie().pairs().is_empty()); + } + + #[test] + fn pairs_are_empty_on_empty_storage() { + assert!(TrieBackend::with_memorydb(MemoryDB::new(), Default::default()).pairs().is_empty()); + } + + #[test] + fn storage_root_is_non_default() { + assert!(test_trie().storage_root(::std::iter::empty()).0 != [0; 32]); + } + + #[test] + fn storage_root_transaction_is_empty() { + assert!(test_trie().storage_root(::std::iter::empty()).1.drain().is_empty()); + } + + #[test] + fn storage_root_transaction_is_non_empty() { + let (new_root, mut tx) = test_trie().storage_root(vec![(b"new-key".to_vec(), Some(b"new-value".to_vec()))]); + assert!(!tx.drain().is_empty()); + assert!(new_root != test_trie().storage_root(::std::iter::empty()).0); + } +} From 508055a519e1dc81db72ef7aeed118444acb6f2c Mon Sep 17 00:00:00 2001 From: Svyatoslav Nikolsky Date: Mon, 18 Jun 2018 11:55:19 +0300 Subject: [PATCH 3/4] redunant return_value removed --- substrate/client/src/call_executor.rs | 12 +++--------- substrate/client/src/light.rs | 4 ++-- substrate/network/src/message.rs | 2 -- substrate/network/src/on_demand.rs | 9 ++++----- substrate/network/src/protocol.rs | 8 ++++---- 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/substrate/client/src/call_executor.rs b/substrate/client/src/call_executor.rs index 302b78d9a6078..cd67ec4d97722 100644 --- a/substrate/client/src/call_executor.rs +++ b/substrate/client/src/call_executor.rs @@ -163,7 +163,7 @@ impl CallExecutor for RemoteCallExecutor } /// Check remote execution proof using given backend. -pub fn check_execution_proof(backend: &B, executor: &E, request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> Result +pub fn check_execution_proof(backend: &B, executor: &E, request: &RemoteCallRequest, remote_proof: Vec>) -> Result where B: backend::RemoteBackend, E: CodeExecutor, @@ -178,13 +178,11 @@ pub fn check_execution_proof(backend: &B, executor: &E, request: &R } /// Check remote execution proof using given state root. -fn do_check_execution_proof(local_state_root: H, executor: &E, request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> Result +fn do_check_execution_proof(local_state_root: H, executor: &E, request: &RemoteCallRequest, remote_proof: Vec>) -> Result where E: CodeExecutor, H: Into<[u8; 32]>, // TODO: remove when patricia_trie generic. { - let (remote_result, remote_proof) = remote_proof; - let mut changes = OverlayedChanges::default(); let (local_result, _) = state_machine::proof_check( local_state_root.into(), @@ -194,10 +192,6 @@ fn do_check_execution_proof(local_state_root: H, executor: &E, request: &R &request.method, &request.call_data)?; - if local_result != remote_result { - return Err(error::ErrorKind::InvalidExecutionProof.into()); - } - Ok(CallResult { return_data: local_result, changes }) } @@ -218,7 +212,7 @@ mod tests { .unwrap().storage_root(::std::iter::empty()).0; // 'fetch' execution proof from remote node - let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap(); + let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap().1; // check remote execution proof locally let local_executor = test_client::NativeExecutor::new(); diff --git a/substrate/client/src/light.rs b/substrate/client/src/light.rs index 77308dbdcec7c..c9c28981af8e5 100644 --- a/substrate/client/src/light.rs +++ b/substrate/client/src/light.rs @@ -55,7 +55,7 @@ pub trait Fetcher: Send + Sync { /// Light client remote data checker. pub trait FetchChecker: Send + Sync { /// Check remote method execution proof. - fn check_execution_proof(&self, request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> error::Result; + fn check_execution_proof(&self, request: &RemoteCallRequest, remote_proof: Vec>) -> error::Result; } /// Light client backend. @@ -215,7 +215,7 @@ impl FetchChecker for LightDataChecker B: BlockT, <::Header as HeaderT>::Hash: Into<[u8; 32]>, // TODO: remove when patricia_trie generic. { - fn check_execution_proof(&self, request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> error::Result { + fn check_execution_proof(&self, request: &RemoteCallRequest, remote_proof: Vec>) -> error::Result { check_execution_proof(&*self.backend, &self.executor, request, remote_proof) } } diff --git a/substrate/network/src/message.rs b/substrate/network/src/message.rs index 1b1b3c1424de0..0fac00b928e56 100644 --- a/substrate/network/src/message.rs +++ b/substrate/network/src/message.rs @@ -162,8 +162,6 @@ pub enum Direction { pub struct RemoteCallResponse { /// Id of a request this response was made for. pub id: RequestId, - /// Method return value. - pub value: Vec, /// Execution proof. pub proof: Vec>, } diff --git a/substrate/network/src/on_demand.rs b/substrate/network/src/on_demand.rs index e36bf5deca286..67c5adf923da0 100644 --- a/substrate/network/src/on_demand.rs +++ b/substrate/network/src/on_demand.rs @@ -160,7 +160,7 @@ impl OnDemandService for OnDemand where fn on_remote_response(&self, io: &mut SyncIo, peer: PeerId, response: message::RemoteCallResponse) { let mut core = self.core.lock(); match core.remove(peer, response.id) { - Some(request) => match self.checker.check_execution_proof(&request.request, (response.value, response.proof)) { + Some(request) => match self.checker.check_execution_proof(&request.request, response.proof) { Ok(response) => { // we do not bother if receiver has been dropped already let _ = request.sender.send(response); @@ -312,10 +312,10 @@ mod tests { } impl FetchChecker for DummyFetchChecker { - fn check_execution_proof(&self, _request: &RemoteCallRequest, remote_proof: (Vec, Vec>)) -> client::error::Result { + fn check_execution_proof(&self, _request: &RemoteCallRequest, _remote_proof: Vec>) -> client::error::Result { match self.ok { true => Ok(client::CallResult { - return_data: remote_proof.0, + return_data: vec![42], changes: Default::default(), }), false => Err(client::error::ErrorKind::Backend("Test error".into()).into()), @@ -338,7 +338,6 @@ mod tests { fn receive_response(on_demand: &OnDemand, network: &mut TestIo, peer: PeerId, id: message::RequestId) { on_demand.on_remote_response(network, peer, message::RemoteCallResponse { id: id, - value: vec![1], proof: vec![vec![2]], }); } @@ -431,7 +430,7 @@ mod tests { let response = on_demand.remote_call(RemoteCallRequest { block: Default::default(), method: "test".into(), call_data: vec![] }); let thread = ::std::thread::spawn(move || { let result = response.wait().unwrap(); - assert_eq!(result.return_data, vec![1]); + assert_eq!(result.return_data, vec![42]); }); receive_response(&*on_demand, &mut network, 0, 0); diff --git a/substrate/network/src/protocol.rs b/substrate/network/src/protocol.rs index d49b7be1c8139..f5ec034a21450 100644 --- a/substrate/network/src/protocol.rs +++ b/substrate/network/src/protocol.rs @@ -506,17 +506,17 @@ impl Protocol where fn on_remote_call_request(&self, io: &mut SyncIo, peer_id: PeerId, request: message::RemoteCallRequest) { trace!(target: "sync", "Remote request {} from {} ({} at {})", request.id, peer_id, request.method, request.block); - let (value, proof) = match self.chain.execution_proof(&request.block, &request.method, &request.data) { - Ok((value, proof)) => (value, proof), + let proof = match self.chain.execution_proof(&request.block, &request.method, &request.data) { + Ok((_, proof)) => proof, Err(error) => { trace!(target: "sync", "Remote request {} from {} ({} at {}) failed with: {}", request.id, peer_id, request.method, request.block, error); - (Default::default(), Default::default()) + Default::default() }, }; self.send_message(io, peer_id, GenericMessage::RemoteCallResponse(message::RemoteCallResponse { - id: request.id, value, proof, + id: request.id, proof, })); } From bdd236360ebe263ea8a52621e8b34660c85ba017 Mon Sep 17 00:00:00 2001 From: Svyatoslav Nikolsky Date: Mon, 18 Jun 2018 16:22:39 +0300 Subject: [PATCH 4/4] use Trie::get_with to record trie proofs --- substrate/state-machine/src/backend.rs | 4 +- .../state-machine/src/proving_backend.rs | 43 +++++------------- substrate/state-machine/src/trie_backend.rs | 44 +++++-------------- 3 files changed, 25 insertions(+), 66 deletions(-) diff --git a/substrate/state-machine/src/backend.rs b/substrate/state-machine/src/backend.rs index fe547c5841648..cbd2fc6ccf192 100644 --- a/substrate/state-machine/src/backend.rs +++ b/substrate/state-machine/src/backend.rs @@ -24,8 +24,8 @@ use trie_backend::{TryIntoTrieBackend, TrieBackend}; /// A state backend is used to read state data and can have changes committed /// to it. /// -/// The clone operation should be cheap. -pub trait Backend: TryIntoTrieBackend + Clone { +/// The clone operation (if implemented) should be cheap. +pub trait Backend: TryIntoTrieBackend { /// An error type when fetching data is not possible. type Error: super::Error; diff --git a/substrate/state-machine/src/proving_backend.rs b/substrate/state-machine/src/proving_backend.rs index 37fb52fc40be3..688dba6ef23bb 100644 --- a/substrate/state-machine/src/proving_backend.rs +++ b/substrate/state-machine/src/proving_backend.rs @@ -16,21 +16,19 @@ //! Proving state machine backend. -use std::collections::HashSet; -use parking_lot::Mutex; +use std::cell::RefCell; use ethereum_types::H256 as TrieH256; -use hashdb::{HashDB, DBValue}; +use hashdb::HashDB; use memorydb::MemoryDB; -use patricia_trie::{TrieDB, TrieError, Trie}; -use trie_backend::{TrieBackend, TrieLookupRecorder, Ephemeral}; +use patricia_trie::{TrieDB, TrieError, Trie, Recorder}; +use trie_backend::{TrieBackend, Ephemeral}; use {Error, ExecutionError, Backend, TryIntoTrieBackend}; /// Patricia trie-based backend which also tracks all touched storage trie values. /// These can be sent to remote node and used as a proof of execution. -#[derive(Clone)] pub struct ProvingBackend { backend: TrieBackend, - proof_recorder: ProofRecorder, + proof_recorder: RefCell, } impl ProvingBackend { @@ -38,15 +36,16 @@ impl ProvingBackend { pub fn new(backend: TrieBackend) -> Self { ProvingBackend { backend, - proof_recorder: ProofRecorder::default(), + proof_recorder: RefCell::new(Recorder::new()), } } /// Consume the backend, extracting the gathered proof in lexicographical order /// by value. pub fn extract_proof(self) -> Vec> { - self.proof_recorder.proof.into_inner().into_iter() - .map(DBValue::into_vec) + self.proof_recorder.into_inner().drain() + .into_iter() + .map(|n| n.data.to_vec()) .collect() } } @@ -60,13 +59,14 @@ impl Backend for ProvingBackend { let eph = Ephemeral::new( self.backend.backend_storage(), &mut read_overlay, - &self.proof_recorder, ); let map_e = |e: Box| format!("Trie lookup error: {}", e); + let mut proof_recorder = self.proof_recorder.try_borrow_mut() + .expect("only fails when already borrowed; storage() is non-reentrant; qed"); TrieDB::new(&eph, &self.backend.root()).map_err(map_e)? - .get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e) + .get_with(key, &mut *proof_recorder).map(|x| x.map(|val| val.to_vec())).map_err(map_e) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -86,25 +86,6 @@ impl TryIntoTrieBackend for ProvingBackend { } } -#[derive(Debug, Default)] -struct ProofRecorder { - proof: Mutex>, -} - -impl TrieLookupRecorder for ProofRecorder { - fn record(&self, value: &DBValue) { - self.proof.lock().insert(value.clone()); - } -} - -impl Clone for ProofRecorder { - fn clone(&self) -> Self { - ProofRecorder { - proof: Mutex::new(self.proof.lock().clone()), - } - } -} - /// Create proof check backend. pub fn create_proof_check_backend(root: TrieH256, proof: Vec>) -> Result> { let mut db = MemoryDB::new(); diff --git a/substrate/state-machine/src/trie_backend.rs b/substrate/state-machine/src/trie_backend.rs index f973debc21fbc..5a9af8f57b8f2 100644 --- a/substrate/state-machine/src/trie_backend.rs +++ b/substrate/state-machine/src/trie_backend.rs @@ -85,7 +85,6 @@ impl Backend for TrieBackend { let eph = Ephemeral { storage: &self.storage, overlay: &mut read_overlay, - recorder: &DummyRecorder, }; let map_e = |e: Box| format!("Trie lookup error: {}", e); @@ -99,7 +98,6 @@ impl Backend for TrieBackend { let eph = Ephemeral { storage: &self.storage, overlay: &mut read_overlay, - recorder: &DummyRecorder, }; let collect_all = || -> Result<_, Box> { @@ -131,7 +129,6 @@ impl Backend for TrieBackend { let mut eph = Ephemeral { storage: &self.storage, overlay: &mut write_overlay, - recorder: &DummyRecorder, }; let mut trie = TrieDBMut::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully @@ -157,34 +154,21 @@ impl TryIntoTrieBackend for TrieBackend { } } -pub struct Ephemeral<'a, R: 'a> { +pub struct Ephemeral<'a> { storage: &'a TrieBackendStorage, overlay: &'a mut MemoryDB, - recorder: &'a R, } -pub trait TrieLookupRecorder: Clone { - fn record(&self, value: &DBValue); -} - -#[derive(Clone)] -struct DummyRecorder; - -impl TrieLookupRecorder for DummyRecorder { - fn record(&self, _value: &DBValue) {} -} - -impl<'a, R> Ephemeral<'a, R> { - pub fn new(storage: &'a TrieBackendStorage, overlay: &'a mut MemoryDB, recorder: &'a R) -> Self { +impl<'a> Ephemeral<'a> { + pub fn new(storage: &'a TrieBackendStorage, overlay: &'a mut MemoryDB) -> Self { Ephemeral { storage, overlay, - recorder, } } } -impl<'a, R> HashDB for Ephemeral<'a, R> where R: 'a + TrieLookupRecorder + Send + Sync { +impl<'a> HashDB for Ephemeral<'a> { fn keys(&self) -> HashMap { self.overlay.keys() // TODO: iterate backing } @@ -198,19 +182,13 @@ impl<'a, R> HashDB for Ephemeral<'a, R> where R: 'a + TrieLookupRecorder + Send Some(val) } } - None => { - match self.storage.get(&key.0[..]) { - Ok(Some(x)) => { - self.recorder.record(&x); - Some(x) - }, - Ok(None) => None, - Err(e) => { - warn!(target: "trie", "Failed to read from DB: {}", e); - None - } - } - } + None => match self.storage.get(&key.0[..]) { + Ok(x) => x, + Err(e) => { + warn!(target: "trie", "Failed to read from DB: {}", e); + None + }, + }, } }