From e96dfbbeb082c80150ee69d1f236a466f18b0a74 Mon Sep 17 00:00:00 2001 From: clangenb Date: Wed, 3 Jun 2020 08:58:09 +0200 Subject: [PATCH 1/6] [enclave] extracted verify worker response. --- enclave/src/lib.rs | 48 +++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index 49874f373e..f705c33d07 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -459,6 +459,33 @@ fn handle_call_worker_xt( let responses: Vec>> = worker_request(requests, node_url)?; + let update_map = verify_worker_responses(responses, header)?; + + Stf::update_storage(&mut state, update_map); + + debug!("execute STF"); + if let Err(e) = Stf::execute(&mut state, stf_call_signed, calls) { + error!("Error performing Stf::execute. Error: {:?}", e); + return Ok(()); + } + + let state_hash = state::write(state, &shard)?; + + let xt_call = [SUBSRATEE_REGISTRY_MODULE, CALL_CONFIRMED]; + let call_hash = blake2_256(&request_vec); + debug!("Call hash 0x{}", hex::encode_hex(&call_hash)); + + calls.push(OpaqueCall( + (xt_call, shard, call_hash, state_hash.encode()).encode(), + )); + + Ok(()) +} + +fn verify_worker_responses( + responses: Vec>>, + header: Header, +) -> SgxResult, Vec>> { let mut update_map = HashMap::new(); for response in responses.iter() { match response { @@ -486,26 +513,7 @@ fn handle_call_worker_xt( } } } - - Stf::update_storage(&mut state, update_map); - - debug!("execute STF"); - if let Err(e) = Stf::execute(&mut state, stf_call_signed, calls) { - error!("Error performing Stf::execute. Error: {:?}", e); - return Ok(()); - } - - let state_hash = state::write(state, &shard)?; - - let xt_call = [SUBSRATEE_REGISTRY_MODULE, CALL_CONFIRMED]; - let call_hash = blake2_256(&request_vec); - debug!("Call hash 0x{}", hex::encode_hex(&call_hash)); - - calls.push(OpaqueCall( - (xt_call, shard, call_hash, state_hash.encode()).encode(), - )); - - Ok(()) + Ok(update_map) } extern "C" { From eeca1a460e03cc87f3613ea8d71ce3e67025ca24 Mon Sep 17 00:00:00 2001 From: clangenb Date: Wed, 3 Jun 2020 08:58:52 +0200 Subject: [PATCH 2/6] [STF/enclave] whip update storages upon block import --- enclave/src/lib.rs | 11 +++++++++++ stf/src/sgx.rs | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index f705c33d07..995deaa3ab 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -328,6 +328,17 @@ pub unsafe extern "C" fn sync_chain_relay( Err(e) => return e, }; + // debug!("Update STF storage upon block import!"); + // let requests = Stf::storage_hashes_to_update_on_block() + // .into_iter() + // .map(|key| WorkerRequest::ChainStorage(key, Some(header.hash()))) + // .collect(); + // + // let responses: Vec>> = worker_request(requests, node_url)?; + // + // let update_map = verify_worker_responses(responses, header)?; + // Stf::update_storage(&mut state, update_map); + if let Err(_e) = stf_post_actions(validator, calls, xt_slice, *nonce) { return sgx_status_t::SGX_ERROR_UNEXPECTED; } diff --git a/stf/src/sgx.rs b/stf/src/sgx.rs index 1695b01af8..9e526a8e66 100644 --- a/stf/src/sgx.rs +++ b/stf/src/sgx.rs @@ -203,6 +203,13 @@ impl Stf { }; key_hashes } + + pub fn storage_hashes_to_update_on_block() -> Vec> { + // let key_hashes = Vec::new(); + // key_hashes.push(storage_value_key("dummy", "dummy")); + // key_hashes + Vec::new() + } } // get the AccountInfo key where the nonce is stored From fb6638d67ff1ae9842f9e0e0ea5f305778f495f7 Mon Sep 17 00:00:00 2001 From: clangenb Date: Wed, 3 Jun 2020 10:10:52 +0200 Subject: [PATCH 3/6] [enclave] perform state updates on all shards upon block import --- enclave/src/lib.rs | 93 ++++++++++++++++++++++++-------------------- enclave/src/state.rs | 22 ++++++++++- stf/src/sgx.rs | 2 +- 3 files changed, 72 insertions(+), 45 deletions(-) diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index 995deaa3ab..abca7194c1 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -309,7 +309,8 @@ pub unsafe extern "C" fn sync_chain_relay( Err(e) => return e, }; - for signed_block in blocks.iter() { + let mut calls = Vec::new(); + for signed_block in blocks.into_iter() { validator .check_xt_inclusion(validator.num_relays, &signed_block.block) .unwrap(); // panic can only happen if relay_id is does not exist @@ -321,23 +322,16 @@ pub unsafe extern "C" fn sync_chain_relay( error!("Block verification failed. Error : {:?}", e); return sgx_status_t::SGX_ERROR_UNEXPECTED; } - } - let calls = match scan_blocks_for_relevant_xt(blocks, node_url) { - Ok(c) => c, - Err(e) => return e, - }; + match scan_block_for_relevant_xt(&signed_block.block, node_url) { + Ok(c) => calls.extend(c.into_iter()), + Err(e) => return e, + }; - // debug!("Update STF storage upon block import!"); - // let requests = Stf::storage_hashes_to_update_on_block() - // .into_iter() - // .map(|key| WorkerRequest::ChainStorage(key, Some(header.hash()))) - // .collect(); - // - // let responses: Vec>> = worker_request(requests, node_url)?; - // - // let update_map = verify_worker_responses(responses, header)?; - // Stf::update_storage(&mut state, update_map); + if let Err(_) = update_states(signed_block.block.header, node_url) { + error!("Error performing state updates upon block import") + } + } if let Err(_e) = stf_post_actions(validator, calls, xt_slice, *nonce) { return sgx_status_t::SGX_ERROR_UNEXPECTED; @@ -346,35 +340,50 @@ pub unsafe extern "C" fn sync_chain_relay( sgx_status_t::SGX_SUCCESS } +pub fn update_states(header: Header, node_url: &[u8]) -> SgxResult<()> { + debug!("Update STF storage upon block import!"); + let requests = Stf::storage_hashes_to_update_on_block() + .into_iter() + .map(|key| WorkerRequest::ChainStorage(key, Some(header.hash()))) + .collect(); + + let responses: Vec>> = worker_request(requests, node_url)?; + let update_map = verify_worker_responses(responses, header)?; + + let shards = state::list_shards()?; + debug!("found shards: {:?}", shards); + for s in shards { + let mut state = state::load(&s)?; + Stf::update_storage(&mut state, &update_map); + state::write(state, &s)?; + } + Ok(()) +} + /// Scans blocks for extrinsics that ask the enclave to execute some actions. -pub fn scan_blocks_for_relevant_xt( - blocks: Vec>, - node_url: &[u8], -) -> SgxResult> { +pub fn scan_block_for_relevant_xt(block: &Block, node_url: &[u8]) -> SgxResult> { debug!("Scanning blocks for relevant xt"); let mut calls = Vec::::new(); - for block in blocks.iter() { - for xt_opaque in block.block.extrinsics.iter() { - if let Ok(xt) = - UncheckedExtrinsicV4::::decode(&mut xt_opaque.0.encode().as_slice()) - { - // confirm call decodes successfully as well - if xt.function.0 == [SUBSRATEE_REGISTRY_MODULE, SHIELD_FUNDS] { - if let Err(e) = handle_shield_funds_xt(&mut calls, xt) { - error!("Error performing shieldfunds. Error: {:?}", e); - } + for xt_opaque in block.extrinsics.iter() { + if let Ok(xt) = + UncheckedExtrinsicV4::::decode(&mut xt_opaque.0.encode().as_slice()) + { + // confirm call decodes successfully as well + if xt.function.0 == [SUBSRATEE_REGISTRY_MODULE, SHIELD_FUNDS] { + if let Err(e) = handle_shield_funds_xt(&mut calls, xt) { + error!("Error performing shieldfunds. Error: {:?}", e); } - }; - - if let Ok(xt) = - UncheckedExtrinsicV4::::decode(&mut xt_opaque.0.encode().as_slice()) - { - if xt.function.0 == [SUBSRATEE_REGISTRY_MODULE, CALL_WORKER] { - if let Err(e) = - handle_call_worker_xt(&mut calls, xt, block.block.header.clone(), node_url) - { - error!("Error performing worker call: Error: {:?}", e); - } + } + }; + + if let Ok(xt) = + UncheckedExtrinsicV4::::decode(&mut xt_opaque.0.encode().as_slice()) + { + if xt.function.0 == [SUBSRATEE_REGISTRY_MODULE, CALL_WORKER] { + if let Err(e) = + handle_call_worker_xt(&mut calls, xt, block.header.clone(), node_url) + { + error!("Error performing worker call: Error: {:?}", e); } } } @@ -472,7 +481,7 @@ fn handle_call_worker_xt( let update_map = verify_worker_responses(responses, header)?; - Stf::update_storage(&mut state, update_map); + Stf::update_storage(&mut state, &update_map); debug!("execute STF"); if let Err(e) = Stf::execute(&mut state, stf_call_signed, calls) { diff --git a/enclave/src/state.rs b/enclave/src/state.rs index c3fd9a4839..9e42d59084 100644 --- a/enclave/src/state.rs +++ b/enclave/src/state.rs @@ -15,6 +15,7 @@ */ +use std::fs; use std::vec::Vec; use log::*; @@ -25,8 +26,9 @@ use crate::aes; use crate::constants::{ENCRYPTED_STATE_FILE, SHARDS_PATH}; use crate::hex; use crate::io; -use base58::ToBase58; -use codec::Encode; +use crate::utils::UnwrapOrSgxErrorUnexpected; +use base58::{FromBase58, ToBase58}; +use codec::{Decode, Encode}; use sgx_externalities::SgxExternalitiesTrait; use sp_core::H256; use substratee_stf::{ShardIdentifier, State as StfState, Stf}; @@ -108,6 +110,22 @@ fn encrypt(mut state: Vec) -> SgxResult> { Ok(state) } +pub fn list_shards() -> SgxResult> { + let files = fs::read_dir(SHARDS_PATH).sgx_error()?; + let mut shards = Vec::new(); + for file in files { + let s = file + .sgx_error()? + .file_name() + .into_string() + .sgx_error()? + .from_base58() + .sgx_error()?; + shards.push(ShardIdentifier::decode(&mut s.as_slice()).sgx_error()?); + } + Ok(shards) +} + pub fn test_encrypted_state_io_works() { let path = "test_state_file.bin"; let plaintext = b"The quick brown fox jumps over the lazy dog."; diff --git a/stf/src/sgx.rs b/stf/src/sgx.rs index 9e526a8e66..e645d003b7 100644 --- a/stf/src/sgx.rs +++ b/stf/src/sgx.rs @@ -69,7 +69,7 @@ impl Stf { ext } - pub fn update_storage(ext: &mut State, map_update: HashMap, Vec>) { + pub fn update_storage(ext: &mut State, map_update: &HashMap, Vec>) { ext.execute_with(|| { map_update .iter() From e2d1ea807d152eeb4bdbca811f7b0ef66a7436c7 Mon Sep 17 00:00:00 2001 From: clangenb Date: Wed, 3 Jun 2020 10:54:24 +0200 Subject: [PATCH 4/6] [enclave] Create shard if it does not exist. Currently, the workers follow an auto-join policy for new shards. --- enclave/src/lib.rs | 10 ++++++++-- enclave/src/state.rs | 11 +++++++++++ worker/src/main.rs | 16 ---------------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index abca7194c1..9d63c00870 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -400,7 +400,10 @@ fn handle_shield_funds_xt( call, account_encrypted, amount, shard ); - let mut state = state::load(&shard)?; + let mut state = match state::exists(&shard) { + true => state::load(&shard)?, + false => Stf::init_state(), + }; debug!("decrypt the call"); let rsa_keypair = rsa3072::unseal_pair()?; @@ -469,7 +472,10 @@ fn handle_call_worker_xt( return Ok(()); } - let mut state = state::load(&shard)?; + let mut state = match state::exists(&shard) { + true => state::load(&shard)?, + false => Stf::init_state(), + }; debug!("Update STF storage!"); let requests = Stf::get_storage_hashes_to_update(&stf_call_signed) diff --git a/enclave/src/state.rs b/enclave/src/state.rs index 9e42d59084..3bd82e5286 100644 --- a/enclave/src/state.rs +++ b/enclave/src/state.rs @@ -31,6 +31,7 @@ use base58::{FromBase58, ToBase58}; use codec::{Decode, Encode}; use sgx_externalities::SgxExternalitiesTrait; use sp_core::H256; +use std::path::Path; use substratee_stf::{ShardIdentifier, State as StfState, Stf}; pub fn load(shard: &ShardIdentifier) -> SgxResult { @@ -81,6 +82,16 @@ pub fn write(state: StfState, shard: &ShardIdentifier) -> SgxResult { Ok(state_hash.into()) } +pub fn exists(shard: &ShardIdentifier) -> bool { + Path::new(&format!( + "{}/{}/{}", + SHARDS_PATH, + shard.encode().to_base58(), + ENCRYPTED_STATE_FILE + )) + .exists() +} + fn read(path: &str) -> SgxResult> { let mut bytes = match io::read(path) { Ok(vec) => match vec.len() { diff --git a/worker/src/main.rs b/worker/src/main.rs index cdd5336568..6380240d86 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -209,7 +209,6 @@ fn worker(node_url: &str, w_ip: &str, w_port: &str, mu_ra_port: &str, shard: &Sh // ------------------------------------------------------------------------ // check for required files check_files(); - ensure_shard_initialized(shard); // ------------------------------------------------------------------------ // initialize the enclave #[cfg(feature = "production")] @@ -583,21 +582,6 @@ fn get_balance(api: &Api, who: &AccountId32) -> u128 { } } -fn ensure_shard_initialized(shard: &ShardIdentifier) { - let shardenc = shard.encode().to_base58(); - if !Path::new(&format!( - "{}/{}/{}", - constants::SHARDS_PATH, - &shardenc, - constants::ENCRYPTED_STATE_FILE - )) - .exists() - { - panic!("shard {} hasn't been initialized", shardenc); - } - debug!("state file is present for shard {}", shardenc); -} - pub fn check_files() { debug!("*** Check files"); let files = vec![ From 6b56ed27bea06e98962ec16801b44987c3ff3c26 Mon Sep 17 00:00:00 2001 From: clangenb Date: Wed, 3 Jun 2020 10:59:49 +0200 Subject: [PATCH 5/6] clippy --- enclave/src/lib.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index 9d63c00870..9917b43f28 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -328,7 +328,7 @@ pub unsafe extern "C" fn sync_chain_relay( Err(e) => return e, }; - if let Err(_) = update_states(signed_block.block.header, node_url) { + if update_states(signed_block.block.header, node_url).is_err() { error!("Error performing state updates upon block import") } } @@ -400,9 +400,10 @@ fn handle_shield_funds_xt( call, account_encrypted, amount, shard ); - let mut state = match state::exists(&shard) { - true => state::load(&shard)?, - false => Stf::init_state(), + let mut state = if state::exists(&shard) { + state::load(&shard)? + } else { + Stf::init_state() }; debug!("decrypt the call"); @@ -472,9 +473,10 @@ fn handle_call_worker_xt( return Ok(()); } - let mut state = match state::exists(&shard) { - true => state::load(&shard)?, - false => Stf::init_state(), + let mut state = if state::exists(&shard) { + state::load(&shard)? + } else { + Stf::init_state() }; debug!("Update STF storage!"); From e8b85d4e5c3c384bb470cf59a03d26abcc566ef4 Mon Sep 17 00:00:00 2001 From: clangenb Date: Wed, 3 Jun 2020 11:05:56 +0200 Subject: [PATCH 6/6] [enclave] do not return an error if could not execute relevant extrinsic in stf. Only log the error. --- enclave/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index 9917b43f28..73204e509a 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -325,7 +325,7 @@ pub unsafe extern "C" fn sync_chain_relay( match scan_block_for_relevant_xt(&signed_block.block, node_url) { Ok(c) => calls.extend(c.into_iter()), - Err(e) => return e, + Err(_) => error!("Error executing relevant extrinsics"), }; if update_states(signed_block.block.header, node_url).is_err() {