From b1561e7a77c32a469eb49c8fb634b30cfd217964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 19 Nov 2020 13:57:58 +0100 Subject: [PATCH 01/58] primitives/runtime: initial changes on supporting multiple Justifications --- Cargo.lock | 2 + client/api/src/backend.rs | 12 +- client/api/src/client.rs | 4 +- client/api/src/in_mem.rs | 26 ++-- client/consensus/aura/src/lib.rs | 4 +- client/consensus/babe/src/lib.rs | 4 +- client/consensus/babe/src/tests.rs | 2 +- .../manual-seal/src/finalize_block.rs | 4 +- client/consensus/manual-seal/src/lib.rs | 4 +- client/consensus/manual-seal/src/rpc.rs | 8 +- client/consensus/pow/src/lib.rs | 4 +- client/db/Cargo.toml | 1 + client/db/src/changes_tries_storage.rs | 5 +- client/db/src/lib.rs | 18 +-- client/finality-grandpa/src/environment.rs | 13 +- client/finality-grandpa/src/finality_proof.rs | 131 ++++++++++-------- client/finality-grandpa/src/import.rs | 18 ++- client/finality-grandpa/src/tests.rs | 2 +- client/light/src/backend.rs | 8 +- client/light/src/blockchain.rs | 4 +- client/network/src/behaviour.rs | 4 +- client/network/src/block_requests.rs | 13 +- client/network/src/gossip/tests.rs | 2 +- client/network/src/protocol.rs | 4 +- client/network/src/protocol/message.rs | 4 +- client/network/src/protocol/sync.rs | 4 +- client/network/src/service/tests.rs | 2 +- client/network/test/src/lib.rs | 12 +- client/service/src/client/client.rs | 16 +-- client/service/test/Cargo.toml | 3 + client/service/test/src/client/mod.rs | 5 +- primitives/blockchain/src/backend.rs | 4 +- .../consensus/common/src/block_import.rs | 6 +- .../consensus/common/src/import_queue.rs | 8 +- .../common/src/import_queue/basic_queue.rs | 12 +- primitives/runtime/src/generic/block.rs | 4 +- primitives/runtime/src/lib.rs | 5 +- test-utils/client/src/client_ext.rs | 12 +- 38 files changed, 220 insertions(+), 174 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 446b4442cd0ef..4ef395ba152b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6602,6 +6602,7 @@ dependencies = [ "sp-consensus", "sp-core", "sp-database", + "sp-finality-grandpa", "sp-keyring", "sp-runtime", "sp-state-machine", @@ -7418,6 +7419,7 @@ dependencies = [ "sp-consensus", "sp-core", "sp-externalities", + "sp-finality-grandpa", "sp-panic-handler", "sp-runtime", "sp-state-machine", diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 47fec977f5e82..e665df4fe81c0 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use std::collections::{HashMap, HashSet}; use sp_core::ChangesTrieConfigurationRange; use sp_core::offchain::{OffchainStorage,storage::OffchainOverlayedChanges}; -use sp_runtime::{generic::BlockId, Justification, Storage}; +use sp_runtime::{generic::BlockId, Justifications, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor}; use sp_state_machine::{ ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction, @@ -148,7 +148,7 @@ pub trait BlockImportOperation { &mut self, header: Block::Header, body: Option>, - justification: Option, + justification: Option, state: NewBlockState, ) -> sp_blockchain::Result<()>; @@ -195,7 +195,7 @@ pub trait BlockImportOperation { fn mark_finalized( &mut self, id: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()>; /// Mark a block as new head. If both block import and set head are specified, set head /// overrides block import's best block rule. @@ -226,7 +226,7 @@ pub trait Finalizer> { &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()>; @@ -247,7 +247,7 @@ pub trait Finalizer> { fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()>; @@ -429,7 +429,7 @@ pub trait Backend: AuxStore + Send + Sync { fn finalize_block( &self, block: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()>; /// Returns reference to blockchain backend. diff --git a/client/api/src/client.rs b/client/api/src/client.rs index f97daa487638f..53ebbe2d78837 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -21,7 +21,7 @@ use sp_core::storage::StorageKey; use sp_runtime::{ traits::{Block as BlockT, NumberFor}, generic::{BlockId, SignedBlock}, - Justification, + Justifications, }; use sp_consensus::BlockOrigin; @@ -89,7 +89,7 @@ pub trait BlockBackend { fn block_status(&self, id: &BlockId) -> sp_blockchain::Result; /// Get block justification set by id. - fn justification(&self, id: &BlockId) -> sp_blockchain::Result>; + fn justification(&self, id: &BlockId) -> sp_blockchain::Result>; /// Get block hash by number. fn block_hash(&self, number: NumberFor) -> sp_blockchain::Result>; diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index ded030fb8046f..fed7f9488fdb3 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -27,7 +27,7 @@ use sp_core::{ }; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HashFor}; -use sp_runtime::{Justification, Storage}; +use sp_runtime::{Justifications, Storage}; use sp_state_machine::{ ChangesTrieTransaction, InMemoryBackend, Backend as StateBackend, StorageCollection, ChildStorageCollection, @@ -51,12 +51,12 @@ struct PendingBlock { #[derive(PartialEq, Eq, Clone)] enum StoredBlock { - Header(B::Header, Option), - Full(B, Option), + Header(B::Header, Option), + Full(B, Option), } impl StoredBlock { - fn new(header: B::Header, body: Option>, just: Option) -> Self { + fn new(header: B::Header, body: Option>, just: Option) -> Self { match body { Some(body) => StoredBlock::Full(B::new(header, body), just), None => StoredBlock::Header(header, just), @@ -70,7 +70,7 @@ impl StoredBlock { } } - fn justification(&self) -> Option<&Justification> { + fn justification(&self) -> Option<&Justifications> { match *self { StoredBlock::Header(_, ref j) | StoredBlock::Full(_, ref j) => j.as_ref() } @@ -83,7 +83,7 @@ impl StoredBlock { } } - fn into_inner(self) -> (B::Header, Option>, Option) { + fn into_inner(self) -> (B::Header, Option>, Option) { match self { StoredBlock::Header(header, just) => (header, None, just), StoredBlock::Full(block, just) => { @@ -164,7 +164,7 @@ impl Blockchain { &self, hash: Block::Hash, header: ::Header, - justification: Option, + justification: Option, body: Option::Extrinsic>>, new_state: NewBlockState, ) -> sp_blockchain::Result<()> { @@ -272,7 +272,7 @@ impl Blockchain { Ok(()) } - fn finalize_header(&self, id: BlockId, justification: Option) -> sp_blockchain::Result<()> { + fn finalize_header(&self, id: BlockId, justification: Option) -> sp_blockchain::Result<()> { let hash = match self.header(id)? { Some(h) => h.hash(), None => return Err(sp_blockchain::Error::UnknownBlock(format!("{}", id))), @@ -365,7 +365,7 @@ impl blockchain::Backend for Blockchain { })) } - fn justification(&self, id: BlockId) -> sp_blockchain::Result> { + fn justification(&self, id: BlockId) -> sp_blockchain::Result> { Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).and_then(|b| b.justification().map(|x| x.clone())) )) @@ -485,7 +485,7 @@ pub struct BlockImportOperation { old_state: InMemoryBackend>, new_state: Option<> as StateBackend>>::Transaction>, aux: Vec<(Vec, Option>)>, - finalized_blocks: Vec<(BlockId, Option)>, + finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, } @@ -502,7 +502,7 @@ impl backend::BlockImportOperation for BlockImportOperatio &mut self, header: ::Header, body: Option::Extrinsic>>, - justification: Option, + justification: Option, state: NewBlockState, ) -> sp_blockchain::Result<()> { assert!(self.pending_block.is_none(), "Only one block per operation is allowed"); @@ -570,7 +570,7 @@ impl backend::BlockImportOperation for BlockImportOperatio fn mark_finalized( &mut self, block: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()> { self.finalized_blocks.push((block, justification)); Ok(()) @@ -688,7 +688,7 @@ impl backend::Backend for Backend where Block::Hash fn finalize_block( &self, block: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()> { self.blockchain.finalize_header(block, justification) } diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index 246b39771277d..b36af788a4f6e 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -59,7 +59,7 @@ use sp_core::crypto::Public; use sp_application_crypto::{AppKey, AppPublic}; use sp_runtime::{ generic::{BlockId, OpaqueDigestItemId}, - traits::NumberFor, Justification, + traits::NumberFor, Justifications, }; use sp_runtime::traits::{Block as BlockT, Header, DigestItemFor, Zero, Member}; use sp_api::ProvideRuntimeApi; @@ -590,7 +590,7 @@ impl Verifier for AuraVerifier where &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justification: Option, mut body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { let mut inherent_data = self.inherent_data_providers diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 3f2a583482afb..a77158ed84714 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -84,7 +84,7 @@ use sp_core::crypto::Public; use sp_application_crypto::AppKey; use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore}; use sp_runtime::{ - generic::{BlockId, OpaqueDigestItemId}, Justification, + generic::{BlockId, OpaqueDigestItemId}, Justifications, traits::{Block as BlockT, Header, DigestItemFor, Zero}, }; use sp_api::{ProvideRuntimeApi, NumberFor}; @@ -974,7 +974,7 @@ where &mut self, origin: BlockOrigin, header: Block::Header, - justification: Option, + justification: Option, mut body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { trace!( diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 6e0536c85ced7..4784a0254f1ae 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -239,7 +239,7 @@ impl Verifier for TestVerifier { &mut self, origin: BlockOrigin, mut header: TestHeader, - justification: Option, + justification: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { // apply post-sealing mutations (i.e. stripping seal, if desired). diff --git a/client/consensus/manual-seal/src/finalize_block.rs b/client/consensus/manual-seal/src/finalize_block.rs index 5780a25f97256..ed1ecce4c9a01 100644 --- a/client/consensus/manual-seal/src/finalize_block.rs +++ b/client/consensus/manual-seal/src/finalize_block.rs @@ -18,7 +18,7 @@ use crate::rpc; use sp_runtime::{ - Justification, + Justifications, traits::Block as BlockT, generic::BlockId, }; @@ -33,7 +33,7 @@ pub struct FinalizeBlockParams { /// sender to report errors/success to the rpc. pub sender: rpc::Sender<()>, /// finalization justification - pub justification: Option, + pub justification: Option, /// Finalizer trait object. pub finalizer: Arc, /// phantom type to pin the Backend type diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 9c4465f82fda1..31e79c667d61a 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -27,7 +27,7 @@ use sp_consensus::{ }; use sp_blockchain::HeaderBackend; use sp_inherents::InherentDataProviders; -use sp_runtime::{traits::Block as BlockT, Justification}; +use sp_runtime::{traits::Block as BlockT, Justifications}; use sc_client_api::backend::{Backend as ClientBackend, Finalizer}; use sc_transaction_pool::txpool; use std::{sync::Arc, marker::PhantomData}; @@ -57,7 +57,7 @@ impl Verifier for ManualSealVerifier { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justification: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { let mut import_params = BlockImportParams::new(origin, header); diff --git a/client/consensus/manual-seal/src/rpc.rs b/client/consensus/manual-seal/src/rpc.rs index 690b6c1eb9996..2fbfdeab3cd08 100644 --- a/client/consensus/manual-seal/src/rpc.rs +++ b/client/consensus/manual-seal/src/rpc.rs @@ -26,7 +26,7 @@ use futures::{ SinkExt }; use serde::{Deserialize, Serialize}; -use sp_runtime::Justification; +use sp_runtime::Justifications; pub use self::gen_client::Client as ManualSealClient; /// Future's type for jsonrpc @@ -60,7 +60,7 @@ pub enum EngineCommand { /// sender to report errors/success to the rpc. sender: Sender<()>, /// finalization justification - justification: Option, + justification: Option, } } @@ -81,7 +81,7 @@ pub trait ManualSealApi { fn finalize_block( &self, hash: Hash, - justification: Option + justification: Option ) -> FutureResult; } @@ -129,7 +129,7 @@ impl ManualSealApi for ManualSeal { Box::new(future.map_err(Error::from).compat()) } - fn finalize_block(&self, hash: Hash, justification: Option) -> FutureResult { + fn finalize_block(&self, hash: Hash, justification: Option) -> FutureResult { let mut sink = self.import_block_channel.clone(); let future = async move { let (sender, receiver) = oneshot::channel(); diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index e353ed6358a00..cf1201b2f0b87 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -44,7 +44,7 @@ use parking_lot::Mutex; use sc_client_api::{BlockOf, backend::AuxStore, BlockchainEvents}; use sp_blockchain::{HeaderBackend, ProvideCache, well_known_cache_keys::Id as CacheKeyId}; use sp_block_builder::BlockBuilder as BlockBuilderApi; -use sp_runtime::{Justification, RuntimeString}; +use sp_runtime::{Justifications, RuntimeString}; use sp_runtime::generic::{BlockId, Digest, DigestItem}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use sp_api::ProvideRuntimeApi; @@ -458,7 +458,7 @@ impl Verifier for PowVerifier where &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justification: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { let hash = header.hash(); diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 70a0b19532593..7c2bbbde1b7cd 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -42,6 +42,7 @@ prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0. sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } sp-tracing = { version = "2.0.0", path = "../../primitives/tracing" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-finality-grandpa = { version = "2.0.0", path = "../../primitives/finality-grandpa" } quickcheck = "0.9" kvdb-rocksdb = "0.9.1" tempfile = "3" diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index a2299a82337a0..db42190f95558 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -529,7 +529,8 @@ mod tests { }; use sp_blockchain::HeaderBackend as BlockchainHeaderBackend; use sp_core::H256; - use sp_runtime::testing::{Digest, Header}; + use sp_finality_grandpa::GRANDPA_ENGINE_ID; + use sp_runtime::testing::{Digest, Header}; use sp_runtime::traits::{Hash, BlakeTwo256}; use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage}; use crate::Backend; @@ -953,7 +954,7 @@ mod tests { let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); let config1 = Some(ChangesTrieConfiguration::new(2, 6)); let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - backend.finalize_block(BlockId::Number(1), Some(vec![42])).unwrap(); + backend.finalize_block(BlockId::Number(1), Some(vec![(GRANDPA_ENGINE_ID, vec![42])])).unwrap(); let config2 = Some(ChangesTrieConfiguration::new(2, 7)); let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); let config2_1 = Some(ChangesTrieConfiguration::new(2, 8)); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 8254e652f68b0..ba8f215e7ec37 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -70,7 +70,7 @@ use sp_core::ChangesTrieConfiguration; use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; -use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Storage}; +use sp_runtime::{generic::{DigestItem, BlockId}, Justifications, Storage}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, }; @@ -338,7 +338,7 @@ pub(crate) mod columns { struct PendingBlock { header: Block::Header, - justification: Option, + justification: Option, body: Option>, leaf_state: NewBlockState, } @@ -486,7 +486,7 @@ impl sc_client_api::blockchain::Backend for BlockchainDb) -> ClientResult> { + fn justification(&self, id: BlockId) -> ClientResult> { match read_db(&*self.db, columns::KEY_LOOKUP, columns::JUSTIFICATION, id)? { Some(justification) => match Decode::decode(&mut &justification[..]) { Ok(justification) => Ok(Some(justification)), @@ -610,7 +610,7 @@ pub struct BlockImportOperation { changes_trie_config_update: Option>, pending_block: Option>, aux_ops: Vec<(Vec, Option>)>, - finalized_blocks: Vec<(BlockId, Option)>, + finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, commit_state: bool, } @@ -651,7 +651,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc &mut self, header: Block::Header, body: Option>, - justification: Option, + justification: Option, leaf_state: NewBlockState, ) -> ClientResult<()> { assert!(self.pending_block.is_none(), "Only one block per operation is allowed"); @@ -746,7 +746,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc fn mark_finalized( &mut self, block: BlockId, - justification: Option, + justification: Option, ) -> ClientResult<()> { self.finalized_blocks.push((block, justification)); Ok(()) @@ -1026,7 +1026,7 @@ impl Backend { hash: &Block::Hash, header: &Block::Header, last_finalized: Option, - justification: Option, + justification: Option, changes_trie_cache_ops: &mut Option>, finalization_displaced: &mut Option>>, ) -> ClientResult<(Block::Hash, ::Number, bool, bool)> { @@ -1501,7 +1501,7 @@ impl sc_client_api::backend::Backend for Backend { fn finalize_block( &self, block: BlockId, - justification: Option, + justification: Option, ) -> ClientResult<()> { let mut transaction = Transaction::new(); let hash = self.blockchain.expect_block_hash_from_id(&block)?; @@ -2354,7 +2354,7 @@ pub(crate) mod tests { let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); let _ = insert_header(&backend, 1, block0, None, Default::default()); - let justification = Some(vec![1, 2, 3]); + let justification = Some(vec![(sp_finality_grandpa::GRANDPA_ENGINE_ID, vec![1, 2, 3])]); backend.finalize_block(BlockId::Number(1), justification.clone()).unwrap(); assert_eq!( diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index 790be2a221788..4d26f0ef7ffaa 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -55,7 +55,7 @@ use crate::justification::GrandpaJustification; use crate::until_imported::UntilVoteTargetImported; use crate::voting_rule::VotingRule; use sp_finality_grandpa::{ - AuthorityId, AuthoritySignature, Equivocation, EquivocationProof, + AuthorityId, AuthoritySignature, Equivocation, EquivocationProof, GRANDPA_ENGINE_ID, GrandpaApi, RoundNumber, SetId, }; use prometheus_endpoint::{register, Counter, Gauge, PrometheusError, U64}; @@ -1289,10 +1289,13 @@ where // ideally some handle to a synchronization oracle would be used // to avoid unconditionally notifying. - client.apply_finality(import_op, BlockId::Hash(hash), justification, true).map_err(|e| { - warn!(target: "afg", "Error applying finality to block {:?}: {:?}", (hash, number), e); - e - })?; + let justification = justification.map(|j| vec![(GRANDPA_ENGINE_ID, j.clone())]); + client + .apply_finality(import_op, BlockId::Hash(hash), justification, true) + .map_err(|e| { + warn!(target: "afg", "Error applying finality to block {:?}: {:?}", (hash, number), e); + e + })?; telemetry!(CONSENSUS_INFO; "afg.finalized_blocks_up_to"; "number" => ?number, "hash" => ?hash, ); diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index bf367ab3f4a55..ace2ccc9ad5a5 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -51,12 +51,15 @@ use sc_client_api::{ use parity_scale_codec::{Encode, Decode}; use finality_grandpa::BlockNumberOps; use sp_runtime::{ - Justification, generic::BlockId, + EncodedJustification, generic::BlockId, traits::{NumberFor, Block as BlockT, Header as HeaderT, One}, }; use sp_core::storage::StorageKey; use sc_telemetry::{telemetry, CONSENSUS_INFO}; -use sp_finality_grandpa::{AuthorityId, AuthorityList, VersionedAuthorityList, GRANDPA_AUTHORITIES_KEY}; +use sp_finality_grandpa::{ + AuthorityId, AuthorityList, VersionedAuthorityList, + GRANDPA_AUTHORITIES_KEY, GRANDPA_ENGINE_ID, +}; use crate::justification::GrandpaJustification; use crate::VoterSet; @@ -235,7 +238,7 @@ pub struct FinalityProofFragment { /// The hash of block F for which justification is provided. pub block: Header::Hash, /// Justification of the block F. - pub justification: Vec, + pub justification: EncodedJustification, /// The set of headers in the range (U; F] that we believe are unknown to the caller. Ordered. pub unknown_headers: Vec
, /// Optional proof of execution of GRANDPA::authorities() at the `block`. @@ -332,56 +335,58 @@ pub(crate) fn prove_finality, J>( } if let Some(justification) = blockchain.justification(current_id)? { - // check if the current block enacts new GRANDPA authorities set - let new_authorities = authorities_provider.authorities(¤t_id)?; - let new_authorities_proof = if current_authorities != new_authorities { - current_authorities = new_authorities; - Some(authorities_provider.prove_authorities(¤t_id)?) - } else { - None - }; - - // prepare finality proof for the current block - let current = blockchain.expect_block_hash_from_id(&BlockId::Number(current_number))?; - let proof_fragment = FinalityProofFragment { - block: current, - justification, - unknown_headers: ::std::mem::take(&mut unknown_headers), - authorities_proof: new_authorities_proof, - }; - - // append justification to finality proof if required - let justifies_end_block = current_number >= end_number; - let justifies_authority_set_change = proof_fragment.authorities_proof.is_some(); - if justifies_end_block || justifies_authority_set_change { - // check if the proof is generated by the requested authority set - if finality_proof.is_empty() { - let justification_check_result = J::decode_and_verify( - &proof_fragment.justification, - authorities_set_id, - &begin_authorities, - ); - if justification_check_result.is_err() { - trace!( - target: "afg", - "Can not provide finality proof with requested set id #{}\ - (possible forced change?). Returning empty proof.", + if let Some((_, grandpa_justification)) = justification.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + // check if the current block enacts new GRANDPA authorities set + let new_authorities = authorities_provider.authorities(¤t_id)?; + let new_authorities_proof = if current_authorities != new_authorities { + current_authorities = new_authorities; + Some(authorities_provider.prove_authorities(¤t_id)?) + } else { + None + }; + + // prepare finality proof for the current block + let current = blockchain.expect_block_hash_from_id(&BlockId::Number(current_number))?; + let proof_fragment = FinalityProofFragment { + block: current, + justification: grandpa_justification, + unknown_headers: ::std::mem::take(&mut unknown_headers), + authorities_proof: new_authorities_proof, + }; + + // append justification to finality proof if required + let justifies_end_block = current_number >= end_number; + let justifies_authority_set_change = proof_fragment.authorities_proof.is_some(); + if justifies_end_block || justifies_authority_set_change { + // check if the proof is generated by the requested authority set + if finality_proof.is_empty() { + let justification_check_result = J::decode_and_verify( + &proof_fragment.justification, authorities_set_id, + &begin_authorities, ); - - return Ok(None); + if justification_check_result.is_err() { + trace!( + target: "afg", + "Can not provide finality proof with requested set id #{}\ + (possible forced change?). Returning empty proof.", + authorities_set_id, + ); + + return Ok(None); + } } - } - finality_proof.push(proof_fragment); - latest_proof_fragment = None; - } else { - latest_proof_fragment = Some(proof_fragment); - } + finality_proof.push(proof_fragment); + latest_proof_fragment = None; + } else { + latest_proof_fragment = Some(proof_fragment); + } - // we don't need to provide more justifications - if justifies_end_block { - break; + // we don't need to provide more justifications + if justifies_end_block { + break; + } } } @@ -558,7 +563,7 @@ pub(crate) trait ProvableJustification: Encode + Decode { /// Decode and verify justification. fn decode_and_verify( - justification: &Justification, + justification: &EncodedJustification, set_id: u64, authorities: &[(AuthorityId, u64)], ) -> ClientResult { @@ -664,11 +669,12 @@ pub(crate) mod tests { } fn test_blockchain() -> InMemoryBlockchain { + use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = InMemoryBlockchain::::new(); - blockchain.insert(header(0).hash(), header(0), Some(vec![0]), None, NewBlockState::Final).unwrap(); - blockchain.insert(header(1).hash(), header(1), Some(vec![1]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(0).hash(), header(0), Some(vec![(ID, vec![0])]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(1).hash(), header(1), Some(vec![(ID, vec![1])]), None, NewBlockState::Final).unwrap(); blockchain.insert(header(2).hash(), header(2), None, None, NewBlockState::Best).unwrap(); - blockchain.insert(header(3).hash(), header(3), Some(vec![3]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(3).hash(), header(3), Some(vec![(ID, vec![3])]), None, NewBlockState::Final).unwrap(); blockchain } @@ -714,12 +720,13 @@ pub(crate) mod tests { #[test] fn finality_proof_fails_for_non_canonical_block() { + use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = test_blockchain(); blockchain.insert(header(4).hash(), header(4), None, None, NewBlockState::Best).unwrap(); blockchain.insert(side_header(4).hash(), side_header(4), None, None, NewBlockState::Best).unwrap(); blockchain.insert(second_side_header(5).hash(), second_side_header(5), None, None, NewBlockState::Best) .unwrap(); - blockchain.insert(header(5).hash(), header(5), Some(vec![5]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(5).hash(), header(5), Some(vec![(ID, vec![5])]), None, NewBlockState::Final).unwrap(); // chain is 1 -> 2 -> 3 -> 4 -> 5 // \> 4' -> 5' @@ -759,12 +766,13 @@ pub(crate) mod tests { #[test] fn finality_proof_works_without_authorities_change() { + use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = test_blockchain(); let authorities = vec![(AuthorityId::from_slice(&[1u8; 32]), 1u64)]; let just4 = TestJustification((0, authorities.clone()), vec![4]).encode(); let just5 = TestJustification((0, authorities.clone()), vec![5]).encode(); - blockchain.insert(header(4).hash(), header(4), Some(just4), None, NewBlockState::Final).unwrap(); - blockchain.insert(header(5).hash(), header(5), Some(just5.clone()), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, just4)]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(5).hash(), header(5), Some(vec![(ID, just5.clone())]), None, NewBlockState::Final).unwrap(); // blocks 4 && 5 are finalized with justification // => since authorities are the same, we only need justification for 5 @@ -788,8 +796,9 @@ pub(crate) mod tests { #[test] fn finality_proof_finalized_earlier_block_if_no_justification_for_target_is_known() { + use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = test_blockchain(); - blockchain.insert(header(4).hash(), header(4), Some(vec![4]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, vec![4])]), None, NewBlockState::Final).unwrap(); blockchain.insert(header(5).hash(), header(5), None, None, NewBlockState::Final).unwrap(); // block 4 is finalized with justification + we request for finality of 5 @@ -814,6 +823,7 @@ pub(crate) mod tests { #[test] fn finality_proof_works_with_authorities_change() { + use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = test_blockchain(); let auth3 = vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)]; let auth5 = vec![(AuthorityId::from_slice(&[5u8; 32]), 1u64)]; @@ -821,10 +831,10 @@ pub(crate) mod tests { let just4 = TestJustification((0, auth3.clone()), vec![4]).encode(); let just5 = TestJustification((0, auth3.clone()), vec![5]).encode(); let just7 = TestJustification((1, auth5.clone()), vec![7]).encode(); - blockchain.insert(header(4).hash(), header(4), Some(just4), None, NewBlockState::Final).unwrap(); - blockchain.insert(header(5).hash(), header(5), Some(just5.clone()), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, just4)]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(5).hash(), header(5), Some(vec![(ID, just5.clone())]), None, NewBlockState::Final).unwrap(); blockchain.insert(header(6).hash(), header(6), None, None, NewBlockState::Final).unwrap(); - blockchain.insert(header(7).hash(), header(7), Some(just7.clone()), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(7).hash(), header(7), Some(vec![(ID, just7.clone())]), None, NewBlockState::Final).unwrap(); // when querying for finality of 6, we assume that the #3 is the last block known to the requester // => since we only have justification for #7, we provide #7 @@ -1007,13 +1017,14 @@ pub(crate) mod tests { #[test] fn finality_proof_is_none_if_first_justification_is_generated_by_unknown_set() { + use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; // this is the case for forced change: set_id has been forcibly increased on full node // and light node missed that // => justification verification will fail on light node anyways, so we do not return // finality proof at all let blockchain = test_blockchain(); let just4 = TestJustification((0, vec![(AuthorityId::from_slice(&[42u8; 32]), 1u64)]), vec![4]).encode(); - blockchain.insert(header(4).hash(), header(4), Some(just4), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, just4)]), None, NewBlockState::Final).unwrap(); let proof_of_4 = prove_finality::<_, _, TestJustification>( &blockchain, diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 89f9d0c16ad7c..0c97ac6b75ec7 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -33,7 +33,7 @@ use sp_consensus::{ SelectChain, }; use sp_finality_grandpa::{ConsensusLog, ScheduledChange, SetId, GRANDPA_ENGINE_ID}; -use sp_runtime::Justification; +use sp_runtime::Justifications; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{ Block as BlockT, DigestFor, Header as HeaderT, NumberFor, Zero, @@ -128,7 +128,7 @@ impl JustificationImport &mut self, hash: Block::Hash, number: NumberFor, - justification: Justification, + justification: Justifications, ) -> Result<(), Self::Error> { // this justification was requested by the sync service, therefore we // are not sure if it should enact a change or not. it could have been a @@ -615,12 +615,22 @@ where &mut self, hash: Block::Hash, number: NumberFor, - justification: Justification, + justification: Justifications, enacts_change: bool, initial_sync: bool, ) -> Result<(), ConsensusError> { + let grandpa_justification = + match justification.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + Some((_, grandpa_justification)) => grandpa_justification, + None => { + return Err(ConsensusError::ClientImport( + "JON(WIP): expected a grandpa justification.".into(), + )) + } + }; + let justification = GrandpaJustification::decode_and_verify_finalizes( - &justification, + &grandpa_justification, (hash, number), self.authority_set.set_id(), &self.authority_set.current_authorities(), diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index ef8168e84f66c..14ff1305f92ec 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -1625,7 +1625,7 @@ fn imports_justification_for_regular_blocks_on_import() { // we import the block with justification attached let mut import = BlockImportParams::new(BlockOrigin::File, block.header); - import.justification = Some(justification.encode()); + import.justification = Some(vec![(GRANDPA_ENGINE_ID, justification.encode())]); import.body = Some(block.extrinsics); import.fork_choice = Some(ForkChoiceStrategy::LongestChain); diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index be7953e528bd8..7484bf29c6dc8 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -32,7 +32,7 @@ use sp_state_machine::{ Backend as StateBackend, TrieBackend, InMemoryBackend, ChangesTrieTransaction, StorageCollection, ChildStorageCollection, }; -use sp_runtime::{generic::BlockId, Justification, Storage}; +use sp_runtime::{generic::BlockId, Justifications, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header, HashFor}; use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sc_client_api::{ @@ -194,7 +194,7 @@ impl ClientBackend for Backend> fn finalize_block( &self, block: BlockId, - _justification: Option, + _justification: Option, ) -> ClientResult<()> { self.blockchain.storage().finalize_header(block) } @@ -278,7 +278,7 @@ impl BlockImportOperation for ImportOperation &mut self, header: Block::Header, _body: Option>, - _justification: Option, + _justification: Option, state: NewBlockState, ) -> ClientResult<()> { self.leaf_state = state; @@ -356,7 +356,7 @@ impl BlockImportOperation for ImportOperation fn mark_finalized( &mut self, block: BlockId, - _justification: Option, + _justification: Option, ) -> ClientResult<()> { self.finalized_blocks.push(block); Ok(()) diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs index 3b5753f2849d5..b953b0c3802e9 100644 --- a/client/light/src/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -21,7 +21,7 @@ use std::sync::Arc; -use sp_runtime::{Justification, generic::BlockId}; +use sp_runtime::{Justifications, generic::BlockId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}; use sp_blockchain::{ @@ -109,7 +109,7 @@ impl BlockchainBackend for Blockchain where Block: BlockT, S Err(ClientError::NotAvailableOnLightClient) } - fn justification(&self, _id: BlockId) -> ClientResult> { + fn justification(&self, _id: BlockId) -> ClientResult> { Err(ClientError::NotAvailableOnLightClient) } diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index b2914a5e0a72b..8638db99b9594 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -30,7 +30,7 @@ use libp2p::kad::record; use libp2p::swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess, PollParameters}; use log::debug; use sp_consensus::{BlockOrigin, import_queue::{IncomingBlock, Origin}}; -use sp_runtime::{traits::{Block as BlockT, NumberFor}, Justification}; +use sp_runtime::{traits::{Block as BlockT, NumberFor}, Justifications}; use std::{ borrow::Cow, collections::{HashSet, VecDeque}, @@ -73,7 +73,7 @@ pub struct Behaviour { /// Event generated by `Behaviour`. pub enum BehaviourOut { BlockImport(BlockOrigin, Vec>), - JustificationImport(Origin, B::Hash, NumberFor, Justification), + JustificationImport(Origin, B::Hash, NumberFor, Justifications), /// Started a random iterative Kademlia discovery query. RandomKademliaStarted(ProtocolId), diff --git a/client/network/src/block_requests.rs b/client/network/src/block_requests.rs index ace63e6e1cdd4..077fbd196bbfb 100644 --- a/client/network/src/block_requests.rs +++ b/client/network/src/block_requests.rs @@ -422,6 +422,12 @@ where Vec::new() }; + // WIP(JON): Need to encode the whole set of Justifications + let a = justification.unwrap_or_default(); + let justification = a + .first() + .map(|y| y.1.clone()); + let block_data = schema::v1::BlockData { hash: hash.encode(), header: if get_header { @@ -649,7 +655,12 @@ where None }, justification: if !block_data.justification.is_empty() { - Some(block_data.justification) + // WIP(JON): Need to decode the whole set of Justifications + // let id = sp_finality_grandpa::GRANDPA_ENGINE_ID; + const GRANDPA_ENGINE_ID: [u8; 4] = *b"FRNK"; + let just = vec![(GRANDPA_ENGINE_ID, block_data.justification)]; + Some(just) + // Some(block_data.justification) } else if block_data.is_empty_justification { Some(Vec::new()) } else { diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index 93b69f7b64c8e..8edfe54a6b643 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -49,7 +49,7 @@ fn build_test_full_node(config: config::NetworkConfiguration) &mut self, origin: sp_consensus::BlockOrigin, header: B::Header, - justification: Option, + justification: Option, body: Option>, ) -> Result< ( diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 9e589330b7fbb..3d6ecaa7dbc3c 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -37,7 +37,7 @@ use sp_consensus::{ import_queue::{BlockImportResult, BlockImportError, IncomingBlock, Origin} }; use codec::{Decode, DecodeAll, Encode}; -use sp_runtime::{generic::BlockId, Justification}; +use sp_runtime::{generic::BlockId, Justifications}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub }; @@ -1334,7 +1334,7 @@ impl Protocol { #[must_use] pub enum CustomMessageOutcome { BlockImport(BlockOrigin, Vec>), - JustificationImport(Origin, B::Hash, NumberFor, Justification), + JustificationImport(Origin, B::Hash, NumberFor, Justifications), /// Notification protocols have been opened with a remote. NotificationStreamOpened { remote: PeerId, diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 4213d56bbf022..6db4353dd3671 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -148,7 +148,7 @@ pub struct RemoteReadResponse { pub mod generic { use bitflags::bitflags; use codec::{Encode, Decode, Input, Output}; - use sp_runtime::Justification; + use sp_runtime::Justifications; use super::{ RemoteReadResponse, Transactions, Direction, RequestId, BlockAttributes, RemoteCallResponse, ConsensusEngineId, @@ -234,7 +234,7 @@ pub mod generic { /// Block message queue if requested. pub message_queue: Option>, /// Justification if requested. - pub justification: Option, + pub justification: Option, } /// Identifies starting point of a block sequence. diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index ced789446da60..e7e0e82844e2b 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -42,7 +42,7 @@ use extra_requests::ExtraRequests; use libp2p::PeerId; use log::{debug, trace, warn, info, error}; use sp_runtime::{ - Justification, + Justifications, generic::BlockId, traits::{Block as BlockT, Header, NumberFor, Zero, One, CheckedSub, SaturatedConversion, Hash, HashFor} }; @@ -387,7 +387,7 @@ pub enum OnBlockJustification { peer: PeerId, hash: B::Hash, number: NumberFor, - justification: Justification + justification: Justifications } } diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 225a3ae98ab5d..24f54cdacbec0 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -50,7 +50,7 @@ fn build_test_full_node(config: config::NetworkConfiguration) &mut self, origin: sp_consensus::BlockOrigin, header: B::Header, - justification: Option, + justification: Option, body: Option>, ) -> Result< ( diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index a70ecb4fb0484..f17c77ef3c44d 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -57,7 +57,7 @@ use sp_core::H256; use sc_network::config::ProtocolConfig; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; -use sp_runtime::Justification; +use sp_runtime::Justifications; use substrate_test_runtime_client::{self, AccountKeyring}; use sc_service::client::Client; pub use sc_network::config::EmptyTransactionPool; @@ -103,7 +103,7 @@ impl Verifier for PassThroughVerifier { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justification: Option, body: Option> ) -> Result<(BlockImportParams, Option)>>), String> { let maybe_keys = header.digest() @@ -178,7 +178,7 @@ impl PeersClient { } } - pub fn justification(&self, block: &BlockId) -> ClientResult> { + pub fn justification(&self, block: &BlockId) -> ClientResult> { match *self { PeersClient::Full(ref client, ref _backend) => client.justification(block), PeersClient::Light(ref client, ref _backend) => client.justification(block), @@ -202,7 +202,7 @@ impl PeersClient { pub fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool ) -> ClientResult<()> { match *self { @@ -554,7 +554,7 @@ impl Verifier for VerifierAdapter { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justification: Option, body: Option> ) -> Result<(BlockImportParams, Option)>>), String> { let hash = header.hash(); @@ -951,7 +951,7 @@ impl JustificationImport for ForceFinalized { &mut self, hash: H256, _number: NumberFor, - justification: Justification, + justification: Justifications, ) -> Result<(), Self::Error> { self.0.finalize_block(BlockId::Hash(hash), Some(justification), true) .map_err(|_| ConsensusError::InvalidJustification.into()) diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index f72fcb810769f..e217e61909983 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -37,7 +37,7 @@ use sp_core::{ use sp_keystore::SyncCryptoStorePtr; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_runtime::{ - Justification, BuildStorage, + Justifications, BuildStorage, generic::{BlockId, SignedBlock, DigestItem}, traits::{ Block as BlockT, Header as HeaderT, Zero, NumberFor, @@ -686,7 +686,7 @@ impl Client where origin: BlockOrigin, hash: Block::Hash, import_headers: PrePostHeader, - justification: Option, + justification: Option, body: Option>, storage_changes: Option, Block>>, new_cache: HashMap>, @@ -906,7 +906,7 @@ impl Client where &self, operation: &mut ClientImportOperation, block: Block::Hash, - justification: Option, + justification: Option, best_block: Block::Hash, notify: bool, ) -> sp_blockchain::Result<()> { @@ -1808,7 +1808,7 @@ impl Finalizer for Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { let last_best = self.backend.blockchain().info().best_hash; @@ -1825,7 +1825,7 @@ impl Finalizer for Client where fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { self.lock_import_and_run(|operation| { @@ -1844,7 +1844,7 @@ impl Finalizer for &Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { (**self).apply_finality(operation, id, justification, notify) @@ -1853,7 +1853,7 @@ impl Finalizer for &Client where fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { (**self).finalize_block(id, justification, notify) @@ -1933,7 +1933,7 @@ impl BlockBackend for Client } } - fn justification(&self, id: &BlockId) -> sp_blockchain::Result> { + fn justification(&self, id: &BlockId) -> sp_blockchain::Result> { self.backend.blockchain().justification(*id) } diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index 84ac84e630d00..6a6e597d965a0 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -42,3 +42,6 @@ sc-executor = { version = "0.8.0", path = "../../executor" } sp-panic-handler = { version = "2.0.0", path = "../../../primitives/panic-handler" } parity-scale-codec = "1.3.4" sp-tracing = { version = "2.0.0", path = "../../../primitives/tracing" } + +[dev-dependencies] +sp-finality-grandpa = { version = "2.0.0", path = "../../../primitives/finality-grandpa" } diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 23d6a34297328..7420ead2b6632 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -47,6 +47,7 @@ use sp_consensus::{ BlockOrigin, SelectChain, BlockImport, Error as ConsensusError, BlockCheckParams, ImportResult, BlockStatus, BlockImportParams, ForkChoiceStrategy, }; +use sp_finality_grandpa::GRANDPA_ENGINE_ID; use sp_storage::StorageKey; use sp_trie::{TrieConfiguration, trie_types::Layout}; use sp_runtime::{generic::BlockId, DigestItem}; @@ -1021,7 +1022,7 @@ fn import_with_justification() { client.import(BlockOrigin::Own, a2.clone()).unwrap(); // A2 -> A3 - let justification = vec![1, 2, 3]; + let justification = vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]; let a3 = client.new_block_at( &BlockId::Hash(a2.hash()), Default::default(), @@ -1093,7 +1094,7 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { ); // importing B1 as finalized should trigger a re-org and set it as new best - let justification = vec![1, 2, 3]; + let justification = vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]; client.import_justified(BlockOrigin::Own, b1.clone(), justification).unwrap(); assert_eq!( diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index 01a7a59d6f94f..b8ba6e671c2fd 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use sp_runtime::generic::BlockId; -use sp_runtime::Justification; +use sp_runtime::Justifications; use log::warn; use parking_lot::RwLock; @@ -85,7 +85,7 @@ pub trait Backend: HeaderBackend + HeaderMetadata) -> Result::Extrinsic>>>; /// Get block justification. Returns `None` if justification does not exist. - fn justification(&self, id: BlockId) -> Result>; + fn justification(&self, id: BlockId) -> Result>; /// Get last finalized block hash. fn last_finalized(&self) -> Result; /// Returns data cache reference, if it is enabled on this backend. diff --git a/primitives/consensus/common/src/block_import.rs b/primitives/consensus/common/src/block_import.rs index 0100041fc0a0c..92397e480de3f 100644 --- a/primitives/consensus/common/src/block_import.rs +++ b/primitives/consensus/common/src/block_import.rs @@ -18,7 +18,7 @@ //! Block import helpers. use sp_runtime::traits::{Block as BlockT, DigestItemFor, Header as HeaderT, NumberFor, HashFor}; -use sp_runtime::Justification; +use sp_runtime::Justifications; use serde::{Serialize, Deserialize}; use std::borrow::Cow; use std::collections::HashMap; @@ -129,7 +129,7 @@ pub struct BlockImportParams { /// post-runtime digests are pushed back on after. pub header: Block::Header, /// Justification provided for this block from the outside. - pub justification: Option, + pub justification: Option, /// Digest items that have been added after the runtime for external /// work, like a consensus signature. pub post_digests: Vec>, @@ -340,6 +340,6 @@ pub trait JustificationImport { &mut self, hash: B::Hash, number: NumberFor, - justification: Justification, + justification: Justifications, ) -> Result<(), Self::Error>; } diff --git a/primitives/consensus/common/src/import_queue.rs b/primitives/consensus/common/src/import_queue.rs index b32ca0133d995..0bfc6d48ea302 100644 --- a/primitives/consensus/common/src/import_queue.rs +++ b/primitives/consensus/common/src/import_queue.rs @@ -28,7 +28,7 @@ use std::collections::HashMap; -use sp_runtime::{Justification, traits::{Block as BlockT, Header as _, NumberFor}}; +use sp_runtime::{Justifications, traits::{Block as BlockT, Header as _, NumberFor}}; use crate::{ error::Error as ConsensusError, @@ -69,7 +69,7 @@ pub struct IncomingBlock { /// Block body if requested. pub body: Option::Extrinsic>>, /// Justification if requested. - pub justification: Option, + pub justification: Option, /// The peer, we received this from pub origin: Option, /// Allow importing the block skipping state verification if parent state is missing. @@ -90,7 +90,7 @@ pub trait Verifier: Send + Sync { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justification: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String>; } @@ -108,7 +108,7 @@ pub trait ImportQueue: Send { who: Origin, hash: B::Hash, number: NumberFor, - justification: Justification + justification: Justifications ); /// Polls for actions to perform on the network. /// diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index b426c39100e69..2d1cb56af53b1 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -18,7 +18,7 @@ use std::{mem, pin::Pin, time::Duration, marker::PhantomData}; use futures::{prelude::*, task::Context, task::Poll}; use futures_timer::Delay; -use sp_runtime::{Justification, traits::{Block as BlockT, Header as HeaderT, NumberFor}}; +use sp_runtime::{Justifications, traits::{Block as BlockT, Header as HeaderT, NumberFor}}; use sp_utils::mpsc::{TracingUnboundedSender, tracing_unbounded}; use prometheus_endpoint::Registry; @@ -117,7 +117,7 @@ impl ImportQueue for BasicQueue who: Origin, hash: B::Hash, number: NumberFor, - justification: Justification, + justification: Justifications, ) { let res = self.justification_sender.unbounded_send( worker_messages::ImportJustification(who, hash, number, justification), @@ -143,7 +143,7 @@ mod worker_messages { use super::*; pub struct ImportBlocks(pub BlockOrigin, pub Vec>); - pub struct ImportJustification(pub Origin, pub B::Hash, pub NumberFor, pub Justification); + pub struct ImportJustification(pub Origin, pub B::Hash, pub NumberFor, pub Justifications); } struct BlockImportWorker { @@ -281,7 +281,7 @@ impl BlockImportWorker { who: Origin, hash: B::Hash, number: NumberFor, - justification: Justification + justification: Justifications ) { let started = wasm_timer::Instant::now(); let success = self.justification_import.as_mut().map(|justification_import| { @@ -439,7 +439,7 @@ mod tests { &mut self, origin: BlockOrigin, header: Header, - _justification: Option, + _justification: Option, _body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { Ok((BlockImportParams::new(origin, header), None)) @@ -473,7 +473,7 @@ mod tests { &mut self, _hash: Hash, _number: BlockNumber, - _justification: Justification, + _justification: Justifications, ) -> Result<(), Self::Error> { Ok(()) } diff --git a/primitives/runtime/src/generic/block.rs b/primitives/runtime/src/generic/block.rs index 4a758b7416dec..184e7deadc412 100644 --- a/primitives/runtime/src/generic/block.rs +++ b/primitives/runtime/src/generic/block.rs @@ -30,7 +30,7 @@ use crate::traits::{ self, Member, Block as BlockT, Header as HeaderT, MaybeSerialize, MaybeMallocSizeOf, NumberFor, }; -use crate::Justification; +use crate::Justifications; /// Something to identify a block. #[derive(PartialEq, Eq, Clone, RuntimeDebug)] @@ -112,5 +112,5 @@ pub struct SignedBlock { /// Full block. pub block: Block, /// Block justification. - pub justification: Option, + pub justification: Option, } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index ccd50334af660..3ad3ad5da63fa 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -95,7 +95,10 @@ pub use either::Either; /// the block itself would allow swapping justifications to change the block's hash /// (and thus fork the chain). Sending a `Justification` alongside a block instead /// bypasses this problem. -pub type Justification = Vec; +pub type EncodedJustification = Vec; +/// Collection of Justifications, since we might have more than one stored per block. +// WIP(JON): Consider making `Justifications` strongly typed. +pub type Justifications = Vec<(ConsensusEngineId, EncodedJustification)>; use traits::{Verify, Lazy}; diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index 43e89c8f10bc1..05f1ffdb5cf3e 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -24,7 +24,7 @@ use sp_consensus::{ BlockImportParams, BlockImport, BlockOrigin, Error as ConsensusError, ForkChoiceStrategy, }; -use sp_runtime::Justification; +use sp_runtime::Justifications; use sp_runtime::traits::{Block as BlockT}; use sp_runtime::generic::BlockId; use codec::alloc::collections::hash_map::HashMap; @@ -35,7 +35,7 @@ pub trait ClientExt: Sized { fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()>; /// Returns hash of the genesis block. @@ -59,7 +59,7 @@ pub trait ClientBlockImportExt: Sized { &mut self, origin: BlockOrigin, block: Block, - justification: Justification + justification: Justifications ) -> Result<(), ConsensusError>; } @@ -73,7 +73,7 @@ impl ClientExt for Client fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()> { Finalizer::finalize_block(self, id, justification, true) } @@ -119,7 +119,7 @@ impl ClientBlockImportExt for std::sync::A &mut self, origin: BlockOrigin, block: Block, - justification: Justification, + justification: Justifications, ) -> Result<(), ConsensusError> { let (header, extrinsics) = block.deconstruct(); let mut import = BlockImportParams::new(origin, header); @@ -168,7 +168,7 @@ impl ClientBlockImportExt for Client Result<(), ConsensusError> { let (header, extrinsics) = block.deconstruct(); let mut import = BlockImportParams::new(origin, header); From 16bfd2c6d3ed608b7b97ede589e03c53d0ec219b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 30 Nov 2020 22:48:04 +0100 Subject: [PATCH 02/58] primitives/runtime: make Justifications strongly typed --- client/db/src/changes_tries_storage.rs | 5 +- client/db/src/lib.rs | 3 +- client/finality-grandpa/src/environment.rs | 2 +- client/finality-grandpa/src/finality_proof.rs | 54 +++++++++++-------- client/finality-grandpa/src/import.rs | 2 +- client/finality-grandpa/src/tests.rs | 6 +-- client/network/src/block_requests.rs | 9 ++-- client/network/test/src/sync.rs | 22 ++++---- client/service/test/src/client/light.rs | 3 +- client/service/test/src/client/mod.rs | 6 +-- .../common/src/import_queue/basic_queue.rs | 2 +- primitives/runtime/src/lib.rs | 5 +- 12 files changed, 69 insertions(+), 50 deletions(-) diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index db42190f95558..245868a8876f9 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -530,7 +530,7 @@ mod tests { use sp_blockchain::HeaderBackend as BlockchainHeaderBackend; use sp_core::H256; use sp_finality_grandpa::GRANDPA_ENGINE_ID; - use sp_runtime::testing::{Digest, Header}; + use sp_runtime::{Justifications, testing::{Digest, Header}}; use sp_runtime::traits::{Hash, BlakeTwo256}; use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage}; use crate::Backend; @@ -954,7 +954,8 @@ mod tests { let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); let config1 = Some(ChangesTrieConfiguration::new(2, 6)); let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - backend.finalize_block(BlockId::Number(1), Some(vec![(GRANDPA_ENGINE_ID, vec![42])])).unwrap(); + let just1 = Some(Justifications(vec![(GRANDPA_ENGINE_ID, vec![42])])); + backend.finalize_block(BlockId::Number(1), just1).unwrap(); let config2 = Some(ChangesTrieConfiguration::new(2, 7)); let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); let config2_1 = Some(ChangesTrieConfiguration::new(2, 8)); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index ba8f215e7ec37..b63fe7d676660 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1779,6 +1779,7 @@ pub(crate) mod tests { use sp_runtime::generic::DigestItem; use sp_state_machine::{TrieMut, TrieDBMut}; use sp_blockchain::{lowest_common_ancestor, tree_route}; + use sp_finality_grandpa::GRANDPA_ENGINE_ID; pub(crate) type Block = RawBlock>; @@ -2354,7 +2355,7 @@ pub(crate) mod tests { let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); let _ = insert_header(&backend, 1, block0, None, Default::default()); - let justification = Some(vec![(sp_finality_grandpa::GRANDPA_ENGINE_ID, vec![1, 2, 3])]); + let justification = Some(Justifications(vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])])); backend.finalize_block(BlockId::Number(1), justification.clone()).unwrap(); assert_eq!( diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index 4d26f0ef7ffaa..93b9f28bce026 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -1289,7 +1289,7 @@ where // ideally some handle to a synchronization oracle would be used // to avoid unconditionally notifying. - let justification = justification.map(|j| vec![(GRANDPA_ENGINE_ID, j.clone())]); + let justification = justification.map(|j| sp_runtime::Justifications(vec![(GRANDPA_ENGINE_ID, j.clone())])); client .apply_finality(import_op, BlockId::Hash(hash), justification, true) .map_err(|e| { diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index ace2ccc9ad5a5..0012be1f50fe1 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -335,7 +335,7 @@ pub(crate) fn prove_finality, J>( } if let Some(justification) = blockchain.justification(current_id)? { - if let Some((_, grandpa_justification)) = justification.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + if let Some((_, grandpa_justification)) = justification.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { // check if the current block enacts new GRANDPA authorities set let new_authorities = authorities_provider.authorities(¤t_id)?; let new_authorities_proof = if current_authorities != new_authorities { @@ -594,6 +594,7 @@ pub(crate) mod tests { use sc_client_api::in_mem::Blockchain as InMemoryBlockchain; use super::*; use sp_core::crypto::Public; + use sp_runtime::Justifications; pub(crate) type FinalityProof = super::FinalityProof
; @@ -671,10 +672,14 @@ pub(crate) mod tests { fn test_blockchain() -> InMemoryBlockchain { use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = InMemoryBlockchain::::new(); - blockchain.insert(header(0).hash(), header(0), Some(vec![(ID, vec![0])]), None, NewBlockState::Final).unwrap(); - blockchain.insert(header(1).hash(), header(1), Some(vec![(ID, vec![1])]), None, NewBlockState::Final).unwrap(); - blockchain.insert(header(2).hash(), header(2), None, None, NewBlockState::Best).unwrap(); - blockchain.insert(header(3).hash(), header(3), Some(vec![(ID, vec![3])]), None, NewBlockState::Final).unwrap(); + let just0 = Some(Justifications(vec![(ID, vec![0])])); + let just1 = Some(Justifications(vec![(ID, vec![1])])); + let just2 = None; + let just3 = Some(Justifications(vec![(ID, vec![3])])); + blockchain.insert(header(0).hash(), header(0), just0, None, NewBlockState::Final).unwrap(); + blockchain.insert(header(1).hash(), header(1), just1, None, NewBlockState::Final).unwrap(); + blockchain.insert(header(2).hash(), header(2), just2, None, NewBlockState::Best).unwrap(); + blockchain.insert(header(3).hash(), header(3), just3, None, NewBlockState::Final).unwrap(); blockchain } @@ -726,7 +731,8 @@ pub(crate) mod tests { blockchain.insert(side_header(4).hash(), side_header(4), None, None, NewBlockState::Best).unwrap(); blockchain.insert(second_side_header(5).hash(), second_side_header(5), None, None, NewBlockState::Best) .unwrap(); - blockchain.insert(header(5).hash(), header(5), Some(vec![(ID, vec![5])]), None, NewBlockState::Final).unwrap(); + let just = Some(Justifications(vec![(ID, vec![5])])); + blockchain.insert(header(5).hash(), header(5), just, None, NewBlockState::Final).unwrap(); // chain is 1 -> 2 -> 3 -> 4 -> 5 // \> 4' -> 5' @@ -769,10 +775,12 @@ pub(crate) mod tests { use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = test_blockchain(); let authorities = vec![(AuthorityId::from_slice(&[1u8; 32]), 1u64)]; - let just4 = TestJustification((0, authorities.clone()), vec![4]).encode(); - let just5 = TestJustification((0, authorities.clone()), vec![5]).encode(); - blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, just4)]), None, NewBlockState::Final).unwrap(); - blockchain.insert(header(5).hash(), header(5), Some(vec![(ID, just5.clone())]), None, NewBlockState::Final).unwrap(); + let just4_grandpa = TestJustification((0, authorities.clone()), vec![4]).encode(); + let just4 = Justifications(vec![(ID, just4_grandpa)]); + let just5_grandpa = TestJustification((0, authorities.clone()), vec![5]).encode(); + let just5 = Justifications(vec![(ID, just5_grandpa.clone())]); + blockchain.insert(header(4).hash(), header(4), Some(just4), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(5).hash(), header(5), Some(just5), None, NewBlockState::Final).unwrap(); // blocks 4 && 5 are finalized with justification // => since authorities are the same, we only need justification for 5 @@ -788,7 +796,7 @@ pub(crate) mod tests { ).unwrap().unwrap()[..]).unwrap(); assert_eq!(proof_of_5, vec![FinalityProofFragment { block: header(5).hash(), - justification: just5, + justification: just5_grandpa, unknown_headers: Vec::new(), authorities_proof: None, }]); @@ -798,7 +806,7 @@ pub(crate) mod tests { fn finality_proof_finalized_earlier_block_if_no_justification_for_target_is_known() { use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = test_blockchain(); - blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, vec![4])]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(4).hash(), header(4), Some(Justifications(vec![(ID, vec![4])])), None, NewBlockState::Final).unwrap(); blockchain.insert(header(5).hash(), header(5), None, None, NewBlockState::Final).unwrap(); // block 4 is finalized with justification + we request for finality of 5 @@ -828,13 +836,16 @@ pub(crate) mod tests { let auth3 = vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)]; let auth5 = vec![(AuthorityId::from_slice(&[5u8; 32]), 1u64)]; let auth7 = vec![(AuthorityId::from_slice(&[7u8; 32]), 1u64)]; - let just4 = TestJustification((0, auth3.clone()), vec![4]).encode(); - let just5 = TestJustification((0, auth3.clone()), vec![5]).encode(); - let just7 = TestJustification((1, auth5.clone()), vec![7]).encode(); - blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, just4)]), None, NewBlockState::Final).unwrap(); - blockchain.insert(header(5).hash(), header(5), Some(vec![(ID, just5.clone())]), None, NewBlockState::Final).unwrap(); + let just4_grandpa = TestJustification((0, auth3.clone()), vec![4]).encode(); + let just4 = Some(Justifications(vec![(ID, just4_grandpa)])); + let just5_grandpa = TestJustification((0, auth3.clone()), vec![5]).encode(); + let just5 = Some(Justifications(vec![(ID, just5_grandpa.clone())])); + let just7_grandpa = TestJustification((1, auth5.clone()), vec![7]).encode(); + let just7 = Some(Justifications(vec![(ID, just7_grandpa.clone())])); + blockchain.insert(header(4).hash(), header(4), just4, None, NewBlockState::Final).unwrap(); + blockchain.insert(header(5).hash(), header(5), just5.clone(), None, NewBlockState::Final).unwrap(); blockchain.insert(header(6).hash(), header(6), None, None, NewBlockState::Final).unwrap(); - blockchain.insert(header(7).hash(), header(7), Some(vec![(ID, just7.clone())]), None, NewBlockState::Final).unwrap(); + blockchain.insert(header(7).hash(), header(7), just7, None, NewBlockState::Final).unwrap(); // when querying for finality of 6, we assume that the #3 is the last block known to the requester // => since we only have justification for #7, we provide #7 @@ -864,14 +875,14 @@ pub(crate) mod tests { // first fragment provides justification for #5 && authorities set that starts acting from #5 FinalityProofFragment { block: header(5).hash(), - justification: just5, + justification: just5_grandpa, unknown_headers: Vec::new(), authorities_proof: Some(StorageProof::new(vec![vec![50]])), }, // last fragment provides justification for #7 && unknown#7 FinalityProofFragment { block: header(7).hash(), - justification: just7.clone(), + justification: just7_grandpa.clone(), unknown_headers: vec![header(7)], authorities_proof: Some(StorageProof::new(vec![vec![70]])), }, @@ -1024,7 +1035,8 @@ pub(crate) mod tests { // finality proof at all let blockchain = test_blockchain(); let just4 = TestJustification((0, vec![(AuthorityId::from_slice(&[42u8; 32]), 1u64)]), vec![4]).encode(); - blockchain.insert(header(4).hash(), header(4), Some(vec![(ID, just4)]), None, NewBlockState::Final).unwrap(); + let just4 = Some(Justifications(vec![(ID, just4)])); + blockchain.insert(header(4).hash(), header(4), just4, None, NewBlockState::Final).unwrap(); let proof_of_4 = prove_finality::<_, _, TestJustification>( &blockchain, diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 0c97ac6b75ec7..fcedca4419e99 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -620,7 +620,7 @@ where initial_sync: bool, ) -> Result<(), ConsensusError> { let grandpa_justification = - match justification.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + match justification.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { Some((_, grandpa_justification)) => grandpa_justification, None => { return Err(ConsensusError::ClientImport( diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 14ff1305f92ec..65ae848b4f622 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -40,7 +40,7 @@ use sp_consensus::{ }; use std::{collections::{HashMap, HashSet}, pin::Pin}; use parity_scale_codec::Decode; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, HashFor}; +use sp_runtime::{Justifications, traits::{Block as BlockT, Header as HeaderT, HashFor}}; use sp_runtime::generic::{BlockId, DigestItem}; use sp_core::H256; use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore}; @@ -909,7 +909,7 @@ fn test_bad_justification() { let block = || { let block = block.clone(); let mut import = BlockImportParams::new(BlockOrigin::File, block.header); - import.justification = Some(Vec::new()); + import.justification = Some(Justifications(Vec::new())); import.body = Some(block.extrinsics); import.fork_choice = Some(ForkChoiceStrategy::LongestChain); @@ -1625,7 +1625,7 @@ fn imports_justification_for_regular_blocks_on_import() { // we import the block with justification attached let mut import = BlockImportParams::new(BlockOrigin::File, block.header); - import.justification = Some(vec![(GRANDPA_ENGINE_ID, justification.encode())]); + import.justification = Some(Justifications(vec![(GRANDPA_ENGINE_ID, justification.encode())])); import.body = Some(block.extrinsics); import.fork_choice = Some(ForkChoiceStrategy::LongestChain); diff --git a/client/network/src/block_requests.rs b/client/network/src/block_requests.rs index 077fbd196bbfb..391d570736113 100644 --- a/client/network/src/block_requests.rs +++ b/client/network/src/block_requests.rs @@ -406,7 +406,7 @@ where } else { None }; - let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false); + let is_empty_justification = justification.as_ref().map(|j| j.0.is_empty()).unwrap_or(false); let body = if get_body { match self.chain.block_body(&BlockId::Hash(hash))? { @@ -423,8 +423,9 @@ where }; // WIP(JON): Need to encode the whole set of Justifications - let a = justification.unwrap_or_default(); + let a = justification.unwrap_or(sp_runtime::Justifications(Vec::new())); let justification = a + .0 .first() .map(|y| y.1.clone()); @@ -659,10 +660,10 @@ where // let id = sp_finality_grandpa::GRANDPA_ENGINE_ID; const GRANDPA_ENGINE_ID: [u8; 4] = *b"FRNK"; let just = vec![(GRANDPA_ENGINE_ID, block_data.justification)]; - Some(just) + Some(sp_runtime::Justifications(just)) // Some(block_data.justification) } else if block_data.is_empty_justification { - Some(Vec::new()) + Some(sp_runtime::Justifications(Vec::new())) } else { None }, diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 9a488ae4fa49c..8f381d3145ecf 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -22,6 +22,7 @@ use futures::{Future, executor::block_on}; use super::*; use sp_consensus::block_validation::Validation; use substrate_test_runtime::Header; +use sp_runtime::Justifications; fn test_ancestor_search_when_common_is(n: usize) { sp_tracing::try_init_simple(); @@ -252,9 +253,9 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justification(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - net.peer(0).client().finalize_block(BlockId::Number(10), Some(Vec::new()), true).unwrap(); - net.peer(0).client().finalize_block(BlockId::Number(15), Some(Vec::new()), true).unwrap(); - net.peer(0).client().finalize_block(BlockId::Number(20), Some(Vec::new()), true).unwrap(); + net.peer(0).client().finalize_block(BlockId::Number(10), Some(Justifications(Vec::new())), true).unwrap(); + net.peer(0).client().finalize_block(BlockId::Number(15), Some(Justifications(Vec::new())), true).unwrap(); + net.peer(0).client().finalize_block(BlockId::Number(20), Some(Justifications(Vec::new())), true).unwrap(); let h1 = net.peer(1).client().header(&BlockId::Number(10)).unwrap().unwrap(); let h2 = net.peer(1).client().header(&BlockId::Number(15)).unwrap().unwrap(); @@ -269,10 +270,10 @@ fn sync_justifications() { net.poll(cx); for height in (10..21).step_by(5) { - if net.peer(0).client().justification(&BlockId::Number(height)).unwrap() != Some(Vec::new()) { + if net.peer(0).client().justification(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { return Poll::Pending; } - if net.peer(1).client().justification(&BlockId::Number(height)).unwrap() != Some(Vec::new()) { + if net.peer(1).client().justification(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { return Poll::Pending; } } @@ -295,7 +296,7 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(Vec::new()), true).unwrap(); + net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(Justifications(Vec::new())), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); net.peer(1).request_justification(&f2_best, 11); @@ -303,8 +304,8 @@ fn sync_justifications_across_forks() { block_on(futures::future::poll_fn::<(), _>(|cx| { net.poll(cx); - if net.peer(0).client().justification(&BlockId::Number(10)).unwrap() == Some(Vec::new()) && - net.peer(1).client().justification(&BlockId::Number(10)).unwrap() == Some(Vec::new()) + if net.peer(0).client().justification(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) && + net.peer(1).client().justification(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) { Poll::Ready(()) } else { @@ -696,8 +697,9 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), Some(Vec::new()), true).unwrap(); - net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), Some(Vec::new()), true).unwrap(); + let just = Some(Justifications(Vec::new())); + net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); + net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); net.block_until_sync(); diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index f38aef008e11c..021c3ffecd66a 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -30,6 +30,7 @@ use std::sync::Arc; use sp_runtime::{ traits::{BlakeTwo256, HashFor, NumberFor}, generic::BlockId, traits::{Block as _, Header as HeaderT}, Digest, + Justifications, }; use std::collections::HashMap; use parking_lot::Mutex; @@ -378,7 +379,7 @@ fn execution_proof_is_generated_and_checked() { remote_client.import_justified( BlockOrigin::Own, remote_client.new_block(digest).unwrap().build().unwrap().block, - Default::default(), + Justifications(Default::default()), ).unwrap(); } diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 7420ead2b6632..97c2e74264c38 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -50,7 +50,7 @@ use sp_consensus::{ use sp_finality_grandpa::GRANDPA_ENGINE_ID; use sp_storage::StorageKey; use sp_trie::{TrieConfiguration, trie_types::Layout}; -use sp_runtime::{generic::BlockId, DigestItem}; +use sp_runtime::{generic::BlockId, DigestItem, Justifications}; use hex_literal::hex; mod light; @@ -1022,7 +1022,7 @@ fn import_with_justification() { client.import(BlockOrigin::Own, a2.clone()).unwrap(); // A2 -> A3 - let justification = vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]; + let justification = Justifications(vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]); let a3 = client.new_block_at( &BlockId::Hash(a2.hash()), Default::default(), @@ -1094,7 +1094,7 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { ); // importing B1 as finalized should trigger a re-org and set it as new best - let justification = vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]; + let justification = Justifications(vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]); client.import_justified(BlockOrigin::Own, b1.clone(), justification).unwrap(); assert_eq!( diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index 2d1cb56af53b1..883f34b4c8ea6 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -555,7 +555,7 @@ mod tests { libp2p::PeerId::random(), hash, 1, - Vec::new(), + sp_runtime::Justifications(Vec::new()), ))) .unwrap(); diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 3ad3ad5da63fa..f07e2ddfb35ca 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -97,8 +97,9 @@ pub use either::Either; /// bypasses this problem. pub type EncodedJustification = Vec; /// Collection of Justifications, since we might have more than one stored per block. -// WIP(JON): Consider making `Justifications` strongly typed. -pub type Justifications = Vec<(ConsensusEngineId, EncodedJustification)>; +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] +pub struct Justifications(pub Vec<(ConsensusEngineId, EncodedJustification)>); use traits::{Verify, Lazy}; From d72b5eb65eefa15b96b7dbbdfe02462b7cbaff6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 30 Nov 2020 23:55:39 +0100 Subject: [PATCH 03/58] Encode/decode Justifications --- client/network/src/block_requests.rs | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/client/network/src/block_requests.rs b/client/network/src/block_requests.rs index 391d570736113..a3e11683163d7 100644 --- a/client/network/src/block_requests.rs +++ b/client/network/src/block_requests.rs @@ -53,7 +53,7 @@ use libp2p::{ } }; use prost::Message; -use sp_runtime::{generic::BlockId, traits::{Block, Header, One, Zero}}; +use sp_runtime::{generic::BlockId, Justifications, traits::{Block, Header, One, Zero}}; use std::{ cmp::min, collections::{HashMap, VecDeque}, @@ -422,12 +422,7 @@ where Vec::new() }; - // WIP(JON): Need to encode the whole set of Justifications - let a = justification.unwrap_or(sp_runtime::Justifications(Vec::new())); - let justification = a - .0 - .first() - .map(|y| y.1.clone()); + let justification = justification.unwrap_or(sp_runtime::Justifications(Vec::new())).encode(); let block_data = schema::v1::BlockData { hash: hash.encode(), @@ -439,7 +434,7 @@ where body, receipt: Vec::new(), message_queue: Vec::new(), - justification: justification.unwrap_or_default(), + justification, is_empty_justification, }; @@ -656,12 +651,10 @@ where None }, justification: if !block_data.justification.is_empty() { - // WIP(JON): Need to decode the whole set of Justifications - // let id = sp_finality_grandpa::GRANDPA_ENGINE_ID; - const GRANDPA_ENGINE_ID: [u8; 4] = *b"FRNK"; - let just = vec![(GRANDPA_ENGINE_ID, block_data.justification)]; - Some(sp_runtime::Justifications(just)) - // Some(block_data.justification) + let justifications: Justifications = + Decode::decode(&mut &block_data.justification[..]) + .expect("WIP(JON): remove me"); + Some(justifications) } else if block_data.is_empty_justification { Some(sp_runtime::Justifications(Vec::new())) } else { From 88d1e1aba63db4f9965b203b0feade330758e790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 1 Dec 2020 14:58:28 +0100 Subject: [PATCH 04/58] primitives/runtime: add Justification type --- primitives/runtime/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index f07e2ddfb35ca..634dfd617517f 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -96,10 +96,14 @@ pub use either::Either; /// (and thus fork the chain). Sending a `Justification` alongside a block instead /// bypasses this problem. pub type EncodedJustification = Vec; + +/// Justification together with an ID to tag it's origin. +pub type Justification = (ConsensusEngineId, EncodedJustification); + /// Collection of Justifications, since we might have more than one stored per block. #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] -pub struct Justifications(pub Vec<(ConsensusEngineId, EncodedJustification)>); +pub struct Justifications(pub Vec); use traits::{Verify, Lazy}; From 12f8f7e318285193e39d6b029618ca3e3a9b7b3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 2 Dec 2020 11:45:55 +0100 Subject: [PATCH 05/58] backend: apply_finality and finalize_block takes a single Justification --- client/api/src/backend.rs | 6 +++--- .../consensus/manual-seal/src/finalize_block.rs | 4 ++-- client/consensus/manual-seal/src/rpc.rs | 8 ++++---- client/finality-grandpa/src/environment.rs | 2 +- client/finality-grandpa/src/import.rs | 3 ++- client/network/test/src/lib.rs | 8 +++++--- client/network/test/src/sync.rs | 16 ++++++++++------ client/service/src/client/client.rs | 11 ++++++----- primitives/runtime/src/lib.rs | 6 ++++++ test-utils/client/src/client_ext.rs | 6 +++--- 10 files changed, 42 insertions(+), 28 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index e665df4fe81c0..002f96ff1071a 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use std::collections::{HashMap, HashSet}; use sp_core::ChangesTrieConfigurationRange; use sp_core::offchain::{OffchainStorage,storage::OffchainOverlayedChanges}; -use sp_runtime::{generic::BlockId, Justifications, Storage}; +use sp_runtime::{generic::BlockId, Justification, Justifications, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor}; use sp_state_machine::{ ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction, @@ -226,7 +226,7 @@ pub trait Finalizer> { &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()>; @@ -247,7 +247,7 @@ pub trait Finalizer> { fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()>; diff --git a/client/consensus/manual-seal/src/finalize_block.rs b/client/consensus/manual-seal/src/finalize_block.rs index ed1ecce4c9a01..5780a25f97256 100644 --- a/client/consensus/manual-seal/src/finalize_block.rs +++ b/client/consensus/manual-seal/src/finalize_block.rs @@ -18,7 +18,7 @@ use crate::rpc; use sp_runtime::{ - Justifications, + Justification, traits::Block as BlockT, generic::BlockId, }; @@ -33,7 +33,7 @@ pub struct FinalizeBlockParams { /// sender to report errors/success to the rpc. pub sender: rpc::Sender<()>, /// finalization justification - pub justification: Option, + pub justification: Option, /// Finalizer trait object. pub finalizer: Arc, /// phantom type to pin the Backend type diff --git a/client/consensus/manual-seal/src/rpc.rs b/client/consensus/manual-seal/src/rpc.rs index 2fbfdeab3cd08..690b6c1eb9996 100644 --- a/client/consensus/manual-seal/src/rpc.rs +++ b/client/consensus/manual-seal/src/rpc.rs @@ -26,7 +26,7 @@ use futures::{ SinkExt }; use serde::{Deserialize, Serialize}; -use sp_runtime::Justifications; +use sp_runtime::Justification; pub use self::gen_client::Client as ManualSealClient; /// Future's type for jsonrpc @@ -60,7 +60,7 @@ pub enum EngineCommand { /// sender to report errors/success to the rpc. sender: Sender<()>, /// finalization justification - justification: Option, + justification: Option, } } @@ -81,7 +81,7 @@ pub trait ManualSealApi { fn finalize_block( &self, hash: Hash, - justification: Option + justification: Option ) -> FutureResult; } @@ -129,7 +129,7 @@ impl ManualSealApi for ManualSeal { Box::new(future.map_err(Error::from).compat()) } - fn finalize_block(&self, hash: Hash, justification: Option) -> FutureResult { + fn finalize_block(&self, hash: Hash, justification: Option) -> FutureResult { let mut sink = self.import_block_channel.clone(); let future = async move { let (sender, receiver) = oneshot::channel(); diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index 93b9f28bce026..cc301e77d94e7 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -1289,7 +1289,7 @@ where // ideally some handle to a synchronization oracle would be used // to avoid unconditionally notifying. - let justification = justification.map(|j| sp_runtime::Justifications(vec![(GRANDPA_ENGINE_ID, j.clone())])); + let justification = justification.map(|j| (GRANDPA_ENGINE_ID, j.clone())); client .apply_finality(import_op, BlockId::Hash(hash), justification, true) .map_err(|e| { diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index fcedca4419e99..c1b2b6d070838 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -620,11 +620,12 @@ where initial_sync: bool, ) -> Result<(), ConsensusError> { let grandpa_justification = + // WIP(JON): insert check for >1 grandpa justifications? match justification.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { Some((_, grandpa_justification)) => grandpa_justification, None => { return Err(ConsensusError::ClientImport( - "JON(WIP): expected a grandpa justification.".into(), + "WIP(JON): expected a grandpa justification.".into(), )) } }; diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index f17c77ef3c44d..0874b5e65f4cf 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -57,7 +57,7 @@ use sp_core::H256; use sc_network::config::ProtocolConfig; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; -use sp_runtime::Justifications; +use sp_runtime::{Justification, Justifications}; use substrate_test_runtime_client::{self, AccountKeyring}; use sc_service::client::Client; pub use sc_network::config::EmptyTransactionPool; @@ -202,7 +202,7 @@ impl PeersClient { pub fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool ) -> ClientResult<()> { match *self { @@ -953,7 +953,9 @@ impl JustificationImport for ForceFinalized { _number: NumberFor, justification: Justifications, ) -> Result<(), Self::Error> { - self.0.finalize_block(BlockId::Hash(hash), Some(justification), true) + // WIP(JON): Just finalize with the first Justification. TODO is to append the other ones as well + let justification = justification.0.first().cloned(); + self.0.finalize_block(BlockId::Hash(hash), justification, true) .map_err(|_| ConsensusError::InvalidJustification.into()) } } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 8f381d3145ecf..bc390fc2ef9a5 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -22,7 +22,9 @@ use futures::{Future, executor::block_on}; use super::*; use sp_consensus::block_validation::Validation; use substrate_test_runtime::Header; -use sp_runtime::Justifications; +use sp_runtime::{ConsensusEngineId, Justifications}; + +const ID: ConsensusEngineId = *b"TEST"; fn test_ancestor_search_when_common_is(n: usize) { sp_tracing::try_init_simple(); @@ -253,9 +255,10 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justification(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - net.peer(0).client().finalize_block(BlockId::Number(10), Some(Justifications(Vec::new())), true).unwrap(); - net.peer(0).client().finalize_block(BlockId::Number(15), Some(Justifications(Vec::new())), true).unwrap(); - net.peer(0).client().finalize_block(BlockId::Number(20), Some(Justifications(Vec::new())), true).unwrap(); + let just = (ID, Vec::new()); + net.peer(0).client().finalize_block(BlockId::Number(10), Some(just.clone()), true).unwrap(); + net.peer(0).client().finalize_block(BlockId::Number(15), Some(just.clone()), true).unwrap(); + net.peer(0).client().finalize_block(BlockId::Number(20), Some(just.clone()), true).unwrap(); let h1 = net.peer(1).client().header(&BlockId::Number(10)).unwrap().unwrap(); let h2 = net.peer(1).client().header(&BlockId::Number(15)).unwrap().unwrap(); @@ -296,7 +299,8 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(Justifications(Vec::new())), true).unwrap(); + let just = (ID, Vec::new()); + net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(just), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); net.peer(1).request_justification(&f2_best, 11); @@ -697,7 +701,7 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - let just = Some(Justifications(Vec::new())); + let just = Some((ID, Vec::new())); net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index e217e61909983..e00dc1159f4c7 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -37,7 +37,7 @@ use sp_core::{ use sp_keystore::SyncCryptoStorePtr; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_runtime::{ - Justifications, BuildStorage, + Justification, Justifications, BuildStorage, generic::{BlockId, SignedBlock, DigestItem}, traits::{ Block as BlockT, Header as HeaderT, Zero, NumberFor, @@ -1808,11 +1808,12 @@ impl Finalizer for Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { let last_best = self.backend.blockchain().info().best_hash; let to_finalize_hash = self.backend.blockchain().expect_block_hash_from_id(&id)?; + let justification = justification.map(Justifications::from); self.apply_finality_with_block_hash( operation, to_finalize_hash, @@ -1825,7 +1826,7 @@ impl Finalizer for Client where fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { self.lock_import_and_run(|operation| { @@ -1844,7 +1845,7 @@ impl Finalizer for &Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { (**self).apply_finality(operation, id, justification, notify) @@ -1853,7 +1854,7 @@ impl Finalizer for &Client where fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { (**self).finalize_block(id, justification, notify) diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 634dfd617517f..13e1bc067edda 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -105,6 +105,12 @@ pub type Justification = (ConsensusEngineId, EncodedJustification); #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] pub struct Justifications(pub Vec); +impl From for Justifications { + fn from(justification: Justification) -> Self { + Self(vec![justification]) + } +} + use traits::{Verify, Lazy}; /// A module identifier. These are per module and should be stored in a registry somewhere. diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index 05f1ffdb5cf3e..18106665cdb4e 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -24,7 +24,7 @@ use sp_consensus::{ BlockImportParams, BlockImport, BlockOrigin, Error as ConsensusError, ForkChoiceStrategy, }; -use sp_runtime::Justifications; +use sp_runtime::{Justification, Justifications}; use sp_runtime::traits::{Block as BlockT}; use sp_runtime::generic::BlockId; use codec::alloc::collections::hash_map::HashMap; @@ -35,7 +35,7 @@ pub trait ClientExt: Sized { fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()>; /// Returns hash of the genesis block. @@ -73,7 +73,7 @@ impl ClientExt for Client fn finalize_block( &self, id: BlockId, - justification: Option, + justification: Option, ) -> sp_blockchain::Result<()> { Finalizer::finalize_block(self, id, justification, true) } From 75dc977115e8c594e8b835cff2dfe79923ab298b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 2 Dec 2020 14:14:29 +0100 Subject: [PATCH 06/58] manual-seal: create engine id and let rpc take encoded justification --- client/consensus/manual-seal/src/lib.rs | 7 ++++++- client/consensus/manual-seal/src/rpc.rs | 8 ++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 31e79c667d61a..b5c037a212539 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -27,7 +27,7 @@ use sp_consensus::{ }; use sp_blockchain::HeaderBackend; use sp_inherents::InherentDataProviders; -use sp_runtime::{traits::Block as BlockT, Justifications}; +use sp_runtime::{traits::Block as BlockT, Justifications, ConsensusEngineId}; use sc_client_api::backend::{Backend as ClientBackend, Finalizer}; use sc_transaction_pool::txpool; use std::{sync::Arc, marker::PhantomData}; @@ -49,6 +49,10 @@ pub use self::{ }; use sp_api::{ProvideRuntimeApi, TransactionFor}; +/// The `ConsensusEngineId` of Manual Seal. +// WIP(JON): consider creating a new crate primitives/manual-seal for this +pub const MANUAL_SEAL_ENGINE_ID: ConsensusEngineId = [b'm', b'a', b's', b'e']; + /// The verifier for the manual seal engine; instantly finalizes. struct ManualSealVerifier; @@ -193,6 +197,7 @@ pub async fn run_manual_seal( ).await; } EngineCommand::FinalizeBlock { hash, sender, justification } => { + let justification = justification.map(|j| (MANUAL_SEAL_ENGINE_ID, j)); finalize_block( FinalizeBlockParams { hash, diff --git a/client/consensus/manual-seal/src/rpc.rs b/client/consensus/manual-seal/src/rpc.rs index 690b6c1eb9996..77188ad909353 100644 --- a/client/consensus/manual-seal/src/rpc.rs +++ b/client/consensus/manual-seal/src/rpc.rs @@ -26,7 +26,7 @@ use futures::{ SinkExt }; use serde::{Deserialize, Serialize}; -use sp_runtime::Justification; +use sp_runtime::EncodedJustification; pub use self::gen_client::Client as ManualSealClient; /// Future's type for jsonrpc @@ -60,7 +60,7 @@ pub enum EngineCommand { /// sender to report errors/success to the rpc. sender: Sender<()>, /// finalization justification - justification: Option, + justification: Option, } } @@ -81,7 +81,7 @@ pub trait ManualSealApi { fn finalize_block( &self, hash: Hash, - justification: Option + justification: Option ) -> FutureResult; } @@ -129,7 +129,7 @@ impl ManualSealApi for ManualSeal { Box::new(future.map_err(Error::from).compat()) } - fn finalize_block(&self, hash: Hash, justification: Option) -> FutureResult { + fn finalize_block(&self, hash: Hash, justification: Option) -> FutureResult { let mut sink = self.import_block_channel.clone(); let future = async move { let (sender, receiver) = oneshot::channel(); From 9920319dbb51f64b6c44e06198fd375f41109512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 2 Dec 2020 16:08:58 +0100 Subject: [PATCH 07/58] backend: skeleton functions for appending justifications --- client/api/src/backend.rs | 17 +++++++++++++++ client/api/src/in_mem.rs | 18 ++++++++++++++- client/db/src/lib.rs | 15 ++++++++++++- client/light/src/backend.rs | 10 ++++++++- client/service/src/client/client.rs | 34 +++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 002f96ff1071a..f9a529f6d2152 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -197,9 +197,17 @@ pub trait BlockImportOperation { id: BlockId, justification: Option, ) -> sp_blockchain::Result<()>; + /// Mark a block as new head. If both block import and set head are specified, set head /// overrides block import's best block rule. fn mark_head(&mut self, id: BlockId) -> sp_blockchain::Result<()>; + + /// Append justification to an already finalized block. + fn append_justification( + &mut self, + id: BlockId, + justification: Justification + ) -> sp_blockchain::Result<()>; } /// Interface for performing operations on the backend. @@ -251,6 +259,15 @@ pub trait Finalizer> { notify: bool, ) -> sp_blockchain::Result<()>; + /// Append Justification to an already finalized block. + /// + /// The already finalized block may or may not already have a Justification. If it does we append + /// the new Justification to the old one. + fn append_justification( + &self, + id: BlockId, + justification: Justification, + ) -> sp_blockchain::Result<()>; } /// Provides access to an auxiliary database. diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index fed7f9488fdb3..0496662a9f291 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -27,7 +27,7 @@ use sp_core::{ }; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HashFor}; -use sp_runtime::{Justifications, Storage}; +use sp_runtime::{Justification, Justifications, Storage}; use sp_state_machine::{ ChangesTrieTransaction, InMemoryBackend, Backend as StateBackend, StorageCollection, ChildStorageCollection, @@ -487,6 +487,8 @@ pub struct BlockImportOperation { aux: Vec<(Vec, Option>)>, finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, + // WIP(JON) + appended_justifications: Vec<(BlockId, Justification)>, } impl backend::BlockImportOperation for BlockImportOperation where @@ -581,6 +583,19 @@ impl backend::BlockImportOperation for BlockImportOperatio self.set_head = Some(block); Ok(()) } + + fn append_justification( + &mut self, + _block: BlockId, + _justification: Justification, + ) -> sp_blockchain::Result<()> { + // WIP(JON): How we append Justification depends on implementation of mark_finalized. Maybe + // we need to check self.finalized_blocks and update if it block already exists there. + // Otherwise probably need one additional field in BlockImportOperation. + + //self.appended_justifications.push((block, justification)); + todo!() + } } /// In-memory backend. Keeps all states and blocks in memory. @@ -636,6 +651,7 @@ impl backend::Backend for Backend where Block::Hash aux: Default::default(), finalized_blocks: Default::default(), set_head: None, + appended_justifications: Default::default(), }) } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index b63fe7d676660..50b380e38b50e 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -70,7 +70,7 @@ use sp_core::ChangesTrieConfiguration; use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; -use sp_runtime::{generic::{DigestItem, BlockId}, Justifications, Storage}; +use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Justifications, Storage}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, }; @@ -612,6 +612,7 @@ pub struct BlockImportOperation { aux_ops: Vec<(Vec, Option>)>, finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, + appended_justifications: Vec<(BlockId, Justification)>, commit_state: bool, } @@ -757,6 +758,17 @@ impl sc_client_api::backend::BlockImportOperation for Bloc self.set_head = Some(block); Ok(()) } + + fn append_justification( + &mut self, + _block: BlockId, + _justification: Justification, + ) -> sp_blockchain::Result<()> { + // WIP(JON): How we append Justification depends on implementation of mark_finalized. Maybe + // we need to check self.finalized_blocks and update if it block already exists there. + // Otherwise probably need one additional field in BlockImportOperation. + todo!(); + } } struct StorageDb { @@ -1463,6 +1475,7 @@ impl sc_client_api::backend::Backend for Backend { aux_ops: Vec::new(), finalized_blocks: Vec::new(), set_head: None, + appended_justifications: Vec::new(), commit_state: false, }) } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 7484bf29c6dc8..420f7856e3145 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -32,7 +32,7 @@ use sp_state_machine::{ Backend as StateBackend, TrieBackend, InMemoryBackend, ChangesTrieTransaction, StorageCollection, ChildStorageCollection, }; -use sp_runtime::{generic::BlockId, Justifications, Storage}; +use sp_runtime::{generic::BlockId, Justification, Justifications, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header, HashFor}; use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sc_client_api::{ @@ -366,6 +366,14 @@ impl BlockImportOperation for ImportOperation self.set_head = Some(block); Ok(()) } + + fn append_justification( + &mut self, + _block: BlockId, + _justification: Justification, + ) -> ClientResult<()> { + todo!(); + } } impl std::fmt::Debug for GenesisOrUnavailableState { diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index e00dc1159f4c7..c630c5253f1a9 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -964,6 +964,17 @@ impl Client where Ok(()) } + fn append_justification_with_block_hash( + &self, + operation: &mut ClientImportOperation, + block: Block::Hash, + justification: Justification, + ) -> sp_blockchain::Result<()> { + // WIP(JON): check to make sure `block` is already finalized + operation.op.append_justification(BlockId::Hash(block), justification)?; + todo!(); + } + fn notify_finalized( &self, notify_finalized: Vec, @@ -1833,6 +1844,21 @@ impl Finalizer for Client where self.apply_finality(operation, id, justification, notify) }) } + + fn append_justification( + &self, + id: BlockId, + justification: Justification, + )-> sp_blockchain::Result<()> { + let block_hash = self.backend.blockchain().expect_block_hash_from_id(&id)?; + self.lock_import_and_run(|operation| { + self.append_justification_with_block_hash( + operation, + block_hash, + justification, + ) + }) + } } @@ -1859,6 +1885,14 @@ impl Finalizer for &Client where ) -> sp_blockchain::Result<()> { (**self).finalize_block(id, justification, notify) } + + fn append_justification( + &self, + id: BlockId, + justification: Justification, + ) -> blockchain::Result<()> { + (**self).append_justification(id, justification) + } } impl BlockchainEvents for Client From 1f7f2d1c296fc31ece9370816b3cf55669b5406c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 10 Dec 2020 22:07:14 +0100 Subject: [PATCH 08/58] backend: initial implementation append_justification Initial implementation of append_justification on the Backend trait, and also remove unused skeleton functions for append_justificaton on Finaziler trait. k --- Cargo.lock | 1 - client/api/src/backend.rs | 27 +++------ client/api/src/in_mem.rs | 25 +++----- client/db/Cargo.toml | 1 - client/db/src/changes_tries_storage.rs | 7 ++- client/db/src/lib.rs | 81 +++++++++++++++++++++----- client/light/src/backend.rs | 17 +++--- client/service/src/client/client.rs | 34 ----------- 8 files changed, 97 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ef395ba152b5..1252ae7f2e1e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6602,7 +6602,6 @@ dependencies = [ "sp-consensus", "sp-core", "sp-database", - "sp-finality-grandpa", "sp-keyring", "sp-runtime", "sp-state-machine", diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index f9a529f6d2152..162946a9e6d5b 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -201,13 +201,6 @@ pub trait BlockImportOperation { /// Mark a block as new head. If both block import and set head are specified, set head /// overrides block import's best block rule. fn mark_head(&mut self, id: BlockId) -> sp_blockchain::Result<()>; - - /// Append justification to an already finalized block. - fn append_justification( - &mut self, - id: BlockId, - justification: Justification - ) -> sp_blockchain::Result<()>; } /// Interface for performing operations on the backend. @@ -238,7 +231,6 @@ pub trait Finalizer> { notify: bool, ) -> sp_blockchain::Result<()>; - /// Finalize a block. /// /// This will implicitly finalize all blocks up to it and @@ -258,16 +250,6 @@ pub trait Finalizer> { justification: Option, notify: bool, ) -> sp_blockchain::Result<()>; - - /// Append Justification to an already finalized block. - /// - /// The already finalized block may or may not already have a Justification. If it does we append - /// the new Justification to the old one. - fn append_justification( - &self, - id: BlockId, - justification: Justification, - ) -> sp_blockchain::Result<()>; } /// Provides access to an auxiliary database. @@ -449,6 +431,15 @@ pub trait Backend: AuxStore + Send + Sync { justification: Option, ) -> sp_blockchain::Result<()>; + /// Append justification to the block with the given Id. + /// + /// This should only be called for blocks that are already finalized. + fn append_justification( + &self, + block: BlockId, + justification: Justification, + ) -> sp_blockchain::Result<()>; + /// Returns reference to blockchain backend. fn blockchain(&self) -> &Self::Blockchain; diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 0496662a9f291..45435cca0f827 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -487,8 +487,6 @@ pub struct BlockImportOperation { aux: Vec<(Vec, Option>)>, finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, - // WIP(JON) - appended_justifications: Vec<(BlockId, Justification)>, } impl backend::BlockImportOperation for BlockImportOperation where @@ -583,19 +581,6 @@ impl backend::BlockImportOperation for BlockImportOperatio self.set_head = Some(block); Ok(()) } - - fn append_justification( - &mut self, - _block: BlockId, - _justification: Justification, - ) -> sp_blockchain::Result<()> { - // WIP(JON): How we append Justification depends on implementation of mark_finalized. Maybe - // we need to check self.finalized_blocks and update if it block already exists there. - // Otherwise probably need one additional field in BlockImportOperation. - - //self.appended_justifications.push((block, justification)); - todo!() - } } /// In-memory backend. Keeps all states and blocks in memory. @@ -651,7 +636,6 @@ impl backend::Backend for Backend where Block::Hash aux: Default::default(), finalized_blocks: Default::default(), set_head: None, - appended_justifications: Default::default(), }) } @@ -709,6 +693,15 @@ impl backend::Backend for Backend where Block::Hash self.blockchain.finalize_header(block, justification) } + fn append_justification( + &self, + _block: BlockId, + _justification: Justification, + ) -> sp_blockchain::Result<()> { + // WIP(JON) + todo!(); + } + fn blockchain(&self) -> &Self::Blockchain { &self.blockchain } diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 7c2bbbde1b7cd..70a0b19532593 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -42,7 +42,6 @@ prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0. sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } sp-tracing = { version = "2.0.0", path = "../../primitives/tracing" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -sp-finality-grandpa = { version = "2.0.0", path = "../../primitives/finality-grandpa" } quickcheck = "0.9" kvdb-rocksdb = "0.9.1" tempfile = "3" diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 245868a8876f9..14518fe7c9d0a 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -529,14 +529,15 @@ mod tests { }; use sp_blockchain::HeaderBackend as BlockchainHeaderBackend; use sp_core::H256; - use sp_finality_grandpa::GRANDPA_ENGINE_ID; - use sp_runtime::{Justifications, testing::{Digest, Header}}; + use sp_runtime::{ConsensusEngineId, Justifications, testing::{Digest, Header}}; use sp_runtime::traits::{Hash, BlakeTwo256}; use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage}; use crate::Backend; use crate::tests::{Block, insert_header, prepare_changes}; use super::*; + const CONS0_ENGINE_ID: ConsensusEngineId = *b"CON0"; + fn changes(number: u64) -> Option, Vec)>> { Some(vec![(number.to_le_bytes().to_vec(), number.to_le_bytes().to_vec())]) } @@ -954,7 +955,7 @@ mod tests { let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); let config1 = Some(ChangesTrieConfiguration::new(2, 6)); let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - let just1 = Some(Justifications(vec![(GRANDPA_ENGINE_ID, vec![42])])); + let just1 = Some(Justifications(vec![(CONS0_ENGINE_ID, vec![42])])); backend.finalize_block(BlockId::Number(1), just1).unwrap(); let config2 = Some(ChangesTrieConfiguration::new(2, 7)); let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 50b380e38b50e..0d865f78c0cfd 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -612,7 +612,6 @@ pub struct BlockImportOperation { aux_ops: Vec<(Vec, Option>)>, finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, - appended_justifications: Vec<(BlockId, Justification)>, commit_state: bool, } @@ -758,17 +757,6 @@ impl sc_client_api::backend::BlockImportOperation for Bloc self.set_head = Some(block); Ok(()) } - - fn append_justification( - &mut self, - _block: BlockId, - _justification: Justification, - ) -> sp_blockchain::Result<()> { - // WIP(JON): How we append Justification depends on implementation of mark_finalized. Maybe - // we need to check self.finalized_blocks and update if it block already exists there. - // Otherwise probably need one additional field in BlockImportOperation. - todo!(); - } } struct StorageDb { @@ -1475,7 +1463,6 @@ impl sc_client_api::backend::Backend for Backend { aux_ops: Vec::new(), finalized_blocks: Vec::new(), set_head: None, - appended_justifications: Vec::new(), commit_state: false, }) } @@ -1537,6 +1524,43 @@ impl sc_client_api::backend::Backend for Backend { Ok(()) } + fn append_justification( + &self, + block: BlockId, + justification: Justification, + ) -> ClientResult<()> { + // WIP(JON): Check is block is finalized first + + // Create a Transaction. + let mut transaction: Transaction = Transaction::new(); + let hash = self.blockchain.expect_block_hash_from_id(&block)?; + let header = self.blockchain.expect_header(block)?; + let number = *header.number(); + + // Read and merge Justification + use sp_blockchain::Backend; + let justifications = if let Some(mut stored_justifications) = self.blockchain.justification(block)? { + stored_justifications.0.push(justification); + stored_justifications + } else { + Justifications::from(justification) + }; + + // WIP(JON): Reject if we now have more than one per consensus engine. + + // Set Justification in Transaction in the same way as in finalize_block_with_transaction + transaction.set_from_vec( + columns::JUSTIFICATION, + &utils::number_and_hash_to_lookup_key(number, hash)?, + justifications.encode(), + ); + + // Commit to storage + self.storage.db.commit(transaction)?; + + Ok(()) + } + fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage> { Some(&self.changes_tries_storage) } @@ -1787,12 +1811,15 @@ pub(crate) mod tests { use sp_core::H256; use sc_client_api::backend::{Backend as BTrait, BlockImportOperation as Op}; use sc_client_api::blockchain::Backend as BLBTrait; + use sp_runtime::ConsensusEngineId; use sp_runtime::testing::{Header, Block as RawBlock, ExtrinsicWrapper}; use sp_runtime::traits::{Hash, BlakeTwo256}; use sp_runtime::generic::DigestItem; use sp_state_machine::{TrieMut, TrieDBMut}; use sp_blockchain::{lowest_common_ancestor, tree_route}; - use sp_finality_grandpa::GRANDPA_ENGINE_ID; + + const CONS0_ENGINE_ID: ConsensusEngineId = *b"CON0"; + const CONS1_ENGINE_ID: ConsensusEngineId = *b"CON1"; pub(crate) type Block = RawBlock>; @@ -2368,7 +2395,7 @@ pub(crate) mod tests { let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); let _ = insert_header(&backend, 1, block0, None, Default::default()); - let justification = Some(Justifications(vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])])); + let justification = Some(Justifications(vec![(CONS0_ENGINE_ID, vec![1, 2, 3])])); backend.finalize_block(BlockId::Number(1), justification.clone()).unwrap(); assert_eq!( @@ -2377,6 +2404,30 @@ pub(crate) mod tests { ); } + #[test] + fn test_append_justification_to_finalized_block() { + use sc_client_api::blockchain::{Backend as BlockChainBackend}; + + let backend = Backend::::new_test(10, 10); + + let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); + let _ = insert_header(&backend, 1, block0, None, Default::default()); + + let just0 = (CONS0_ENGINE_ID, vec![1, 2, 3]); + backend.finalize_block( + BlockId::Number(1), + Some(Justifications(vec![just0.clone()])), + ).unwrap(); + + let just1 = (CONS1_ENGINE_ID, vec![4, 5]); + backend.append_justification(BlockId::Number(1), just1.clone()).unwrap(); + + assert_eq!( + backend.blockchain().justification(BlockId::Number(1)).unwrap(), + Some(Justifications(vec![just0, just1])), + ); + } + #[test] fn test_finalize_multiple_blocks_in_single_op() { let backend = Backend::::new_test(10, 10); diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 420f7856e3145..e61fd7331c0d2 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -199,6 +199,15 @@ impl ClientBackend for Backend> self.blockchain.storage().finalize_header(block) } + fn append_justification( + &self, + _block: BlockId, + _justification: Justification, + ) -> ClientResult<()> { + // WIP(JON) + todo!(); + } + fn blockchain(&self) -> &Blockchain { &self.blockchain } @@ -366,14 +375,6 @@ impl BlockImportOperation for ImportOperation self.set_head = Some(block); Ok(()) } - - fn append_justification( - &mut self, - _block: BlockId, - _justification: Justification, - ) -> ClientResult<()> { - todo!(); - } } impl std::fmt::Debug for GenesisOrUnavailableState { diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index c630c5253f1a9..e00dc1159f4c7 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -964,17 +964,6 @@ impl Client where Ok(()) } - fn append_justification_with_block_hash( - &self, - operation: &mut ClientImportOperation, - block: Block::Hash, - justification: Justification, - ) -> sp_blockchain::Result<()> { - // WIP(JON): check to make sure `block` is already finalized - operation.op.append_justification(BlockId::Hash(block), justification)?; - todo!(); - } - fn notify_finalized( &self, notify_finalized: Vec, @@ -1844,21 +1833,6 @@ impl Finalizer for Client where self.apply_finality(operation, id, justification, notify) }) } - - fn append_justification( - &self, - id: BlockId, - justification: Justification, - )-> sp_blockchain::Result<()> { - let block_hash = self.backend.blockchain().expect_block_hash_from_id(&id)?; - self.lock_import_and_run(|operation| { - self.append_justification_with_block_hash( - operation, - block_hash, - justification, - ) - }) - } } @@ -1885,14 +1859,6 @@ impl Finalizer for &Client where ) -> sp_blockchain::Result<()> { (**self).finalize_block(id, justification, notify) } - - fn append_justification( - &self, - id: BlockId, - justification: Justification, - ) -> blockchain::Result<()> { - (**self).append_justification(id, justification) - } } impl BlockchainEvents for Client From 523038a50c76da5fa1c00d36f7a65d4ff3a19b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 11 Dec 2020 11:01:22 +0100 Subject: [PATCH 09/58] backend: guard against duplicate consensus engine id --- client/db/src/lib.rs | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 0d865f78c0cfd..2706041d2a954 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1529,7 +1529,7 @@ impl sc_client_api::backend::Backend for Backend { block: BlockId, justification: Justification, ) -> ClientResult<()> { - // WIP(JON): Check is block is finalized first + // WIP(JON): Check if block is finalized first // Create a Transaction. let mut transaction: Transaction = Transaction::new(); @@ -1539,14 +1539,22 @@ impl sc_client_api::backend::Backend for Backend { // Read and merge Justification use sp_blockchain::Backend; - let justifications = if let Some(mut stored_justifications) = self.blockchain.justification(block)? { - stored_justifications.0.push(justification); - stored_justifications - } else { - Justifications::from(justification) - }; - - // WIP(JON): Reject if we now have more than one per consensus engine. + let justifications = + if let Some(mut stored_justifications) = self.blockchain.justification(block)? { + if stored_justifications + .0 + .iter() + .find(|stored| stored.0 == justification.0) + .is_some() + { + // WIP(JON): Maybe create new error type for this + return Err(ClientError::BadJustification("Duplicate".into())); + } + stored_justifications.0.push(justification); + stored_justifications + } else { + Justifications::from(justification) + }; // Set Justification in Transaction in the same way as in finalize_block_with_transaction transaction.set_from_vec( @@ -2422,6 +2430,12 @@ pub(crate) mod tests { let just1 = (CONS1_ENGINE_ID, vec![4, 5]); backend.append_justification(BlockId::Number(1), just1.clone()).unwrap(); + let just2 = (CONS1_ENGINE_ID, vec![6, 7]); + assert!(matches!( + backend.append_justification(BlockId::Number(1), just2), + Err(ClientError::BadJustification(_)) + )); + assert_eq!( backend.blockchain().justification(BlockId::Number(1)).unwrap(), Some(Justifications(vec![just0, just1])), From 52a0dffe06173b2daf6bd838eda0f2ac6f4bb795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 14 Dec 2020 11:32:19 +0100 Subject: [PATCH 10/58] client/db: add check for block finality --- client/db/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 2706041d2a954..007639e146d36 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1529,14 +1529,17 @@ impl sc_client_api::backend::Backend for Backend { block: BlockId, justification: Justification, ) -> ClientResult<()> { - // WIP(JON): Check if block is finalized first - - // Create a Transaction. let mut transaction: Transaction = Transaction::new(); let hash = self.blockchain.expect_block_hash_from_id(&block)?; let header = self.blockchain.expect_header(block)?; let number = *header.number(); + // Check if block is finalized first + // WIP(JON): is this really the correct way to check if a block is final? + if number > self.blockchain.info().finalized_number { + return Err(ClientError::BadJustification("WIP: block not finalized".into())); + } + // Read and merge Justification use sp_blockchain::Backend; let justifications = From a25d33d6d20a6f07583728a7273dab9bc17483c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 14 Dec 2020 17:17:42 +0100 Subject: [PATCH 11/58] client/api: add append_justification to in_mem db --- client/api/src/in_mem.rs | 102 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 4 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 45435cca0f827..272a51ff17143 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -295,6 +295,39 @@ impl Blockchain { Ok(()) } + fn append_justification(&self, id: BlockId, justification: Justification) + -> sp_blockchain::Result<()> + { + let hash = self.expect_block_hash_from_id(&id)?; + let mut storage = self.storage.write(); + + let block = storage + .blocks + .get_mut(&hash) + .expect("hash was fetched from a block in the db; qed"); + + let block_justification = match block { + StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j + }; + + if let Some(stored_justifications) = block_justification { + if stored_justifications + .0 + .iter() + .find(|stored| stored.0 == justification.0) + .is_some() + { + // WIP(JON): Maybe create new error type for this + return Err(sp_blockchain::Error::BadJustification("Duplicate".into())); + } + stored_justifications.0.push(justification); + } else { + *block_justification = Some(Justifications::from(justification)); + }; + + Ok(()) + } + fn write_aux(&self, ops: Vec<(Vec, Option>)>) { let mut storage = self.storage.write(); for (k, v) in ops { @@ -695,11 +728,10 @@ impl backend::Backend for Backend where Block::Hash fn append_justification( &self, - _block: BlockId, - _justification: Justification, + block: BlockId, + justification: Justification, ) -> sp_blockchain::Result<()> { - // WIP(JON) - todo!(); + self.blockchain.append_justification(block, justification) } fn blockchain(&self) -> &Self::Blockchain { @@ -772,3 +804,65 @@ pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use crate::{NewBlockState, in_mem::Blockchain}; + use sp_api::{BlockId, HeaderT}; + use sp_runtime::{ConsensusEngineId, Justifications}; + use sp_blockchain::Backend; + use substrate_test_runtime::{Block, Header, H256}; + + pub const ID1: ConsensusEngineId = *b"TST1"; + pub const ID2: ConsensusEngineId = *b"TST2"; + + fn header(number: u64) -> Header { + let parent_hash = match number { + 0 => Default::default(), + _ => header(number - 1).hash(), + }; + Header::new(number, H256::from_low_u64_be(0), H256::from_low_u64_be(0), parent_hash, Default::default()) + } + + fn test_blockchain() -> Blockchain { + let blockchain = Blockchain::::new(); + let just0 = Some(Justifications(vec![(ID1, vec![0])])); + let just1 = Some(Justifications(vec![(ID1, vec![1])])); + let just2 = None; + let just3 = Some(Justifications(vec![(ID1, vec![3])])); + blockchain.insert(header(0).hash(), header(0), just0, None, NewBlockState::Final).unwrap(); + blockchain.insert(header(1).hash(), header(1), just1, None, NewBlockState::Final).unwrap(); + blockchain.insert(header(2).hash(), header(2), just2, None, NewBlockState::Best).unwrap(); + blockchain.insert(header(3).hash(), header(3), just3, None, NewBlockState::Final).unwrap(); + blockchain + } + + #[test] + fn append_and_retreive_justifications() { + let blockchain = test_blockchain(); + let last_finalized = blockchain.last_finalized().unwrap(); + let block = BlockId::Hash(last_finalized); + + blockchain.append_justification(block, (ID2, vec![4])).unwrap(); + assert_eq!( + blockchain.justification(block).unwrap(), + Some(Justifications(vec![ + (ID1, vec![3]), + (ID2, vec![4]), + ])), + ); + } + + #[test] + fn store_duplicate_justifications_is_forbidden() { + let blockchain = test_blockchain(); + let last_finalized = blockchain.last_finalized().unwrap(); + let block = BlockId::Hash(last_finalized); + + blockchain.append_justification(block, (ID2, vec![0])).unwrap(); + assert!(matches!( + blockchain.append_justification(block, (ID2, vec![1])), + Err(sp_blockchain::Error::BadJustification(_)), + )); + } +} From f023ccada8fe67c2722af3a93a2254947dae819a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 15 Dec 2020 13:24:49 +0100 Subject: [PATCH 12/58] client/light: add no-op append_justification --- client/light/src/backend.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index e61fd7331c0d2..cf44f35e9cbcb 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -204,8 +204,7 @@ impl ClientBackend for Backend> _block: BlockId, _justification: Justification, ) -> ClientResult<()> { - // WIP(JON) - todo!(); + Ok(()) } fn blockchain(&self) -> &Blockchain { From 544c8e2acbc237ccbb1b7b95561f7dfc7deba15b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 14 Jan 2021 11:47:10 +0100 Subject: [PATCH 13/58] network: fix decode call for Justification --- Cargo.lock | 1 + client/network/Cargo.toml | 1 + client/network/src/protocol.rs | 11 ++++++----- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23060f8342db8..39e05683b83a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7188,6 +7188,7 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-core", + "sp-finality-grandpa", "sp-keyring", "sp-runtime", "sp-test-primitives", diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index a300dac19bfcc..85274cf61dc40 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -53,6 +53,7 @@ smallvec = "1.5.0" sp-arithmetic = { version = "2.0.0", path = "../../primitives/arithmetic" } sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } sp-consensus = { version = "0.8.0", path = "../../primitives/consensus/common" } +sp-finality-grandpa = { version = "2.0.0", path = "../../primitives/finality-grandpa" } sp-core = { version = "2.0.0", path = "../../primitives/core" } sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } sp-utils = { version = "2.0.0", path = "../../primitives/utils" } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index c56e1df1f62ff..5244445f63a4c 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -47,7 +47,7 @@ use sp_consensus::{ block_validation::BlockAnnounceValidator, import_queue::{BlockImportResult, BlockImportError, IncomingBlock, Origin} }; -use sp_runtime::{generic::BlockId, Justifications}; +use sp_runtime::{generic::BlockId, EncodedJustification, Justifications}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub }; @@ -770,10 +770,11 @@ impl Protocol { None }, justification: if !block_data.justification.is_empty() { - let justifications: Justifications = - Decode::decode(&mut &block_data.justification[..]) - .expect("WIP(JON): remove me"); - Some(justifications) + // For compatibility we assume that the incoming Justifications is an + // `EncodedJustification`, as before. + let justification: EncodedJustification = block_data.justification; + let justifications = vec![(sp_finality_grandpa::GRANDPA_ENGINE_ID, justification)]; + Some(Justifications(justifications)) } else if block_data.is_empty_justification { Some(Justifications(Vec::new())) } else { From 6aabece22b2e7a5e4d4c4626d13d7cd4978df49e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 14 Jan 2021 15:35:09 +0100 Subject: [PATCH 14/58] network: only send a single Justification in BlockData --- client/network/src/block_request_handler.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 407d351ef2064..2510ce6a2cfd4 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -28,9 +28,9 @@ use futures::channel::{mpsc, oneshot}; use futures::stream::StreamExt; use log::debug; use prost::Message; -use sp_runtime::Justifications; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, Header, One, Zero}; +use sp_finality_grandpa::GRANDPA_ENGINE_ID; use std::cmp::min; use std::sync::{Arc}; use std::time::Duration; @@ -122,15 +122,21 @@ impl BlockRequestHandler { let number = *header.number(); let hash = header.hash(); let parent_hash = *header.parent_hash(); - let justification = if get_justification { + let justifications = if get_justification { self.client.justification(&BlockId::Hash(hash))? } else { None }; - let is_empty_justification = justification - .as_ref() - .map(|j| j.0.is_empty()) - .unwrap_or(false); + + // To keep compatibility we only send the grandpa justification + let justification = justifications + .map(|just| + match just.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + Some((_, grandpa_justification)) => grandpa_justification, + None => Vec::new(), + }) + .unwrap_or(Vec::new()); + let is_empty_justification = justification.is_empty(); let body = if get_body { match self.client.block_body(&BlockId::Hash(hash))? { @@ -146,8 +152,6 @@ impl BlockRequestHandler { Vec::new() }; - let justification = justification.unwrap_or(Justifications(Vec::new())).encode(); - let block_data = crate::schema::v1::BlockData { hash: hash.encode(), header: if get_header { From 10e9f22899ea577cfe95f6493bb8c3848cdcf361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 14 Jan 2021 15:44:27 +0100 Subject: [PATCH 15/58] network: minor comment update --- client/network/src/protocol.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 5244445f63a4c..7c5b88e79fa3d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -771,7 +771,7 @@ impl Protocol { }, justification: if !block_data.justification.is_empty() { // For compatibility we assume that the incoming Justifications is an - // `EncodedJustification`, as before. + // `EncodedJustification`, as before, and that it can only be for GRANDPA. let justification: EncodedJustification = block_data.justification; let justifications = vec![(sp_finality_grandpa::GRANDPA_ENGINE_ID, justification)]; Some(Justifications(justifications)) From 3613fdba4585d55eb73cfed2e940bb5a6d92afce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 14 Jan 2021 22:00:36 +0100 Subject: [PATCH 16/58] protocol: update field names to distinguish single justification --- client/network/src/protocol.rs | 4 ++-- client/network/src/protocol/message.rs | 4 ++-- client/network/src/protocol/sync.rs | 10 +++++----- client/network/src/protocol/sync/blocks.rs | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 7c5b88e79fa3d..f9af6ccf3a8d2 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -769,7 +769,7 @@ impl Protocol { } else { None }, - justification: if !block_data.justification.is_empty() { + justifications: if !block_data.justification.is_empty() { // For compatibility we assume that the incoming Justifications is an // `EncodedJustification`, as before, and that it can only be for GRANDPA. let justification: EncodedJustification = block_data.justification; @@ -1232,7 +1232,7 @@ impl Protocol { body: None, receipt: None, message_queue: None, - justification: None, + justifications: None, }, ], }, diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 887e269b53bf6..1e97f4cf2f0c6 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -233,8 +233,8 @@ pub mod generic { pub receipt: Option>, /// Block message queue if requested. pub message_queue: Option>, - /// Justification if requested. - pub justification: Option, + /// Justifications if requested. + pub justifications: Option, } /// Identifies starting point of a block sequence. diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index fd5ef035ba8e5..a13ab32da03d0 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -821,7 +821,7 @@ impl ChainSync { hash: block_data.block.hash, header: block_data.block.header, body: block_data.block.body, - justification: block_data.block.justification, + justification: block_data.block.justifications, origin: block_data.origin, allow_missing_state: true, import_existing: false, @@ -840,7 +840,7 @@ impl ChainSync { hash: b.hash, header: b.header, body: b.body, - justification: b.justification, + justification: b.justifications, origin: Some(who.clone()), allow_missing_state: true, import_existing: false, @@ -949,7 +949,7 @@ impl ChainSync { hash: b.hash, header: b.header, body: b.body, - justification: b.justification, + justification: b.justifications, origin: Some(who.clone()), allow_missing_state: true, import_existing: false, @@ -1020,7 +1020,7 @@ impl ChainSync { return Err(BadPeer(who, rep::BAD_JUSTIFICATION)); } - block.justification + block.justifications } else { // we might have asked the peer for a justification on a block that we assumed it // had but didn't (regardless of whether it had a justification for it or not). @@ -2054,7 +2054,7 @@ mod test { body: Some(b.deconstruct().1), receipt: None, message_queue: None, - justification: None, + justifications: None, } ).collect(), } diff --git a/client/network/src/protocol/sync/blocks.rs b/client/network/src/protocol/sync/blocks.rs index 60492f24ed8c3..a277a1524f773 100644 --- a/client/network/src/protocol/sync/blocks.rs +++ b/client/network/src/protocol/sync/blocks.rs @@ -227,7 +227,7 @@ mod test { body: None, message_queue: None, receipt: None, - justification: None, + justifications: None, }).collect() } From ce5dad9deec692fbfdfc7faf3b22180d55fcf3a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 14 Jan 2021 22:46:04 +0100 Subject: [PATCH 17/58] client: further field renames to plural --- client/network/src/protocol/sync.rs | 6 +++--- client/network/test/src/block_import.rs | 4 ++-- client/service/src/chain_ops/import_blocks.rs | 2 +- primitives/consensus/common/src/import_queue.rs | 4 ++-- primitives/consensus/common/src/import_queue/basic_queue.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index a13ab32da03d0..5fd9a694523c3 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -821,7 +821,7 @@ impl ChainSync { hash: block_data.block.hash, header: block_data.block.header, body: block_data.block.body, - justification: block_data.block.justifications, + justifications: block_data.block.justifications, origin: block_data.origin, allow_missing_state: true, import_existing: false, @@ -840,7 +840,7 @@ impl ChainSync { hash: b.hash, header: b.header, body: b.body, - justification: b.justifications, + justifications: b.justifications, origin: Some(who.clone()), allow_missing_state: true, import_existing: false, @@ -949,7 +949,7 @@ impl ChainSync { hash: b.hash, header: b.header, body: b.body, - justification: b.justifications, + justifications: b.justifications, origin: Some(who.clone()), allow_missing_state: true, import_existing: false, diff --git a/client/network/test/src/block_import.rs b/client/network/test/src/block_import.rs index 4000e53420b4a..e90ca978fd268 100644 --- a/client/network/test/src/block_import.rs +++ b/client/network/test/src/block_import.rs @@ -35,13 +35,13 @@ fn prepare_good_block() -> (TestClient, Hash, u64, PeerId, IncomingBlock) let (hash, number) = (client.block_hash(1).unwrap().unwrap(), 1); let header = client.header(&BlockId::Number(1)).unwrap(); - let justification = client.justification(&BlockId::Number(1)).unwrap(); + let justifications = client.justification(&BlockId::Number(1)).unwrap(); let peer_id = PeerId::random(); (client, hash, number, peer_id.clone(), IncomingBlock { hash, header, body: Some(Vec::new()), - justification, + justifications, origin: Some(peer_id.clone()), allow_missing_state: false, import_existing: false, diff --git a/client/service/src/chain_ops/import_blocks.rs b/client/service/src/chain_ops/import_blocks.rs index 3f918e05120e9..23a6b9f01e300 100644 --- a/client/service/src/chain_ops/import_blocks.rs +++ b/client/service/src/chain_ops/import_blocks.rs @@ -168,7 +168,7 @@ fn import_block_to_queue( hash, header: Some(header), body: Some(extrinsics), - justification: signed_block.justification, + justifications: signed_block.justification, origin: None, allow_missing_state: false, import_existing: force, diff --git a/primitives/consensus/common/src/import_queue.rs b/primitives/consensus/common/src/import_queue.rs index 0891d830a1fcd..b399bd7b8a5e6 100644 --- a/primitives/consensus/common/src/import_queue.rs +++ b/primitives/consensus/common/src/import_queue.rs @@ -69,7 +69,7 @@ pub struct IncomingBlock { /// Block body if requested. pub body: Option::Extrinsic>>, /// Justification if requested. - pub justification: Option, + pub justifications: Option, /// The peer, we received this from pub origin: Option, /// Allow importing the block skipping state verification if parent state is missing. @@ -182,7 +182,7 @@ pub(crate) fn import_single_block_metered, Transaction ) -> Result>, BlockImportError> { let peer = block.origin; - let (header, justification) = match (block.header, block.justification) { + let (header, justification) = match (block.header, block.justifications) { (Some(header), justification) => (header, justification), (None, _) => { if let Some(ref peer) = peer { diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index feb60ef6236c4..e3e8633ed25e0 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -537,7 +537,7 @@ mod tests { hash, header: Some(header), body: None, - justification: None, + justifications: None, origin: None, allow_missing_state: false, import_existing: false, From b0f9b5c91e98f1965f7c1c37742fb014a517c798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 14 Jan 2021 23:23:38 +0100 Subject: [PATCH 18/58] client: update function names to plural justifications --- client/api/src/client.rs | 4 ++-- client/api/src/in_mem.rs | 8 ++++---- client/db/src/lib.rs | 8 ++++---- client/finality-grandpa/src/finality_proof.rs | 2 +- client/finality-grandpa/src/tests.rs | 10 +++++----- client/light/src/blockchain.rs | 2 +- client/network/src/block_request_handler.rs | 2 +- client/network/test/src/block_import.rs | 2 +- client/network/test/src/lib.rs | 6 +++--- client/network/test/src/sync.rs | 12 ++++++------ client/service/src/client/client.rs | 6 +++--- client/service/test/src/client/mod.rs | 6 +++--- primitives/blockchain/src/backend.rs | 4 ++-- 13 files changed, 36 insertions(+), 36 deletions(-) diff --git a/client/api/src/client.rs b/client/api/src/client.rs index aa823d5c192ca..f75dc42a01b00 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -90,8 +90,8 @@ pub trait BlockBackend { /// Get block status. fn block_status(&self, id: &BlockId) -> sp_blockchain::Result; - /// Get block justification set by id. - fn justification(&self, id: &BlockId) -> sp_blockchain::Result>; + /// Get block justifications set by id. + fn justifications(&self, id: &BlockId) -> sp_blockchain::Result>; /// Get block hash by number. fn block_hash(&self, number: NumberFor) -> sp_blockchain::Result>; diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index bfa5bf8d073c9..7953d90162438 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -70,7 +70,7 @@ impl StoredBlock { } } - fn justification(&self) -> Option<&Justifications> { + fn justifications(&self) -> Option<&Justifications> { match *self { StoredBlock::Header(_, ref j) | StoredBlock::Full(_, ref j) => j.as_ref() } @@ -398,9 +398,9 @@ impl blockchain::Backend for Blockchain { })) } - fn justification(&self, id: BlockId) -> sp_blockchain::Result> { + fn justifications(&self, id: BlockId) -> sp_blockchain::Result> { Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).and_then(|b| - b.justification().map(|x| x.clone())) + b.justifications().map(|x| x.clone())) )) } @@ -845,7 +845,7 @@ mod tests { blockchain.append_justification(block, (ID2, vec![4])).unwrap(); assert_eq!( - blockchain.justification(block).unwrap(), + blockchain.justifications(block).unwrap(), Some(Justifications(vec![ (ID1, vec![3]), (ID2, vec![4]), diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 21033dadc13e7..f68ddbeaadeca 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -486,7 +486,7 @@ impl sc_client_api::blockchain::Backend for BlockchainDb) -> ClientResult> { + fn justifications(&self, id: BlockId) -> ClientResult> { match read_db(&*self.db, columns::KEY_LOOKUP, columns::JUSTIFICATION, id)? { Some(justification) => match Decode::decode(&mut &justification[..]) { Ok(justification) => Ok(Some(justification)), @@ -1543,7 +1543,7 @@ impl sc_client_api::backend::Backend for Backend { // Read and merge Justification use sp_blockchain::Backend; let justifications = - if let Some(mut stored_justifications) = self.blockchain.justification(block)? { + if let Some(mut stored_justifications) = self.blockchain.justifications(block)? { if stored_justifications .0 .iter() @@ -2410,7 +2410,7 @@ pub(crate) mod tests { backend.finalize_block(BlockId::Number(1), justification.clone()).unwrap(); assert_eq!( - backend.blockchain().justification(BlockId::Number(1)).unwrap(), + backend.blockchain().justifications(BlockId::Number(1)).unwrap(), justification, ); } @@ -2440,7 +2440,7 @@ pub(crate) mod tests { )); assert_eq!( - backend.blockchain().justification(BlockId::Number(1)).unwrap(), + backend.blockchain().justifications(BlockId::Number(1)).unwrap(), Some(Justifications(vec![just0, just1])), ); } diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index 80227514e6af9..8110f96bba2f4 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -334,7 +334,7 @@ pub(crate) fn prove_finality, J>( unknown_headers.push(unknown_header); } - if let Some(justification) = blockchain.justification(current_id)? { + if let Some(justification) = blockchain.justifications(current_id)? { if let Some((_, grandpa_justification)) = justification.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { // check if the current block enacts new GRANDPA authorities set let new_authorities = authorities_provider.authorities(¤t_id)?; diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 5f45439ad9e29..df5a6a747d07a 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -411,7 +411,7 @@ fn finalize_3_voters_no_observers() { // normally there's no justification for finalized blocks assert!( - net.lock().peer(0).client().justification(&BlockId::Number(20)).unwrap().is_none(), + net.lock().peer(0).client().justifications(&BlockId::Number(20)).unwrap().is_none(), "Extra justification for block#1", ); } @@ -653,7 +653,7 @@ fn justification_is_generated_periodically() { // when block#32 (justification_period) is finalized, justification // is required => generated for i in 0..3 { - assert!(net.lock().peer(i).client().justification(&BlockId::Number(32)).unwrap().is_some()); + assert!(net.lock().peer(i).client().justifications(&BlockId::Number(32)).unwrap().is_some()); } } @@ -698,12 +698,12 @@ fn sync_justifications_on_change_blocks() { // the first 3 peers are grandpa voters and therefore have already finalized // block 21 and stored a justification for i in 0..3 { - assert!(net.lock().peer(i).client().justification(&BlockId::Number(21)).unwrap().is_some()); + assert!(net.lock().peer(i).client().justifications(&BlockId::Number(21)).unwrap().is_some()); } // the last peer should get the justification by syncing from other peers futures::executor::block_on(futures::future::poll_fn(move |cx| { - if net.lock().peer(3).client().justification(&BlockId::Number(21)).unwrap().is_none() { + if net.lock().peer(3).client().justifications(&BlockId::Number(21)).unwrap().is_none() { net.lock().poll(cx); Poll::Pending } else { @@ -1631,7 +1631,7 @@ fn imports_justification_for_regular_blocks_on_import() { // the justification should be imported and available from the client assert!( - client.justification(&BlockId::Hash(block_hash)).unwrap().is_some(), + client.justifications(&BlockId::Hash(block_hash)).unwrap().is_some(), ); } diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs index 43d4ae55ed661..7cf46beeea84a 100644 --- a/client/light/src/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -109,7 +109,7 @@ impl BlockchainBackend for Blockchain where Block: BlockT, S Err(ClientError::NotAvailableOnLightClient) } - fn justification(&self, _id: BlockId) -> ClientResult> { + fn justifications(&self, _id: BlockId) -> ClientResult> { Err(ClientError::NotAvailableOnLightClient) } diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 2510ce6a2cfd4..6aeb905cd1782 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -123,7 +123,7 @@ impl BlockRequestHandler { let hash = header.hash(); let parent_hash = *header.parent_hash(); let justifications = if get_justification { - self.client.justification(&BlockId::Hash(hash))? + self.client.justifications(&BlockId::Hash(hash))? } else { None }; diff --git a/client/network/test/src/block_import.rs b/client/network/test/src/block_import.rs index e90ca978fd268..200c7357c4244 100644 --- a/client/network/test/src/block_import.rs +++ b/client/network/test/src/block_import.rs @@ -35,7 +35,7 @@ fn prepare_good_block() -> (TestClient, Hash, u64, PeerId, IncomingBlock) let (hash, number) = (client.block_hash(1).unwrap().unwrap(), 1); let header = client.header(&BlockId::Number(1)).unwrap(); - let justifications = client.justification(&BlockId::Number(1)).unwrap(); + let justifications = client.justifications(&BlockId::Number(1)).unwrap(); let peer_id = PeerId::random(); (client, hash, number, peer_id.clone(), IncomingBlock { hash, diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 4ef4be17766f4..05f476f8c90ba 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -180,10 +180,10 @@ impl PeersClient { } } - pub fn justification(&self, block: &BlockId) -> ClientResult> { + pub fn justifications(&self, block: &BlockId) -> ClientResult> { match *self { - PeersClient::Full(ref client, ref _backend) => client.justification(block), - PeersClient::Light(ref client, ref _backend) => client.justification(block), + PeersClient::Full(ref client, ref _backend) => client.justifications(block), + PeersClient::Light(ref client, ref _backend) => client.justifications(block), } } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 6b1f0e207adb3..40d4d0682b336 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -251,8 +251,8 @@ fn sync_justifications() { net.block_until_sync(); // there's currently no justification for block #10 - assert_eq!(net.peer(0).client().justification(&BlockId::Number(10)).unwrap(), None); - assert_eq!(net.peer(1).client().justification(&BlockId::Number(10)).unwrap(), None); + assert_eq!(net.peer(0).client().justifications(&BlockId::Number(10)).unwrap(), None); + assert_eq!(net.peer(1).client().justifications(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification let just = (ID, Vec::new()); @@ -273,10 +273,10 @@ fn sync_justifications() { net.poll(cx); for height in (10..21).step_by(5) { - if net.peer(0).client().justification(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { + if net.peer(0).client().justifications(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { return Poll::Pending; } - if net.peer(1).client().justification(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { + if net.peer(1).client().justifications(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { return Poll::Pending; } } @@ -308,8 +308,8 @@ fn sync_justifications_across_forks() { block_on(futures::future::poll_fn::<(), _>(|cx| { net.poll(cx); - if net.peer(0).client().justification(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) && - net.peer(1).client().justification(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) + if net.peer(0).client().justifications(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) && + net.peer(1).client().justifications(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) { Poll::Ready(()) } else { diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 05a574b9e9ac4..bd39ab31c2d2e 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1903,7 +1903,7 @@ impl BlockBackend for Client } fn block(&self, id: &BlockId) -> sp_blockchain::Result>> { - Ok(match (self.header(id)?, self.body(id)?, self.justification(id)?) { + Ok(match (self.header(id)?, self.body(id)?, self.justifications(id)?) { (Some(header), Some(extrinsics), justification) => Some(SignedBlock { block: Block::new(header, extrinsics), justification }), _ => None, @@ -1914,8 +1914,8 @@ impl BlockBackend for Client Client::block_status(self, id) } - fn justification(&self, id: &BlockId) -> sp_blockchain::Result> { - self.backend.blockchain().justification(*id) + fn justifications(&self, id: &BlockId) -> sp_blockchain::Result> { + self.backend.blockchain().justifications(*id) } fn block_hash(&self, number: NumberFor) -> sp_blockchain::Result> { diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 717f9edcfd880..00a11b56e6aea 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1036,17 +1036,17 @@ fn import_with_justification() { ); assert_eq!( - client.justification(&BlockId::Hash(a3.hash())).unwrap(), + client.justifications(&BlockId::Hash(a3.hash())).unwrap(), Some(justification), ); assert_eq!( - client.justification(&BlockId::Hash(a1.hash())).unwrap(), + client.justifications(&BlockId::Hash(a1.hash())).unwrap(), None, ); assert_eq!( - client.justification(&BlockId::Hash(a2.hash())).unwrap(), + client.justifications(&BlockId::Hash(a2.hash())).unwrap(), None, ); } diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index 4cbe0f9d99c36..245242a1e8748 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -84,8 +84,8 @@ pub trait HeaderBackend: Send + Sync { pub trait Backend: HeaderBackend + HeaderMetadata { /// Get block body. Returns `None` if block is not found. fn body(&self, id: BlockId) -> Result::Extrinsic>>>; - /// Get block justification. Returns `None` if justification does not exist. - fn justification(&self, id: BlockId) -> Result>; + /// Get block justifications. Returns `None` if justification does not exist. + fn justifications(&self, id: BlockId) -> Result>; /// Get last finalized block hash. fn last_finalized(&self) -> Result; /// Returns data cache reference, if it is enabled on this backend. From 2e23810de56d535b35f3374d947fb877aca7cd6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 19 Jan 2021 15:49:01 +0100 Subject: [PATCH 19/58] client/db: upgrade existing database for new format --- Cargo.lock | 1 + client/db/Cargo.toml | 1 + client/db/src/upgrade.rs | 51 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a12c2a0e4f55b..70e884b57a502 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6669,6 +6669,7 @@ dependencies = [ "sp-consensus", "sp-core", "sp-database", + "sp-finality-grandpa", "sp-keyring", "sp-runtime", "sp-state-machine", diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 23f6fa9b1f628..ec33ae89ff2f7 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -33,6 +33,7 @@ sc-executor = { version = "0.8.0", path = "../executor" } sc-state-db = { version = "0.8.0", path = "../state-db" } sp-trie = { version = "2.0.0", path = "../../primitives/trie" } sp-consensus = { version = "0.8.0", path = "../../primitives/consensus/common" } +sp-finality-grandpa = { version = "2.0.0", path = "../../primitives/finality-grandpa" } sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } sp-database = { version = "2.0.0", path = "../../primitives/database" } parity-db = { version = "0.1.2", optional = true } diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index b6e49edba1978..1385a71f8a1de 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -23,17 +23,19 @@ use std::io::{Read, Write, ErrorKind}; use std::path::{Path, PathBuf}; use sp_runtime::traits::Block as BlockT; -use crate::utils::DatabaseType; +use crate::{columns, utils::DatabaseType}; use kvdb_rocksdb::{Database, DatabaseConfig}; +use codec::Encode; /// Version file name. const VERSION_FILE_NAME: &'static str = "db_version"; /// Current db version. -const CURRENT_VERSION: u32 = 2; +const CURRENT_VERSION: u32 = 3; /// Number of columns in v1. const V1_NUM_COLUMNS: u32 = 11; +const V2_NUM_COLUMNS: u32 = 12; /// Upgrade database to current version. pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> { @@ -42,7 +44,11 @@ pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> sp_bl let db_version = current_version(db_path)?; match db_version { 0 => Err(sp_blockchain::Error::Backend(format!("Unsupported database version: {}", db_version)))?, - 1 => migrate_1_to_2::(db_path, db_type)?, + 1 => { + migrate_1_to_2::(db_path, db_type)?; + migrate_2_to_3::(db_path, db_type)? + }, + 2 => migrate_2_to_3::(db_path, db_type)?, CURRENT_VERSION => (), _ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?, } @@ -62,6 +68,41 @@ fn migrate_1_to_2(db_path: &Path, _db_type: DatabaseType) -> sp_b db.add_column().map_err(db_err) } +/// Migration from version2 to version3: +/// - The format of the stored Justification changed to support multiple Justifications. +fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { + println!("JON: migrate_jon()"); + + let db_path = db_path.to_str() + .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; + let db_cfg = DatabaseConfig::with_columns(V2_NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).map_err(db_err)?; + + println!("{}", &db_path); + dbg!(&db.num_columns()); + dbg!(&db.num_keys(columns::JUSTIFICATION)); + dbg!(&db.get_statistics()); + + // Get all the keys we need to update + let keys: Vec<_> = db.iter(columns::JUSTIFICATION).map(|entry| entry.0).collect(); + + // Read and update each entry + let mut transaction = db.transaction(); + for key in keys { + if let Some(justification) = db.get(columns::JUSTIFICATION, &key).map_err(db_err)? { + let justifications = sp_runtime::Justifications(vec![( + sp_finality_grandpa::GRANDPA_ENGINE_ID, + justification, + )]); + + transaction.put_vec(columns::JUSTIFICATION, &key, justifications.encode()); + } + } + db.write(transaction).map_err(db_err)?; + + Ok(()) +} + /// Reads current database version from the file at given path. /// If the file does not exist returns 0. fn current_version(path: &Path) -> sp_blockchain::Result { @@ -141,8 +182,8 @@ mod tests { } #[test] - fn upgrade_from_1_to_2_works() { - for version_from_file in &[None, Some(1)] { + fn upgrade_to_3_works() { + for version_from_file in &[None, Some(1), Some(2)] { let db_dir = tempfile::TempDir::new().unwrap(); let db_path = db_dir.path(); create_db(db_path, *version_from_file); From 0b69a39f29eae75ca7565d43795bf70abb5b2e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 29 Jan 2021 11:19:01 +0100 Subject: [PATCH 20/58] network: remove dependency on grandpa crate --- Cargo.lock | 1 - client/network/Cargo.toml | 1 - client/network/src/block_request_handler.rs | 2 +- client/network/src/lib.rs | 7 ++++++- client/network/src/protocol.rs | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69bc7078737fd..ff6cacfcaad4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7141,7 +7141,6 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-core", - "sp-finality-grandpa", "sp-keyring", "sp-runtime", "sp-test-primitives", diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index a3a2393f11941..bf948ff4dd37d 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -52,7 +52,6 @@ smallvec = "1.5.0" sp-arithmetic = { version = "2.0.0", path = "../../primitives/arithmetic" } sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } sp-consensus = { version = "0.8.0", path = "../../primitives/consensus/common" } -sp-finality-grandpa = { version = "2.0.0", path = "../../primitives/finality-grandpa" } sp-core = { version = "2.0.0", path = "../../primitives/core" } sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } sp-utils = { version = "2.0.0", path = "../../primitives/utils" } diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 6aeb905cd1782..e5acfc33c191d 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -24,13 +24,13 @@ use crate::protocol::{message::BlockAttributes}; use crate::request_responses::{IncomingRequest, ProtocolConfig}; use crate::schema::v1::block_request::FromBlock; use crate::schema::v1::{BlockResponse, Direction}; +use crate::GRANDPA_ENGINE_ID; use futures::channel::{mpsc, oneshot}; use futures::stream::StreamExt; use log::debug; use prost::Message; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, Header, One, Zero}; -use sp_finality_grandpa::GRANDPA_ENGINE_ID; use std::cmp::min; use std::sync::{Arc}; use std::time::Duration; diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index ab7625ff9fe8a..cb343e66114dd 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -273,7 +273,7 @@ pub use service::{ }; pub use sc_peerset::ReputationChange; -use sp_runtime::traits::{Block as BlockT, NumberFor}; +use sp_runtime::{ConsensusEngineId, traits::{Block as BlockT, NumberFor}}; /// The maximum allowed number of established connections per peer. /// @@ -287,6 +287,11 @@ const MAX_CONNECTIONS_PER_PEER: usize = 2; /// The maximum number of concurrent established connections that were incoming. const MAX_CONNECTIONS_ESTABLISHED_INCOMING: u32 = 10_000; +// Redefine this locally to avoid depending on the grandpa crate. +// NOTE: This is purely during a backwards compatible transitionary period and should be removed +// once we can assume all nodes can send and receive multiple Justifications +const GRANDPA_ENGINE_ID: ConsensusEngineId = *b"FRNK"; + /// Minimum Requirements for a Hash within Networking pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {} diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 31b1d37ea00e8..a256511f60213 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -20,7 +20,7 @@ use crate::{ ExHashT, chain::Client, config::{self, ProtocolId, TransactionPool, TransactionImportFuture, TransactionImport}, - error, + error, GRANDPA_ENGINE_ID, request_responses::RequestFailure, utils::{interval, LruHashSet}, }; @@ -760,7 +760,7 @@ impl Protocol { // For compatibility we assume that the incoming Justifications is an // `EncodedJustification`, as before, and that it can only be for GRANDPA. let justification: EncodedJustification = block_data.justification; - let justifications = vec![(sp_finality_grandpa::GRANDPA_ENGINE_ID, justification)]; + let justifications = vec![(GRANDPA_ENGINE_ID, justification)]; Some(Justifications(justifications)) } else if block_data.is_empty_justification { Some(Justifications(Vec::new())) From b91f82157ada662a8e8f8d9a1f108bc1e5b6b1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 29 Jan 2021 14:11:46 +0100 Subject: [PATCH 21/58] db: fix check for finalized block --- client/api/src/in_mem.rs | 1 - client/db/src/lib.rs | 13 ++++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 7953d90162438..3ebe668819b46 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -317,7 +317,6 @@ impl Blockchain { .find(|stored| stored.0 == justification.0) .is_some() { - // WIP(JON): Maybe create new error type for this return Err(sp_blockchain::Error::BadJustification("Duplicate".into())); } stored_justifications.0.push(justification); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 323975d9aff0f..1cdc58b4d5d12 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -57,6 +57,7 @@ use sc_client_api::{ UsageInfo, MemoryInfo, IoInfo, MemorySize, backend::{NewBlockState, PrunableStateChangesTrieStorage, ProvideChtRoots}, leaves::{LeafSet, FinalizationDisplaced}, cht, + utils::is_descendent_of, }; use sp_blockchain::{ Result as ClientResult, Error as ClientError, @@ -1407,7 +1408,7 @@ impl Backend { self.storage.db.commit(transaction)?; - // Apply all in-memory state shanges. + // Apply all in-memory state changes. // Code beyond this point can't fail. if let Some(( @@ -1677,9 +1678,12 @@ impl sc_client_api::backend::Backend for Backend { let number = *header.number(); // Check if block is finalized first - // WIP(JON): is this really the correct way to check if a block is final? - if number > self.blockchain.info().finalized_number { - return Err(ClientError::BadJustification("WIP: block not finalized".into())); + let is_descendent_of = is_descendent_of(&self.blockchain, None); + let last_finalized = self.blockchain.last_finalized()?; + if !is_descendent_of(&hash, &last_finalized)? { + return Err(ClientError::BadJustification( + "Can't append Justification to unfinalized block".into(), + )); } // Read and merge Justification @@ -1692,7 +1696,6 @@ impl sc_client_api::backend::Backend for Backend { .find(|stored| stored.0 == justification.0) .is_some() { - // WIP(JON): Maybe create new error type for this return Err(ClientError::BadJustification("Duplicate".into())); } stored_justifications.0.push(justification); From 9301c86e838ff1e3b6c402f798ff39b1258d075f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 29 Jan 2021 14:29:23 +0100 Subject: [PATCH 22/58] grandpa: check for multiple grandpa justifications hwne importing --- client/finality-grandpa/src/import.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 07d823bb5764a..acf8d5b5c636a 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -619,14 +619,19 @@ where enacts_change: bool, initial_sync: bool, ) -> Result<(), ConsensusError> { + if justification.0.iter().filter(|j| j.0 == GRANDPA_ENGINE_ID).count() > 1 { + return Err(ConsensusError::ClientImport( + "Received multiple GRANDPA Justifications for the same block.".into(), + )); + } + let grandpa_justification = - // WIP(JON): insert check for >1 grandpa justifications? match justification.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { Some((_, grandpa_justification)) => grandpa_justification, None => { return Err(ConsensusError::ClientImport( - "WIP(JON): expected a grandpa justification.".into(), - )) + "GRANDPA can only import GRANDPA Justifications.".into(), + )); } }; From 63291b56a04edeb284c7f64282422688cb0c9d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 1 Feb 2021 08:56:56 +0100 Subject: [PATCH 23/58] backend: update Finalizer trait to take multiple Justifications --- client/api/src/backend.rs | 4 ++-- .../manual-seal/src/finalize_block.rs | 6 ++++-- client/finality-grandpa/src/environment.rs | 7 ++++--- client/network/src/protocol/sync.rs | 2 +- client/network/test/src/lib.rs | 14 ++++++-------- client/network/test/src/sync.rs | 6 +++--- client/service/src/client/client.rs | 19 +++++++++---------- test-utils/client/src/client_ext.rs | 8 ++++---- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 1501e6f75b938..3a6de6d2cb0be 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -227,7 +227,7 @@ pub trait Finalizer> { &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justifications: Option, notify: bool, ) -> sp_blockchain::Result<()>; @@ -247,7 +247,7 @@ pub trait Finalizer> { fn finalize_block( &self, id: BlockId, - justification: Option, + justifications: Option, notify: bool, ) -> sp_blockchain::Result<()>; } diff --git a/client/consensus/manual-seal/src/finalize_block.rs b/client/consensus/manual-seal/src/finalize_block.rs index 76ae6eeeae5ac..9d9547e702a32 100644 --- a/client/consensus/manual-seal/src/finalize_block.rs +++ b/client/consensus/manual-seal/src/finalize_block.rs @@ -20,7 +20,7 @@ use crate::rpc; use sp_runtime::{ - Justification, + Justification, Justifications, traits::Block as BlockT, generic::BlockId, }; @@ -58,7 +58,9 @@ pub async fn finalize_block(params: FinalizeBlockParams) .. } = params; - match finalizer.finalize_block(BlockId::Hash(hash), justification, true) { + let justifications = justification.map(Justifications::from); + + match finalizer.finalize_block(BlockId::Hash(hash), justifications, true) { Err(e) => { log::warn!("Failed to finalize block {:?}", e); rpc::send_result(&mut sender, Err(e.into())) diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index c1a3a11521fa7..5c383e0b2d50e 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -35,7 +35,7 @@ use finality_grandpa::{ voter, voter_set::VoterSet, }; use sp_blockchain::HeaderMetadata; -use sp_runtime::generic::BlockId; +use sp_runtime::{Justifications, generic::BlockId}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, }; @@ -1289,9 +1289,10 @@ where // ideally some handle to a synchronization oracle would be used // to avoid unconditionally notifying. - let justification = justification.map(|j| (GRANDPA_ENGINE_ID, j.clone())); + let justifications = justification + .map(|j| Justifications::from((GRANDPA_ENGINE_ID, j.clone()))); client - .apply_finality(import_op, BlockId::Hash(hash), justification, true) + .apply_finality(import_op, BlockId::Hash(hash), justifications, true) .map_err(|e| { warn!(target: "afg", "Error applying finality to block {:?}: {:?}", (hash, number), e); e diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index bca172e5d333f..6d6adc93b47fe 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -2365,7 +2365,7 @@ mod test { ); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); - let just = (ID, Vec::new()); + let just = Justifications::from((ID, Vec::new())); client.finalize_block(BlockId::Hash(finalized_block.hash()), Some(just)).unwrap(); sync.update_chain_info(&info.best_hash, info.best_number); diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index a870b1aefc92e..11744402cf343 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -62,7 +62,7 @@ use sp_core::H256; use sc_network::config::ProtocolConfig; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; -use sp_runtime::{Justification, Justifications}; +use sp_runtime::Justifications; use substrate_test_runtime_client::{self, AccountKeyring}; use sc_service::client::Client; pub use sc_network::config::EmptyTransactionPool; @@ -207,12 +207,12 @@ impl PeersClient { pub fn finalize_block( &self, id: BlockId, - justification: Option, + justifications: Option, notify: bool ) -> ClientResult<()> { match *self { - PeersClient::Full(ref client, ref _backend) => client.finalize_block(id, justification, notify), - PeersClient::Light(ref client, ref _backend) => client.finalize_block(id, justification, notify), + PeersClient::Full(ref client, ref _backend) => client.finalize_block(id, justifications, notify), + PeersClient::Light(ref client, ref _backend) => client.finalize_block(id, justifications, notify), } } } @@ -1021,11 +1021,9 @@ impl JustificationImport for ForceFinalized { &mut self, hash: H256, _number: NumberFor, - justification: Justifications, + justifications: Justifications, ) -> Result<(), Self::Error> { - // WIP(JON): Just finalize with the first Justification. TODO is to append the other ones as well - let justification = justification.0.first().cloned(); - self.0.finalize_block(BlockId::Hash(hash), justification, true) + self.0.finalize_block(BlockId::Hash(hash), Some(justifications), true) .map_err(|_| ConsensusError::InvalidJustification.into()) } } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index aca03d1109993..62c23cd5e249a 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -255,7 +255,7 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justifications(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - let just = (ID, Vec::new()); + let just = Justifications::from((ID, Vec::new())); net.peer(0).client().finalize_block(BlockId::Number(10), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(15), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(20), Some(just.clone()), true).unwrap(); @@ -299,7 +299,7 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - let just = (ID, Vec::new()); + let just = Justifications::from((ID, Vec::new())); net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(just), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); @@ -701,7 +701,7 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - let just = Some((ID, Vec::new())); + let just = Some(Justifications::from((ID, Vec::new()))); net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index bd39ab31c2d2e..2774d234b4e1d 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -37,7 +37,7 @@ use sp_core::{ use sp_keystore::SyncCryptoStorePtr; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_runtime::{ - Justification, Justifications, BuildStorage, + Justifications, BuildStorage, generic::{BlockId, SignedBlock, DigestItem}, traits::{ Block as BlockT, Header as HeaderT, Zero, NumberFor, @@ -1808,16 +1808,15 @@ impl Finalizer for Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justifications: Option, notify: bool, ) -> sp_blockchain::Result<()> { let last_best = self.backend.blockchain().info().best_hash; let to_finalize_hash = self.backend.blockchain().expect_block_hash_from_id(&id)?; - let justification = justification.map(Justifications::from); self.apply_finality_with_block_hash( operation, to_finalize_hash, - justification, + justifications, last_best, notify, ) @@ -1826,11 +1825,11 @@ impl Finalizer for Client where fn finalize_block( &self, id: BlockId, - justification: Option, + justifications: Option, notify: bool, ) -> sp_blockchain::Result<()> { self.lock_import_and_run(|operation| { - self.apply_finality(operation, id, justification, notify) + self.apply_finality(operation, id, justifications, notify) }) } } @@ -1845,19 +1844,19 @@ impl Finalizer for &Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justification: Option, + justifications: Option, notify: bool, ) -> sp_blockchain::Result<()> { - (**self).apply_finality(operation, id, justification, notify) + (**self).apply_finality(operation, id, justifications, notify) } fn finalize_block( &self, id: BlockId, - justification: Option, + justifications: Option, notify: bool, ) -> sp_blockchain::Result<()> { - (**self).finalize_block(id, justification, notify) + (**self).finalize_block(id, justifications, notify) } } diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index 9ceeae77addf4..fc4b2d23b0d7a 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -24,7 +24,7 @@ use sp_consensus::{ BlockImportParams, BlockImport, BlockOrigin, Error as ConsensusError, ForkChoiceStrategy, }; -use sp_runtime::{Justification, Justifications}; +use sp_runtime::Justifications; use sp_runtime::traits::{Block as BlockT}; use sp_runtime::generic::BlockId; use codec::alloc::collections::hash_map::HashMap; @@ -35,7 +35,7 @@ pub trait ClientExt: Sized { fn finalize_block( &self, id: BlockId, - justification: Option, + justifications: Option, ) -> sp_blockchain::Result<()>; /// Returns hash of the genesis block. @@ -73,9 +73,9 @@ impl ClientExt for Client fn finalize_block( &self, id: BlockId, - justification: Option, + justifications: Option, ) -> sp_blockchain::Result<()> { - Finalizer::finalize_block(self, id, justification, true) + Finalizer::finalize_block(self, id, justifications, true) } fn genesis_hash(&self) -> ::Hash { From 206447c566dd36d11790d1585ceb608af6c7e7e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 1 Feb 2021 09:42:38 +0100 Subject: [PATCH 24/58] db: remove debugging statements in migration code --- client/db/src/upgrade.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 1385a71f8a1de..6d49ee34a2507 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -71,18 +71,11 @@ fn migrate_1_to_2(db_path: &Path, _db_type: DatabaseType) -> sp_b /// Migration from version2 to version3: /// - The format of the stored Justification changed to support multiple Justifications. fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { - println!("JON: migrate_jon()"); - let db_path = db_path.to_str() .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; let db_cfg = DatabaseConfig::with_columns(V2_NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).map_err(db_err)?; - println!("{}", &db_path); - dbg!(&db.num_columns()); - dbg!(&db.num_keys(columns::JUSTIFICATION)); - dbg!(&db.get_statistics()); - // Get all the keys we need to update let keys: Vec<_> = db.iter(columns::JUSTIFICATION).map(|entry| entry.0).collect(); From 4185aa54c960e34e22b6d10fd3e9f50bbc1207e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 1 Feb 2021 09:44:32 +0100 Subject: [PATCH 25/58] manual-seal: update note about engine id --- client/consensus/manual-seal/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index ba14cac54c632..84e5831b7aeac 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -50,7 +50,8 @@ pub use self::{ use sp_api::{ProvideRuntimeApi, TransactionFor}; /// The `ConsensusEngineId` of Manual Seal. -// WIP(JON): consider creating a new crate primitives/manual-seal for this +// We should consider creating a new crate primitives/manual-seal for this, if it ends up being used +// outside of this crate. pub const MANUAL_SEAL_ENGINE_ID: ConsensusEngineId = [b'm', b'a', b's', b'e']; /// The verifier for the manual seal engine; instantly finalizes. From 5cae4b182a72eff4ea2e49291ee015afa98918da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 1 Feb 2021 14:38:26 +0100 Subject: [PATCH 26/58] db: fix check for finalized block --- client/db/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 1a6d82ee7db92..70f7121feafa6 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1678,7 +1678,7 @@ impl sc_client_api::backend::Backend for Backend { // Check if block is finalized first let is_descendent_of = is_descendent_of(&self.blockchain, None); let last_finalized = self.blockchain.last_finalized()?; - if !is_descendent_of(&hash, &last_finalized)? { + if hash != last_finalized && !is_descendent_of(&hash, &last_finalized)? { return Err(ClientError::BadJustification( "Can't append Justification to unfinalized block".into(), )); From 1d7e50bc97350430abe92abc5f430cd803bc7651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 10 Feb 2021 14:28:12 +0100 Subject: [PATCH 27/58] client: update variable name to reflect it is now plural --- client/api/src/backend.rs | 6 ++--- client/api/src/in_mem.rs | 24 +++++++++---------- client/consensus/aura/src/lib.rs | 4 ++-- client/consensus/babe/src/lib.rs | 6 ++--- client/consensus/babe/src/tests.rs | 4 ++-- client/consensus/manual-seal/src/lib.rs | 4 ++-- client/consensus/pow/src/lib.rs | 4 ++-- client/db/src/lib.rs | 22 ++++++++--------- client/finality-grandpa/src/import.rs | 20 ++++++++-------- client/finality-grandpa/src/tests.rs | 4 ++-- client/network/src/gossip/tests.rs | 4 ++-- client/network/src/protocol.rs | 4 ++-- client/network/src/protocol/sync.rs | 4 ++-- client/network/src/service/tests.rs | 4 ++-- client/network/test/src/lib.rs | 8 +++---- client/rpc/src/chain/chain_light.rs | 2 +- client/rpc/src/chain/tests.rs | 2 +- client/service/src/chain_ops/import_blocks.rs | 2 +- client/service/src/client/client.rs | 18 +++++++------- .../consensus/common/src/block_import.rs | 8 +++---- .../consensus/common/src/import_queue.rs | 4 ++-- .../common/src/import_queue/basic_queue.rs | 8 +++---- primitives/runtime/src/generic/block.rs | 2 +- test-utils/client/src/client_ext.rs | 10 ++++---- 24 files changed, 89 insertions(+), 89 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index c240edd645e4a..21c92fec35ecc 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -148,7 +148,7 @@ pub trait BlockImportOperation { &mut self, header: Block::Header, body: Option>, - justification: Option, + justifications: Option, state: NewBlockState, ) -> sp_blockchain::Result<()>; @@ -195,7 +195,7 @@ pub trait BlockImportOperation { fn mark_finalized( &mut self, id: BlockId, - justification: Option, + justifications: Option, ) -> sp_blockchain::Result<()>; /// Mark a block as new head. If both block import and set head are specified, set head @@ -428,7 +428,7 @@ pub trait Backend: AuxStore + Send + Sync { fn finalize_block( &self, block: BlockId, - justification: Option, + justifications: Option, ) -> sp_blockchain::Result<()>; /// Append justification to the block with the given Id. diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 3ebe668819b46..a2c6fd1d93ae9 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -164,7 +164,7 @@ impl Blockchain { &self, hash: Block::Hash, header: ::Header, - justification: Option, + justifications: Option, body: Option::Extrinsic>>, new_state: NewBlockState, ) -> sp_blockchain::Result<()> { @@ -176,7 +176,7 @@ impl Blockchain { { let mut storage = self.storage.write(); storage.leaves.import(hash.clone(), number.clone(), header.parent_hash().clone()); - storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justification)); + storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justifications)); if let NewBlockState::Final = new_state { storage.finalized_hash = hash; @@ -272,7 +272,7 @@ impl Blockchain { Ok(()) } - fn finalize_header(&self, id: BlockId, justification: Option) -> sp_blockchain::Result<()> { + fn finalize_header(&self, id: BlockId, justifications: Option) -> sp_blockchain::Result<()> { let hash = match self.header(id)? { Some(h) => h.hash(), None => return Err(sp_blockchain::Error::UnknownBlock(format!("{}", id))), @@ -281,15 +281,15 @@ impl Blockchain { let mut storage = self.storage.write(); storage.finalized_hash = hash; - if justification.is_some() { + if justifications.is_some() { let block = storage.blocks.get_mut(&hash) .expect("hash was fetched from a block in the db; qed"); - let block_justification = match block { + let block_justifications = match block { StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j }; - *block_justification = justification; + *block_justifications = justifications; } Ok(()) @@ -534,12 +534,12 @@ impl backend::BlockImportOperation for BlockImportOperatio &mut self, header: ::Header, body: Option::Extrinsic>>, - justification: Option, + justifications: Option, state: NewBlockState, ) -> sp_blockchain::Result<()> { assert!(self.pending_block.is_none(), "Only one block per operation is allowed"); self.pending_block = Some(PendingBlock { - block: StoredBlock::new(header, body, justification), + block: StoredBlock::new(header, body, justifications), state, }); Ok(()) @@ -602,9 +602,9 @@ impl backend::BlockImportOperation for BlockImportOperatio fn mark_finalized( &mut self, block: BlockId, - justification: Option, + justifications: Option, ) -> sp_blockchain::Result<()> { - self.finalized_blocks.push((block, justification)); + self.finalized_blocks.push((block, justifications)); Ok(()) } @@ -720,9 +720,9 @@ impl backend::Backend for Backend where Block::Hash fn finalize_block( &self, block: BlockId, - justification: Option, + justifications: Option, ) -> sp_blockchain::Result<()> { - self.blockchain.finalize_header(block, justification) + self.blockchain.finalize_header(block, justifications) } fn append_justification( diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index c9f29677592a0..9b2b01b4e6ed8 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -589,7 +589,7 @@ impl Verifier for AuraVerifier where &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, mut body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { let mut inherent_data = self.inherent_data_providers @@ -662,7 +662,7 @@ impl Verifier for AuraVerifier where let mut import_block = BlockImportParams::new(origin, pre_header); import_block.post_digests.push(seal); import_block.body = body; - import_block.justification = justification; + import_block.justifications = justifications; import_block.fork_choice = Some(ForkChoiceStrategy::LongestChain); import_block.post_hash = Some(hash); diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index aca3b3b7a6729..b9ed31b0f9690 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -979,7 +979,7 @@ where &mut self, origin: BlockOrigin, header: Block::Header, - justification: Option, + justifications: Option, mut body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { trace!( @@ -987,7 +987,7 @@ where "Verifying origin: {:?} header: {:?} justification: {:?} body: {:?}", origin, header, - justification, + justifications, body, ); @@ -1075,7 +1075,7 @@ where let mut import_block = BlockImportParams::new(origin, pre_header); import_block.post_digests.push(verified_info.seal); import_block.body = body; - import_block.justification = justification; + import_block.justifications = justifications; import_block.intermediates.insert( Cow::from(INTERMEDIATE_KEY), Box::new(BabeIntermediate:: { epoch_descriptor }) as Box, diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 8879c447ff95d..1a0b93e5d114c 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -235,12 +235,12 @@ impl Verifier for TestVerifier { &mut self, origin: BlockOrigin, mut header: TestHeader, - justification: Option, + justifications: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { // apply post-sealing mutations (i.e. stripping seal, if desired). (self.mutator)(&mut header, Stage::PostSeal); - self.inner.verify(origin, header, justification, body) + self.inner.verify(origin, header, justifications, body) } } diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 84e5831b7aeac..5838ec7e60f87 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -62,11 +62,11 @@ impl Verifier for ManualSealVerifier { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { let mut import_params = BlockImportParams::new(origin, header); - import_params.justification = justification; + import_params.justifications = justifications; import_params.body = body; import_params.finalized = false; import_params.fork_choice = Some(ForkChoiceStrategy::LongestChain); diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index 26fd2c92d2a7e..80f2128e695ec 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -458,7 +458,7 @@ impl Verifier for PowVerifier where &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { let hash = header.hash(); @@ -471,7 +471,7 @@ impl Verifier for PowVerifier where let mut import_block = BlockImportParams::new(origin, checked_header); import_block.post_digests.push(seal); import_block.body = body; - import_block.justification = justification; + import_block.justifications = justifications; import_block.intermediates.insert( Cow::from(INTERMEDIATE_KEY), Box::new(intermediate) as Box diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 70f7121feafa6..3758655a10662 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -364,7 +364,7 @@ pub(crate) mod columns { struct PendingBlock { header: Block::Header, - justification: Option, + justifications: Option, body: Option>, leaf_state: NewBlockState, } @@ -713,7 +713,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc &mut self, header: Block::Header, body: Option>, - justification: Option, + justifications: Option, leaf_state: NewBlockState, ) -> ClientResult<()> { assert!(self.pending_block.is_none(), "Only one block per operation is allowed"); @@ -723,7 +723,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc self.pending_block = Some(PendingBlock { header, body, - justification, + justifications, leaf_state, }); Ok(()) @@ -808,9 +808,9 @@ impl sc_client_api::backend::BlockImportOperation for Bloc fn mark_finalized( &mut self, block: BlockId, - justification: Option, + justifications: Option, ) -> ClientResult<()> { - self.finalized_blocks.push((block, justification)); + self.finalized_blocks.push((block, justifications)); Ok(()) } @@ -1108,7 +1108,7 @@ impl Backend { hash: &Block::Hash, header: &Block::Header, last_finalized: Option, - justification: Option, + justifications: Option, changes_trie_cache_ops: &mut Option>, finalization_displaced: &mut Option>>, ) -> ClientResult<(Block::Hash, ::Number, bool, bool)> { @@ -1125,11 +1125,11 @@ impl Backend { finalization_displaced, )?; - if let Some(justification) = justification { + if let Some(justifications) = justifications { transaction.set_from_vec( columns::JUSTIFICATION, &utils::number_and_hash_to_lookup_key(number, hash)?, - justification.encode(), + justifications.encode(), ); } Ok((*hash, number, false, true)) @@ -1238,7 +1238,7 @@ impl Backend { }, } } - if let Some(justification) = pending_block.justification { + if let Some(justification) = pending_block.justifications { transaction.set_from_vec(columns::JUSTIFICATION, &lookup_key, justification.encode()); } @@ -1642,7 +1642,7 @@ impl sc_client_api::backend::Backend for Backend { fn finalize_block( &self, block: BlockId, - justification: Option, + justifications: Option, ) -> ClientResult<()> { let mut transaction = Transaction::new(); let hash = self.blockchain.expect_block_hash_from_id(&block)?; @@ -1655,7 +1655,7 @@ impl sc_client_api::backend::Backend for Backend { &hash, &header, None, - justification, + justifications, &mut changes_trie_cache_ops, &mut displaced, )?; diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 3f2f2fe31cc70..c7a5e263bc383 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -128,14 +128,14 @@ impl JustificationImport &mut self, hash: Block::Hash, number: NumberFor, - justification: Justifications, + justifications: Justifications, ) -> Result<(), Self::Error> { // this justification was requested by the sync service, therefore we // are not sure if it should enact a change or not. it could have been a // request made as part of initial sync but that means the justification // wasn't part of the block and was requested asynchronously, probably // makes sense to log in that case. - GrandpaBlockImport::import_justification(self, hash, number, justification, false, false) + GrandpaBlockImport::import_justification(self, hash, number, justifications, false, false) } } @@ -435,7 +435,7 @@ impl BlockImport let pending_changes = self.make_authorities_changes(&mut block, hash, initial_sync)?; // we don't want to finalize on `inner.import_block` - let mut justification = block.justification.take(); + let mut justifications = block.justifications.take(); let import_result = (&*self.inner).import_block(block, new_cache); let mut imported_aux = { @@ -497,17 +497,17 @@ impl BlockImport // need to apply first, drop any justification that might have been provided with // the block to make sure we request them from `sync` which will ensure they'll be // applied in-order. - justification.take(); + justifications.take(); }, _ => {}, } - match justification { - Some(justification) => { + match justifications { + Some(justifications) => { let import_res = self.import_justification( hash, number, - justification, + justifications, needs_justification, initial_sync, ); @@ -615,18 +615,18 @@ where &mut self, hash: Block::Hash, number: NumberFor, - justification: Justifications, + justifications: Justifications, enacts_change: bool, initial_sync: bool, ) -> Result<(), ConsensusError> { - if justification.0.iter().filter(|j| j.0 == GRANDPA_ENGINE_ID).count() > 1 { + if justifications.0.iter().filter(|j| j.0 == GRANDPA_ENGINE_ID).count() > 1 { return Err(ConsensusError::ClientImport( "Received multiple GRANDPA Justifications for the same block.".into(), )); } let grandpa_justification = - match justification.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + match justifications.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { Some((_, grandpa_justification)) => grandpa_justification, None => { return Err(ConsensusError::ClientImport( diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 0728e5d0919ff..579b13e5ba6c9 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -866,7 +866,7 @@ fn test_bad_justification() { let block = || { let block = block.clone(); let mut import = BlockImportParams::new(BlockOrigin::File, block.header); - import.justification = Some(Justifications(Vec::new())); + import.justifications = Some(Justifications(Vec::new())); import.body = Some(block.extrinsics); import.fork_choice = Some(ForkChoiceStrategy::LongestChain); @@ -1572,7 +1572,7 @@ fn imports_justification_for_regular_blocks_on_import() { // we import the block with justification attached let mut import = BlockImportParams::new(BlockOrigin::File, block.header); - import.justification = Some(Justifications(vec![(GRANDPA_ENGINE_ID, justification.encode())])); + import.justifications = Some(Justifications(vec![(GRANDPA_ENGINE_ID, justification.encode())])); import.body = Some(block.extrinsics); import.fork_choice = Some(ForkChoiceStrategy::LongestChain); diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index 623f6b1f31b4e..78456b2073249 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -51,7 +51,7 @@ fn build_test_full_node(network_config: config::NetworkConfiguration) &mut self, origin: sp_consensus::BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, body: Option>, ) -> Result< ( @@ -78,7 +78,7 @@ fn build_test_full_node(network_config: config::NetworkConfiguration) let mut import = sp_consensus::BlockImportParams::new(origin, header); import.body = body; import.finalized = self.0; - import.justification = justification; + import.justifications = justifications; import.fork_choice = Some(sp_consensus::ForkChoiceStrategy::LongestChain); Ok((import, maybe_keys)) } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 258f819b689db..61c6049578d6a 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -811,8 +811,8 @@ impl Protocol { if request.fields == message::BlockAttributes::JUSTIFICATION { match self.sync.on_block_justification(peer_id, block_response) { Ok(sync::OnBlockJustification::Nothing) => CustomMessageOutcome::None, - Ok(sync::OnBlockJustification::Import { peer, hash, number, justification }) => - CustomMessageOutcome::JustificationImport(peer, hash, number, justification), + Ok(sync::OnBlockJustification::Import { peer, hash, number, justifications }) => + CustomMessageOutcome::JustificationImport(peer, hash, number, justifications), Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC); self.peerset_handle.report_peer(id, repu); diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 8bb5c3248c3ec..dd0ac20689c18 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -425,7 +425,7 @@ pub enum OnBlockJustification { peer: PeerId, hash: B::Hash, number: NumberFor, - justification: Justifications + justifications: Justifications } } @@ -1040,7 +1040,7 @@ impl ChainSync { }; if let Some((peer, hash, number, j)) = self.extra_justifications.on_response(who, justification) { - return Ok(OnBlockJustification::Import { peer, hash, number, justification: j }) + return Ok(OnBlockJustification::Import { peer, hash, number, justifications: j }) } } diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index f6cebd092e5be..d8ceaf55250e4 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -51,7 +51,7 @@ fn build_test_full_node(config: config::NetworkConfiguration) &mut self, origin: sp_consensus::BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, body: Option>, ) -> Result< ( @@ -78,7 +78,7 @@ fn build_test_full_node(config: config::NetworkConfiguration) let mut import = sp_consensus::BlockImportParams::new(origin, header); import.body = body; import.finalized = self.0; - import.justification = justification; + import.justifications = justifications; import.fork_choice = Some(sp_consensus::ForkChoiceStrategy::LongestChain); Ok((import, maybe_keys)) } diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 11744402cf343..2c85f2410fc98 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -108,7 +108,7 @@ impl Verifier for PassThroughVerifier { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, body: Option> ) -> Result<(BlockImportParams, Option)>>), String> { let maybe_keys = header.digest() @@ -119,7 +119,7 @@ impl Verifier for PassThroughVerifier { let mut import = BlockImportParams::new(origin, header); import.body = body; import.finalized = self.finalized; - import.justification = justification; + import.justifications = justifications; import.fork_choice = Some(self.fork_choice.clone()); Ok((import, maybe_keys)) @@ -576,11 +576,11 @@ impl Verifier for VerifierAdapter { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, body: Option> ) -> Result<(BlockImportParams, Option)>>), String> { let hash = header.hash(); - self.verifier.lock().verify(origin, header, justification, body).map_err(|e| { + self.verifier.lock().verify(origin, header, justifications, body).map_err(|e| { self.failed_verifications.lock().insert(hash, e.clone()); e }) diff --git a/client/rpc/src/chain/chain_light.rs b/client/rpc/src/chain/chain_light.rs index 41d4d02e33c9d..a3f3db9b7116c 100644 --- a/client/rpc/src/chain/chain_light.rs +++ b/client/rpc/src/chain/chain_light.rs @@ -106,7 +106,7 @@ impl ChainBackend for LightChain( hash, header: Some(header), body: Some(extrinsics), - justifications: signed_block.justification, + justifications: signed_block.justifications, origin: None, allow_missing_state: false, import_existing: force, diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 656e1b8b29ed5..2ead4c7ecb38d 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -610,7 +610,7 @@ impl Client where let BlockImportParams { origin, header, - justification, + justifications, post_digests, body, storage_changes, @@ -622,7 +622,7 @@ impl Client where .. } = import_block; - assert!(justification.is_some() && finalized || justification.is_none()); + assert!(justifications.is_some() && finalized || justifications.is_none()); if !intermediates.is_empty() { return Err(Error::IncompletePipeline) @@ -650,7 +650,7 @@ impl Client where origin, hash, import_headers, - justification, + justifications, body, storage_changes, new_cache, @@ -686,7 +686,7 @@ impl Client where origin: BlockOrigin, hash: Block::Hash, import_headers: PrePostHeader, - justification: Option, + justifications: Option, body: Option>, storage_changes: Option, Block>>, new_cache: HashMap>, @@ -802,7 +802,7 @@ impl Client where operation.op.set_block_data( import_headers.post().clone(), body, - justification, + justifications, leaf_state, )?; @@ -906,7 +906,7 @@ impl Client where &self, operation: &mut ClientImportOperation, block: Block::Hash, - justification: Option, + justifications: Option, best_block: Block::Hash, notify: bool, ) -> sp_blockchain::Result<()> { @@ -948,7 +948,7 @@ impl Client where } assert_eq!(enacted.last().map(|e| e.hash), Some(block)); - operation.op.mark_finalized(BlockId::Hash(block), justification)?; + operation.op.mark_finalized(BlockId::Hash(block), justifications)?; if notify { // sometimes when syncing, tons of blocks can be finalized at once. @@ -1902,8 +1902,8 @@ impl BlockBackend for Client fn block(&self, id: &BlockId) -> sp_blockchain::Result>> { Ok(match (self.header(id)?, self.body(id)?, self.justifications(id)?) { - (Some(header), Some(extrinsics), justification) => - Some(SignedBlock { block: Block::new(header, extrinsics), justification }), + (Some(header), Some(extrinsics), justifications) => + Some(SignedBlock { block: Block::new(header, extrinsics), justifications }), _ => None, }) } diff --git a/primitives/consensus/common/src/block_import.rs b/primitives/consensus/common/src/block_import.rs index 4c06914d7cea8..c381ebcf1159e 100644 --- a/primitives/consensus/common/src/block_import.rs +++ b/primitives/consensus/common/src/block_import.rs @@ -129,7 +129,7 @@ pub struct BlockImportParams { /// post-runtime digests are pushed back on after. pub header: Block::Header, /// Justification provided for this block from the outside. - pub justification: Option, + pub justifications: Option, /// Digest items that have been added after the runtime for external /// work, like a consensus signature. pub post_digests: Vec>, @@ -174,7 +174,7 @@ impl BlockImportParams { ) -> Self { Self { origin, header, - justification: None, + justifications: None, post_digests: Vec::new(), body: None, storage_changes: None, @@ -214,7 +214,7 @@ impl BlockImportParams { BlockImportParams { origin: self.origin, header: self.header, - justification: self.justification, + justifications: self.justifications, post_digests: self.post_digests, body: self.body, storage_changes: None, @@ -340,6 +340,6 @@ pub trait JustificationImport { &mut self, hash: B::Hash, number: NumberFor, - justification: Justifications, + justifications: Justifications, ) -> Result<(), Self::Error>; } diff --git a/primitives/consensus/common/src/import_queue.rs b/primitives/consensus/common/src/import_queue.rs index b399bd7b8a5e6..9bdc20009b8a0 100644 --- a/primitives/consensus/common/src/import_queue.rs +++ b/primitives/consensus/common/src/import_queue.rs @@ -90,7 +90,7 @@ pub trait Verifier: Send + Sync { &mut self, origin: BlockOrigin, header: B::Header, - justification: Option, + justifications: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String>; } @@ -108,7 +108,7 @@ pub trait ImportQueue: Send { who: Origin, hash: B::Hash, number: NumberFor, - justification: Justifications + justifications: Justifications ); /// Polls for actions to perform on the network. /// diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index 960d9d4f752b4..fed1851967013 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -117,10 +117,10 @@ impl ImportQueue for BasicQueue who: Origin, hash: B::Hash, number: NumberFor, - justification: Justifications, + justifications: Justifications, ) { let res = self.justification_sender.unbounded_send( - worker_messages::ImportJustification(who, hash, number, justification), + worker_messages::ImportJustification(who, hash, number, justifications), ); if res.is_err() { @@ -265,11 +265,11 @@ impl BlockImportWorker { who: Origin, hash: B::Hash, number: NumberFor, - justification: Justifications + justifications: Justifications ) { let started = wasm_timer::Instant::now(); let success = self.justification_import.as_mut().map(|justification_import| { - justification_import.import_justification(hash, number, justification) + justification_import.import_justification(hash, number, justifications) .map_err(|e| { debug!( target: "sync", diff --git a/primitives/runtime/src/generic/block.rs b/primitives/runtime/src/generic/block.rs index 944c71e81e68f..1b30d43ccaca7 100644 --- a/primitives/runtime/src/generic/block.rs +++ b/primitives/runtime/src/generic/block.rs @@ -112,5 +112,5 @@ pub struct SignedBlock { /// Full block. pub block: Block, /// Block justification. - pub justification: Option, + pub justifications: Option, } diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index fc4b2d23b0d7a..aab862de503f1 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -59,7 +59,7 @@ pub trait ClientBlockImportExt: Sized { &mut self, origin: BlockOrigin, block: Block, - justification: Justifications + justifications: Justifications ) -> Result<(), ConsensusError>; } @@ -119,11 +119,11 @@ impl ClientBlockImportExt for std::sync::A &mut self, origin: BlockOrigin, block: Block, - justification: Justifications, + justifications: Justifications, ) -> Result<(), ConsensusError> { let (header, extrinsics) = block.deconstruct(); let mut import = BlockImportParams::new(origin, header); - import.justification = Some(justification); + import.justifications = Some(justifications); import.body = Some(extrinsics); import.finalized = true; import.fork_choice = Some(ForkChoiceStrategy::LongestChain); @@ -168,11 +168,11 @@ impl ClientBlockImportExt for Client Result<(), ConsensusError> { let (header, extrinsics) = block.deconstruct(); let mut import = BlockImportParams::new(origin, header); - import.justification = Some(justification); + import.justifications = Some(justifications); import.body = Some(extrinsics); import.finalized = true; import.fork_choice = Some(ForkChoiceStrategy::LongestChain); From 4d1ed9327e4d5e98521712d1a9f5f53a2d4b8c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 10 Feb 2021 15:39:48 +0100 Subject: [PATCH 28/58] grandpa: fix incorrect empty Justications in test --- client/finality-grandpa/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 579b13e5ba6c9..385828d55724b 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -866,7 +866,7 @@ fn test_bad_justification() { let block = || { let block = block.clone(); let mut import = BlockImportParams::new(BlockOrigin::File, block.header); - import.justifications = Some(Justifications(Vec::new())); + import.justifications = Some(Justifications::from((GRANDPA_ENGINE_ID, Vec::new()))); import.body = Some(block.extrinsics); import.fork_choice = Some(ForkChoiceStrategy::LongestChain); From a0cb5e383307912d415aec381aa7cbed235debc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 11 Feb 2021 10:49:00 +0100 Subject: [PATCH 29/58] primitives: make Justifications opaque to avoid being empty --- client/api/src/in_mem.rs | 22 +++++++--------- client/db/src/changes_tries_storage.rs | 2 +- client/db/src/lib.rs | 14 ++++++---- client/db/src/upgrade.rs | 4 +-- client/finality-grandpa/src/finality_proof.rs | 14 +++++----- client/finality-grandpa/src/import.rs | 4 +-- client/finality-grandpa/src/tests.rs | 2 +- client/network/src/block_request_handler.rs | 2 +- client/network/src/protocol.rs | 6 ++--- client/network/test/src/sync.rs | 26 ++++++++++++++++--- client/service/test/src/client/light.rs | 3 ++- client/service/test/src/client/mod.rs | 4 +-- .../common/src/import_queue/basic_queue.rs | 3 ++- primitives/runtime/src/lib.rs | 24 ++++++++++++++++- 14 files changed, 87 insertions(+), 43 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index a2c6fd1d93ae9..532491e4ef2a9 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -312,14 +312,13 @@ impl Blockchain { if let Some(stored_justifications) = block_justification { if stored_justifications - .0 .iter() .find(|stored| stored.0 == justification.0) .is_some() { return Err(sp_blockchain::Error::BadJustification("Duplicate".into())); } - stored_justifications.0.push(justification); + stored_justifications.push(justification); } else { *block_justification = Some(Justifications::from(justification)); }; @@ -825,10 +824,10 @@ mod tests { fn test_blockchain() -> Blockchain { let blockchain = Blockchain::::new(); - let just0 = Some(Justifications(vec![(ID1, vec![0])])); - let just1 = Some(Justifications(vec![(ID1, vec![1])])); + let just0 = Some(Justifications::from((ID1, vec![0]))); + let just1 = Some(Justifications::from((ID1, vec![1]))); let just2 = None; - let just3 = Some(Justifications(vec![(ID1, vec![3])])); + let just3 = Some(Justifications::from((ID1, vec![3]))); blockchain.insert(header(0).hash(), header(0), just0, None, NewBlockState::Final).unwrap(); blockchain.insert(header(1).hash(), header(1), just1, None, NewBlockState::Final).unwrap(); blockchain.insert(header(2).hash(), header(2), just2, None, NewBlockState::Best).unwrap(); @@ -843,13 +842,12 @@ mod tests { let block = BlockId::Hash(last_finalized); blockchain.append_justification(block, (ID2, vec![4])).unwrap(); - assert_eq!( - blockchain.justifications(block).unwrap(), - Some(Justifications(vec![ - (ID1, vec![3]), - (ID2, vec![4]), - ])), - ); + let justifications = { + let mut just = Justifications::from((ID1, vec![3])); + just.push((ID2, vec![4])); + just + }; + assert_eq!(blockchain.justifications(block).unwrap(), Some(justifications)); } #[test] diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 365c167d64808..a1b43ea5dfc6b 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -957,7 +957,7 @@ mod tests { let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); let config1 = Some(ChangesTrieConfiguration::new(2, 6)); let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - let just1 = Some(Justifications(vec![(CONS0_ENGINE_ID, vec![42])])); + let just1 = Some(Justifications::from((CONS0_ENGINE_ID, vec![42]))); backend.finalize_block(BlockId::Number(1), just1).unwrap(); let config2 = Some(ChangesTrieConfiguration::new(2, 7)); let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 3758655a10662..96ec8c26b04f3 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1689,14 +1689,13 @@ impl sc_client_api::backend::Backend for Backend { let justifications = if let Some(mut stored_justifications) = self.blockchain.justifications(block)? { if stored_justifications - .0 .iter() .find(|stored| stored.0 == justification.0) .is_some() { return Err(ClientError::BadJustification("Duplicate".into())); } - stored_justifications.0.push(justification); + stored_justifications.push(justification); stored_justifications } else { Justifications::from(justification) @@ -2562,7 +2561,7 @@ pub(crate) mod tests { let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); let _ = insert_header(&backend, 1, block0, None, Default::default()); - let justification = Some(Justifications(vec![(CONS0_ENGINE_ID, vec![1, 2, 3])])); + let justification = Some(Justifications::from((CONS0_ENGINE_ID, vec![1, 2, 3]))); backend.finalize_block(BlockId::Number(1), justification.clone()).unwrap(); assert_eq!( @@ -2583,7 +2582,7 @@ pub(crate) mod tests { let just0 = (CONS0_ENGINE_ID, vec![1, 2, 3]); backend.finalize_block( BlockId::Number(1), - Some(Justifications(vec![just0.clone()])), + Some(just0.clone().into()), ).unwrap(); let just1 = (CONS1_ENGINE_ID, vec![4, 5]); @@ -2595,9 +2594,14 @@ pub(crate) mod tests { Err(ClientError::BadJustification(_)) )); + let justifications = { + let mut just = Justifications::from(just0); + just.push(just1); + just + }; assert_eq!( backend.blockchain().justifications(BlockId::Number(1)).unwrap(), - Some(Justifications(vec![just0, just1])), + Some(justifications), ); } diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 6d49ee34a2507..b9d74653b0826 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -83,10 +83,10 @@ fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_b let mut transaction = db.transaction(); for key in keys { if let Some(justification) = db.get(columns::JUSTIFICATION, &key).map_err(db_err)? { - let justifications = sp_runtime::Justifications(vec![( + let justifications = sp_runtime::Justifications::from(( sp_finality_grandpa::GRANDPA_ENGINE_ID, justification, - )]); + )); transaction.put_vec(columns::JUSTIFICATION, &key, justifications.encode()); } diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index a7efe35c84a60..d1e48ec6fd1f2 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -208,7 +208,7 @@ where let last_block_for_set_id = BlockId::Number(last_block_for_set); let justification = if let Some(justifications) = blockchain.justifications(last_block_for_set_id)? { - if let Some((_, grandpa_justification)) = justifications.0 + if let Some((_, grandpa_justification)) = justifications .into_iter() .find(|j| j.0 == GRANDPA_ENGINE_ID) { @@ -321,7 +321,7 @@ pub fn prove_warp_sync>( if result.last().as_ref().map(|head| head.header.number()) != Some(&end_number) { let header = blockchain.expect_header(end)?; if let Some(justifications) = blockchain.justifications(BlockId::Number(end_number.clone()))? { - if let Some((_, justification)) = justifications.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + if let Some((_, justification)) = justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { result.push(AuthoritySetProofFragment { header: header.clone(), justification, @@ -356,7 +356,7 @@ fn get_warp_sync_proof_fragment>( })) = crate::import::find_forced_change::(&header) { let dest = block_number + delay; if let Some(justifications) = blockchain.justifications(BlockId::Number(index.clone()))? { - if let Some((_, justification)) = justifications.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + if let Some((_, justification)) = justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { result = Some((AuthoritySetProofFragment { header: header.clone(), justification, @@ -373,7 +373,7 @@ fn get_warp_sync_proof_fragment>( }) = crate::import::find_scheduled_change::(&header) { let dest = index + delay; if let Some(justifications) = blockchain.justifications(BlockId::Number(index.clone()))? { - if let Some((_, justification)) = justifications.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + if let Some((_, justification)) = justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { result = Some((AuthoritySetProofFragment { header: header.clone(), justification, @@ -672,10 +672,10 @@ pub(crate) mod tests { fn test_blockchain() -> InMemoryBlockchain { use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; let blockchain = InMemoryBlockchain::::new(); - let just0 = Some(Justifications(vec![(ID, vec![0])])); - let just1 = Some(Justifications(vec![(ID, vec![1])])); + let just0 = Some(Justifications::from((ID, vec![0]))); + let just1 = Some(Justifications::from((ID, vec![1]))); let just2 = None; - let just3 = Some(Justifications(vec![(ID, vec![3])])); + let just3 = Some(Justifications::from((ID, vec![3]))); blockchain.insert(header(0).hash(), header(0), just0, None, NewBlockState::Final).unwrap(); blockchain.insert(header(1).hash(), header(1), just1, None, NewBlockState::Final).unwrap(); blockchain.insert(header(2).hash(), header(2), just2, None, NewBlockState::Best).unwrap(); diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index c7a5e263bc383..5a4b04827ce1d 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -619,14 +619,14 @@ where enacts_change: bool, initial_sync: bool, ) -> Result<(), ConsensusError> { - if justifications.0.iter().filter(|j| j.0 == GRANDPA_ENGINE_ID).count() > 1 { + if justifications.iter().filter(|j| j.0 == GRANDPA_ENGINE_ID).count() > 1 { return Err(ConsensusError::ClientImport( "Received multiple GRANDPA Justifications for the same block.".into(), )); } let grandpa_justification = - match justifications.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + match justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { Some((_, grandpa_justification)) => grandpa_justification, None => { return Err(ConsensusError::ClientImport( diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 385828d55724b..66ef4812c31f8 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -1572,7 +1572,7 @@ fn imports_justification_for_regular_blocks_on_import() { // we import the block with justification attached let mut import = BlockImportParams::new(BlockOrigin::File, block.header); - import.justifications = Some(Justifications(vec![(GRANDPA_ENGINE_ID, justification.encode())])); + import.justifications = Some((GRANDPA_ENGINE_ID, justification.encode()).into()); import.body = Some(block.extrinsics); import.fork_choice = Some(ForkChoiceStrategy::LongestChain); diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index dd453f3d4d483..62e220ee3b58c 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -131,7 +131,7 @@ impl BlockRequestHandler { // To keep compatibility we only send the grandpa justification let justification = justifications .map(|just| - match just.0.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + match just.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { Some((_, grandpa_justification)) => grandpa_justification, None => Vec::new(), }) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 61c6049578d6a..06d459aa87b99 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -769,10 +769,10 @@ impl Protocol { // For compatibility we assume that the incoming Justifications is an // `EncodedJustification`, as before, and that it can only be for GRANDPA. let justification: EncodedJustification = block_data.justification; - let justifications = vec![(GRANDPA_ENGINE_ID, justification)]; - Some(Justifications(justifications)) + let justification = (GRANDPA_ENGINE_ID, justification); + Some(Justifications::from(justification)) } else if block_data.is_empty_justification { - Some(Justifications(Vec::new())) + Some(Justifications::from((GRANDPA_ENGINE_ID, Vec::new()))) } else { None }, diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 1fdd815ef1598..5fb738dacb1dd 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -273,10 +273,20 @@ fn sync_justifications() { net.poll(cx); for height in (10..21).step_by(5) { - if net.peer(0).client().justifications(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { + if net + .peer(0) + .client() + .justifications(&BlockId::Number(height)) + .unwrap() != Some(Justifications::from((ID, Vec::new()))) + { return Poll::Pending; } - if net.peer(1).client().justifications(&BlockId::Number(height)).unwrap() != Some(Justifications(Vec::new())) { + if net + .peer(1) + .client() + .justifications(&BlockId::Number(height)) + .unwrap() != Some(Justifications::from((ID, Vec::new()))) + { return Poll::Pending; } } @@ -308,8 +318,16 @@ fn sync_justifications_across_forks() { block_on(futures::future::poll_fn::<(), _>(|cx| { net.poll(cx); - if net.peer(0).client().justifications(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) && - net.peer(1).client().justifications(&BlockId::Number(10)).unwrap() == Some(Justifications(Vec::new())) + if net + .peer(0) + .client() + .justifications(&BlockId::Number(10)) + .unwrap() == Some(Justifications::from((ID, Vec::new()))) + && net + .peer(1) + .client() + .justifications(&BlockId::Number(10)) + .unwrap() == Some(Justifications::from((ID, Vec::new()))) { Poll::Ready(()) } else { diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index bdf8cf8b5817d..a80a953402de5 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -372,13 +372,14 @@ fn execution_proof_is_generated_and_checked() { // prepare remote client let mut remote_client = substrate_test_runtime_client::new(); + const ID: sp_runtime::ConsensusEngineId = *b"TEST"; for i in 1u32..3u32 { let mut digest = Digest::default(); digest.push(sp_runtime::generic::DigestItem::Other::(i.to_le_bytes().to_vec())); remote_client.import_justified( BlockOrigin::Own, remote_client.new_block(digest).unwrap().build().unwrap().block, - Justifications(Default::default()), + Justifications::from((ID, Default::default())), ).unwrap(); } diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index d7576107e32e6..083be8b3a3bf2 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1014,7 +1014,7 @@ fn import_with_justification() { client.import(BlockOrigin::Own, a2.clone()).unwrap(); // A2 -> A3 - let justification = Justifications(vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]); + let justification = Justifications::from((GRANDPA_ENGINE_ID, vec![1, 2, 3])); let a3 = client.new_block_at( &BlockId::Hash(a2.hash()), Default::default(), @@ -1086,7 +1086,7 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { ); // importing B1 as finalized should trigger a re-org and set it as new best - let justification = Justifications(vec![(GRANDPA_ENGINE_ID, vec![1, 2, 3])]); + let justification = Justifications::from((GRANDPA_ENGINE_ID, vec![1, 2, 3])); client.import_justified(BlockOrigin::Own, b1.clone(), justification).unwrap(); assert_eq!( diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index fed1851967013..76c7d79b054da 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -538,12 +538,13 @@ mod tests { let mut import_justification = || { let hash = Hash::random(); + const ENGINE_ID: sp_runtime::ConsensusEngineId = *b"TEST"; block_on(finality_sender.send(worker_messages::ImportJustification( libp2p::PeerId::random(), hash, 1, - sp_runtime::Justifications(Vec::new()), + sp_runtime::Justifications::from((ENGINE_ID, Vec::new())), ))) .unwrap(); diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 5b587921a44c2..4b566c316d10e 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -103,7 +103,29 @@ pub type Justification = (ConsensusEngineId, EncodedJustification); /// Collection of Justifications, since we might have more than one stored per block. #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] -pub struct Justifications(pub Vec); +pub struct Justifications(Vec); + +impl Justifications { + /// WIP(JON) + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + /// WIP(JON) + pub fn push(&mut self, justification: Justification) { + self.0.push(justification) + } +} + +impl IntoIterator for Justifications { + type Item = Justification; + type IntoIter = sp_std::vec::IntoIter; + + /// WIP(JON) + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} impl From for Justifications { fn from(justification: Justification) -> Self { From c32e538ed62641c20e5b252048d1ec3fd39edf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 11 Feb 2021 16:53:55 +0100 Subject: [PATCH 30/58] network: fix detecting empty Justification --- client/network/src/block_request_handler.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 62e220ee3b58c..ec450860399cb 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -130,13 +130,18 @@ impl BlockRequestHandler { // To keep compatibility we only send the grandpa justification let justification = justifications - .map(|just| - match just.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { - Some((_, grandpa_justification)) => grandpa_justification, - None => Vec::new(), - }) - .unwrap_or(Vec::new()); - let is_empty_justification = justification.is_empty(); + .map(|just| { + just.into_iter() + .find(|j| j.0 == GRANDPA_ENGINE_ID) + .map(|(_, grandpa_justification)| grandpa_justification) + }); + + let is_empty_justification = justification + .as_ref() + .map(|j| j.as_ref().map(|j| j.is_empty()).unwrap_or(false)) + .unwrap_or(false); + + let justification = justification.unwrap_or_default().unwrap_or_default(); let body = if get_body { match self.client.block_body(&BlockId::Hash(hash))? { From 2545ed2b311b0288b43bb779f98bb88a45da5dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 12 Feb 2021 09:25:24 +0100 Subject: [PATCH 31/58] runtime: doc strings for Justifications functions --- primitives/runtime/src/lib.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 4b566c316d10e..66a9b0811dcb3 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -106,22 +106,26 @@ pub type Justification = (ConsensusEngineId, EncodedJustification); pub struct Justifications(Vec); impl Justifications { - /// WIP(JON) + /// Return an iterator over the Justifications. pub fn iter(&self) -> impl Iterator { self.0.iter() } - /// WIP(JON) + /// Append a justification. pub fn push(&mut self, justification: Justification) { self.0.push(justification) } + + /// Return the encoded Justification for the given consensus engine, if it exists. + pub fn get(&self, engine_id: &ConsensusEngineId) -> Option<&EncodedJustification> { + self.iter().find(|j| &j.0 == engine_id).map(|j| &j.1) + } } impl IntoIterator for Justifications { type Item = Justification; type IntoIter = sp_std::vec::IntoIter; - /// WIP(JON) fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } From 45f9e206fc84a7ae131f92efcb5d3a5b69f6d0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 12 Feb 2021 10:07:34 +0100 Subject: [PATCH 32/58] runtime: add into_justifications --- client/finality-grandpa/src/finality_proof.rs | 11 ++++------- client/finality-grandpa/src/import.rs | 14 +++++--------- client/network/src/block_request_handler.rs | 6 +----- primitives/runtime/src/lib.rs | 9 +++++++-- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index d1e48ec6fd1f2..7b23af0721bab 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -208,10 +208,7 @@ where let last_block_for_set_id = BlockId::Number(last_block_for_set); let justification = if let Some(justifications) = blockchain.justifications(last_block_for_set_id)? { - if let Some((_, grandpa_justification)) = justifications - .into_iter() - .find(|j| j.0 == GRANDPA_ENGINE_ID) - { + if let Some(grandpa_justification) = justifications.into_justification(GRANDPA_ENGINE_ID) { grandpa_justification } else { trace!( @@ -321,7 +318,7 @@ pub fn prove_warp_sync>( if result.last().as_ref().map(|head| head.header.number()) != Some(&end_number) { let header = blockchain.expect_header(end)?; if let Some(justifications) = blockchain.justifications(BlockId::Number(end_number.clone()))? { - if let Some((_, justification)) = justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + if let Some(justification) = justifications.into_justification(GRANDPA_ENGINE_ID) { result.push(AuthoritySetProofFragment { header: header.clone(), justification, @@ -356,7 +353,7 @@ fn get_warp_sync_proof_fragment>( })) = crate::import::find_forced_change::(&header) { let dest = block_number + delay; if let Some(justifications) = blockchain.justifications(BlockId::Number(index.clone()))? { - if let Some((_, justification)) = justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + if let Some(justification) = justifications.into_justification(GRANDPA_ENGINE_ID) { result = Some((AuthoritySetProofFragment { header: header.clone(), justification, @@ -373,7 +370,7 @@ fn get_warp_sync_proof_fragment>( }) = crate::import::find_scheduled_change::(&header) { let dest = index + delay; if let Some(justifications) = blockchain.justifications(BlockId::Number(index.clone()))? { - if let Some((_, justification)) = justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { + if let Some(justification) = justifications.into_justification(GRANDPA_ENGINE_ID) { result = Some((AuthoritySetProofFragment { header: header.clone(), justification, diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 5a4b04827ce1d..9494df893a9fe 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -625,15 +625,11 @@ where )); } - let grandpa_justification = - match justifications.into_iter().find(|j| j.0 == GRANDPA_ENGINE_ID) { - Some((_, grandpa_justification)) => grandpa_justification, - None => { - return Err(ConsensusError::ClientImport( - "GRANDPA can only import GRANDPA Justifications.".into(), - )); - } - }; + let grandpa_justification = justifications + .into_justification(GRANDPA_ENGINE_ID) + .ok_or(ConsensusError::ClientImport( + "GRANDPA can only import GRANDPA Justifications.".into(), + ))?; let justification = GrandpaJustification::decode_and_verify_finalizes( &grandpa_justification, diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index ec450860399cb..ef2194cdac6ae 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -130,11 +130,7 @@ impl BlockRequestHandler { // To keep compatibility we only send the grandpa justification let justification = justifications - .map(|just| { - just.into_iter() - .find(|j| j.0 == GRANDPA_ENGINE_ID) - .map(|(_, grandpa_justification)| grandpa_justification) - }); + .map(|just| just.into_justification(GRANDPA_ENGINE_ID)); let is_empty_justification = justification .as_ref() diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 66a9b0811dcb3..cc948d13900d6 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -117,8 +117,13 @@ impl Justifications { } /// Return the encoded Justification for the given consensus engine, if it exists. - pub fn get(&self, engine_id: &ConsensusEngineId) -> Option<&EncodedJustification> { - self.iter().find(|j| &j.0 == engine_id).map(|j| &j.1) + pub fn get(&self, engine_id: ConsensusEngineId) -> Option<&EncodedJustification> { + self.iter().find(|j| j.0 == engine_id).map(|j| &j.1) + } + + /// Return the encoded Justification + pub fn into_justification(self, engine_id: ConsensusEngineId) -> Option { + self.into_iter().find(|j| j.0 == engine_id).map(|j| j.1) } } From b65b38a70adc425589db4d6f03997e93f1fec4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 16 Feb 2021 10:07:42 +0100 Subject: [PATCH 33/58] primitives: check for duplicates in when adding to Justifications --- client/api/src/in_mem.rs | 7 +------ client/db/src/lib.rs | 7 +------ primitives/runtime/src/lib.rs | 11 ++++++++--- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index a90dc9c099912..e53ed84fca491 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -311,14 +311,9 @@ impl Blockchain { }; if let Some(stored_justifications) = block_justification { - if stored_justifications - .iter() - .find(|stored| stored.0 == justification.0) - .is_some() - { + if !stored_justifications.push(justification) { return Err(sp_blockchain::Error::BadJustification("Duplicate".into())); } - stored_justifications.push(justification); } else { *block_justification = Some(Justifications::from(justification)); }; diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 76216d73c41a9..28a949439bdb1 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1692,14 +1692,9 @@ impl sc_client_api::backend::Backend for Backend { use sp_blockchain::Backend; let justifications = if let Some(mut stored_justifications) = self.blockchain.justifications(block)? { - if stored_justifications - .iter() - .find(|stored| stored.0 == justification.0) - .is_some() - { + if !stored_justifications.push(justification) { return Err(ClientError::BadJustification("Duplicate".into())); } - stored_justifications.push(justification); stored_justifications } else { Justifications::from(justification) diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index cc948d13900d6..5975abc67a0a1 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -111,9 +111,14 @@ impl Justifications { self.0.iter() } - /// Append a justification. - pub fn push(&mut self, justification: Justification) { - self.0.push(justification) + /// Append a justification. Returns false if a Justification with the same ConsensusEngineId + /// already exists, in which case the Justification is not inserted. + pub fn push(&mut self, justification: Justification) -> bool { + if self.get(justification.0).is_some() { + return false; + } + self.0.push(justification); + true } /// Return the encoded Justification for the given consensus engine, if it exists. From c69833ed3c88868c30018cb76bc5c55f2486f333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 16 Feb 2021 12:21:44 +0100 Subject: [PATCH 34/58] network/test: use real grandpa engine id in test --- client/network/test/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 5fb738dacb1dd..e855fc287858c 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -24,7 +24,7 @@ use sp_consensus::block_validation::Validation; use substrate_test_runtime::Header; use sp_runtime::{ConsensusEngineId, Justifications}; -const ID: ConsensusEngineId = *b"TEST"; +const ID: ConsensusEngineId = *b"FRNK"; fn test_ancestor_search_when_common_is(n: usize) { sp_tracing::try_init_simple(); From 8391a9da06cfc646a909607eb2cd397f46c625f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 16 Feb 2021 13:19:49 +0100 Subject: [PATCH 35/58] client: fix reviewer comments --- client/api/src/in_mem.rs | 4 +++- client/db/src/lib.rs | 4 +++- client/network/src/lib.rs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index e53ed84fca491..83ba9519f2c4a 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -312,7 +312,9 @@ impl Blockchain { if let Some(stored_justifications) = block_justification { if !stored_justifications.push(justification) { - return Err(sp_blockchain::Error::BadJustification("Duplicate".into())); + return Err(sp_blockchain::Error::BadJustification( + "Duplicate consensus engine ID".into() + )); } } else { *block_justification = Some(Justifications::from(justification)); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 28a949439bdb1..d05940b1fbce5 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1693,7 +1693,9 @@ impl sc_client_api::backend::Backend for Backend { let justifications = if let Some(mut stored_justifications) = self.blockchain.justifications(block)? { if !stored_justifications.push(justification) { - return Err(ClientError::BadJustification("Duplicate".into())); + return Err(ClientError::BadJustification( + "Duplicate consensus engine ID".into() + )); } stored_justifications } else { diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 39e3026e589a2..3f0c17e7ba838 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -288,7 +288,7 @@ const MAX_CONNECTIONS_PER_PEER: usize = 2; /// The maximum number of concurrent established connections that were incoming. const MAX_CONNECTIONS_ESTABLISHED_INCOMING: u32 = 10_000; -// Redefine this locally to avoid depending on the grandpa crate. +// Redefine this locally to avoid depending on the GRANDPA crate. // NOTE: This is purely during a backwards compatible transitionary period and should be removed // once we can assume all nodes can send and receive multiple Justifications const GRANDPA_ENGINE_ID: ConsensusEngineId = *b"FRNK"; From c9cc2c82319b7fd74a8078ed8e56e91459598b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 16 Feb 2021 16:17:04 +0100 Subject: [PATCH 36/58] primitives: rename Justifications::push to append --- client/api/src/in_mem.rs | 4 ++-- client/db/src/lib.rs | 4 ++-- primitives/runtime/src/lib.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 83ba9519f2c4a..0385a6621b5bc 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -311,7 +311,7 @@ impl Blockchain { }; if let Some(stored_justifications) = block_justification { - if !stored_justifications.push(justification) { + if !stored_justifications.append(justification) { return Err(sp_blockchain::Error::BadJustification( "Duplicate consensus engine ID".into() )); @@ -848,7 +848,7 @@ mod tests { blockchain.append_justification(block, (ID2, vec![4])).unwrap(); let justifications = { let mut just = Justifications::from((ID1, vec![3])); - just.push((ID2, vec![4])); + just.append((ID2, vec![4])); just }; assert_eq!(blockchain.justifications(block).unwrap(), Some(justifications)); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index d05940b1fbce5..b434aa4db6afa 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1692,7 +1692,7 @@ impl sc_client_api::backend::Backend for Backend { use sp_blockchain::Backend; let justifications = if let Some(mut stored_justifications) = self.blockchain.justifications(block)? { - if !stored_justifications.push(justification) { + if !stored_justifications.append(justification) { return Err(ClientError::BadJustification( "Duplicate consensus engine ID".into() )); @@ -2597,7 +2597,7 @@ pub(crate) mod tests { let justifications = { let mut just = Justifications::from(just0); - just.push(just1); + just.append(just1); just }; assert_eq!( diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 5975abc67a0a1..d078fd6e036df 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -97,7 +97,7 @@ pub use either::Either; /// bypasses this problem. pub type EncodedJustification = Vec; -/// Justification together with an ID to tag it's origin. +/// Justification together with an ID to tag its origin. pub type Justification = (ConsensusEngineId, EncodedJustification); /// Collection of Justifications, since we might have more than one stored per block. @@ -113,7 +113,7 @@ impl Justifications { /// Append a justification. Returns false if a Justification with the same ConsensusEngineId /// already exists, in which case the Justification is not inserted. - pub fn push(&mut self, justification: Justification) -> bool { + pub fn append(&mut self, justification: Justification) -> bool { if self.get(justification.0).is_some() { return false; } From 7bbc1569ec387c2e750a44d35f5d7cdfb4689530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 10:20:55 +0100 Subject: [PATCH 37/58] backend: revert changes to Finalizer trait --- client/api/src/backend.rs | 6 +++--- .../consensus/manual-seal/src/finalize_block.rs | 6 ++---- client/finality-grandpa/src/environment.rs | 7 +++---- client/network/src/protocol/sync.rs | 2 +- client/network/test/src/lib.rs | 12 +++++++----- client/service/src/client/client.rs | 17 +++++++++-------- test-utils/client/src/client_ext.rs | 8 ++++---- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 21c92fec35ecc..e4884626557ae 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use std::collections::{HashMap, HashSet}; use sp_core::ChangesTrieConfigurationRange; -use sp_core::offchain::{OffchainStorage}; +use sp_core::offchain::OffchainStorage; use sp_runtime::{generic::BlockId, Justification, Justifications, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor}; use sp_state_machine::{ @@ -227,7 +227,7 @@ pub trait Finalizer> { &self, operation: &mut ClientImportOperation, id: BlockId, - justifications: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()>; @@ -247,7 +247,7 @@ pub trait Finalizer> { fn finalize_block( &self, id: BlockId, - justifications: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()>; } diff --git a/client/consensus/manual-seal/src/finalize_block.rs b/client/consensus/manual-seal/src/finalize_block.rs index 9d9547e702a32..76ae6eeeae5ac 100644 --- a/client/consensus/manual-seal/src/finalize_block.rs +++ b/client/consensus/manual-seal/src/finalize_block.rs @@ -20,7 +20,7 @@ use crate::rpc; use sp_runtime::{ - Justification, Justifications, + Justification, traits::Block as BlockT, generic::BlockId, }; @@ -58,9 +58,7 @@ pub async fn finalize_block(params: FinalizeBlockParams) .. } = params; - let justifications = justification.map(Justifications::from); - - match finalizer.finalize_block(BlockId::Hash(hash), justifications, true) { + match finalizer.finalize_block(BlockId::Hash(hash), justification, true) { Err(e) => { log::warn!("Failed to finalize block {:?}", e); rpc::send_result(&mut sender, Err(e.into())) diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index 30463f8fcba0d..f2fd8f7fe973e 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -35,7 +35,7 @@ use finality_grandpa::{ voter, voter_set::VoterSet, }; use sp_blockchain::HeaderMetadata; -use sp_runtime::{Justifications, generic::BlockId}; +use sp_runtime::generic::BlockId; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, }; @@ -1316,10 +1316,9 @@ where // ideally some handle to a synchronization oracle would be used // to avoid unconditionally notifying. - let justifications = justification - .map(|j| Justifications::from((GRANDPA_ENGINE_ID, j.clone()))); + let justification = justification.map(|j| (GRANDPA_ENGINE_ID, j.clone())); client - .apply_finality(import_op, BlockId::Hash(hash), justifications, true) + .apply_finality(import_op, BlockId::Hash(hash), justification, true) .map_err(|e| { warn!(target: "afg", "Error applying finality to block {:?}: {:?}", (hash, number), e); e diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index dd0ac20689c18..1bd5992d9c0c9 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -2399,7 +2399,7 @@ mod test { ); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); - let just = Justifications::from((ID, Vec::new())); + let just = (ID, Vec::new()); client.finalize_block(BlockId::Hash(finalized_block.hash()), Some(just)).unwrap(); sync.update_chain_info(&info.best_hash, info.best_number); diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 5c0eb7abaa971..bb45fff3be173 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -63,7 +63,7 @@ use sp_core::H256; use sc_network::config::ProtocolConfig; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; -use sp_runtime::Justifications; +use sp_runtime::{Justification, Justifications}; use substrate_test_runtime_client::{self, AccountKeyring}; use sc_service::client::Client; pub use sc_network::config::EmptyTransactionPool; @@ -208,12 +208,12 @@ impl PeersClient { pub fn finalize_block( &self, id: BlockId, - justifications: Option, + justification: Option, notify: bool ) -> ClientResult<()> { match *self { - PeersClient::Full(ref client, ref _backend) => client.finalize_block(id, justifications, notify), - PeersClient::Light(ref client, ref _backend) => client.finalize_block(id, justifications, notify), + PeersClient::Full(ref client, ref _backend) => client.finalize_block(id, justification, notify), + PeersClient::Light(ref client, ref _backend) => client.finalize_block(id, justification, notify), } } } @@ -1034,7 +1034,9 @@ impl JustificationImport for ForceFinalized { _number: NumberFor, justifications: Justifications, ) -> Result<(), Self::Error> { - self.0.finalize_block(BlockId::Hash(hash), Some(justifications), true) + // WIP(JON) + let justification = justifications.into_iter().next(); + self.0.finalize_block(BlockId::Hash(hash), justification, true) .map_err(|_| ConsensusError::InvalidJustification.into()) } } diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 535fd9c047398..6c96470476ae3 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -37,7 +37,7 @@ use sp_core::{ use sp_keystore::SyncCryptoStorePtr; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_runtime::{ - Justifications, BuildStorage, + Justification, Justifications, BuildStorage, generic::{BlockId, SignedBlock, DigestItem}, traits::{ Block as BlockT, Header as HeaderT, Zero, NumberFor, @@ -1811,11 +1811,12 @@ impl Finalizer for Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justifications: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { let last_best = self.backend.blockchain().info().best_hash; let to_finalize_hash = self.backend.blockchain().expect_block_hash_from_id(&id)?; + let justifications = justification.map(Justifications::from); self.apply_finality_with_block_hash( operation, to_finalize_hash, @@ -1828,11 +1829,11 @@ impl Finalizer for Client where fn finalize_block( &self, id: BlockId, - justifications: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { self.lock_import_and_run(|operation| { - self.apply_finality(operation, id, justifications, notify) + self.apply_finality(operation, id, justification, notify) }) } } @@ -1847,19 +1848,19 @@ impl Finalizer for &Client where &self, operation: &mut ClientImportOperation, id: BlockId, - justifications: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { - (**self).apply_finality(operation, id, justifications, notify) + (**self).apply_finality(operation, id, justification, notify) } fn finalize_block( &self, id: BlockId, - justifications: Option, + justification: Option, notify: bool, ) -> sp_blockchain::Result<()> { - (**self).finalize_block(id, justifications, notify) + (**self).finalize_block(id, justification, notify) } } diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index aab862de503f1..2b9c1d787d264 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -24,7 +24,7 @@ use sp_consensus::{ BlockImportParams, BlockImport, BlockOrigin, Error as ConsensusError, ForkChoiceStrategy, }; -use sp_runtime::Justifications; +use sp_runtime::{Justification, Justifications}; use sp_runtime::traits::{Block as BlockT}; use sp_runtime::generic::BlockId; use codec::alloc::collections::hash_map::HashMap; @@ -35,7 +35,7 @@ pub trait ClientExt: Sized { fn finalize_block( &self, id: BlockId, - justifications: Option, + justification: Option, ) -> sp_blockchain::Result<()>; /// Returns hash of the genesis block. @@ -73,9 +73,9 @@ impl ClientExt for Client fn finalize_block( &self, id: BlockId, - justifications: Option, + justification: Option, ) -> sp_blockchain::Result<()> { - Finalizer::finalize_block(self, id, justifications, true) + Finalizer::finalize_block(self, id, justification, true) } fn genesis_hash(&self) -> ::Hash { From a42122b54e36a37fecdb6a260da612dcc2805be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 13:49:10 +0100 Subject: [PATCH 38/58] backend: revert mark_finalized --- client/api/src/backend.rs | 2 +- client/api/src/in_mem.rs | 3 ++- client/db/src/lib.rs | 3 ++- client/light/src/backend.rs | 2 +- client/network/test/src/sync.rs | 6 +++--- client/service/src/client/client.rs | 7 +++---- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index e4884626557ae..399839af7b5b1 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -195,7 +195,7 @@ pub trait BlockImportOperation { fn mark_finalized( &mut self, id: BlockId, - justifications: Option, + justification: Option, ) -> sp_blockchain::Result<()>; /// Mark a block as new head. If both block import and set head are specified, set head diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 0385a6621b5bc..9b82dfa6aa38d 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -605,8 +605,9 @@ impl backend::BlockImportOperation for BlockImportOperatio fn mark_finalized( &mut self, block: BlockId, - justifications: Option, + justification: Option, ) -> sp_blockchain::Result<()> { + let justifications = justification.map(Justifications::from); self.finalized_blocks.push((block, justifications)); Ok(()) } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index b434aa4db6afa..e6ff30151fb89 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -812,8 +812,9 @@ impl sc_client_api::backend::BlockImportOperation for Bloc fn mark_finalized( &mut self, block: BlockId, - justifications: Option, + justification: Option, ) -> ClientResult<()> { + let justifications = justification.map(Justifications::from); self.finalized_blocks.push((block, justifications)); Ok(()) } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 0e511515e77e8..026a151bbb1d1 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -364,7 +364,7 @@ impl BlockImportOperation for ImportOperation fn mark_finalized( &mut self, block: BlockId, - _justification: Option, + _justifications: Option, ) -> ClientResult<()> { self.finalized_blocks.push(block); Ok(()) diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index e855fc287858c..e93e329e1584e 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -255,7 +255,7 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justifications(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - let just = Justifications::from((ID, Vec::new())); + let just = (ID, Vec::new()); net.peer(0).client().finalize_block(BlockId::Number(10), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(15), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(20), Some(just.clone()), true).unwrap(); @@ -309,7 +309,7 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - let just = Justifications::from((ID, Vec::new())); + let just = (ID, Vec::new()); net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(just), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); @@ -719,7 +719,7 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - let just = Some(Justifications::from((ID, Vec::new()))); + let just = Some((ID, Vec::new())); net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 6c96470476ae3..2569b878d9ee5 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -906,7 +906,7 @@ impl Client where &self, operation: &mut ClientImportOperation, block: Block::Hash, - justifications: Option, + justification: Option, best_block: Block::Hash, notify: bool, ) -> sp_blockchain::Result<()> { @@ -948,7 +948,7 @@ impl Client where } assert_eq!(enacted.last().map(|e| e.hash), Some(block)); - operation.op.mark_finalized(BlockId::Hash(block), justifications)?; + operation.op.mark_finalized(BlockId::Hash(block), justification)?; if notify { // sometimes when syncing, tons of blocks can be finalized at once. @@ -1816,11 +1816,10 @@ impl Finalizer for Client where ) -> sp_blockchain::Result<()> { let last_best = self.backend.blockchain().info().best_hash; let to_finalize_hash = self.backend.blockchain().expect_block_hash_from_id(&id)?; - let justifications = justification.map(Justifications::from); self.apply_finality_with_block_hash( operation, to_finalize_hash, - justifications, + justification, last_best, notify, ) From 356176fec1427a1558c143192bf4dbd5bc064734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 14:13:04 +0100 Subject: [PATCH 39/58] backend: revert changes to finalize_block --- client/api/src/backend.rs | 2 +- client/api/src/in_mem.rs | 15 +++++++-------- client/db/src/changes_tries_storage.rs | 4 ++-- client/db/src/lib.rs | 7 ++++--- client/light/src/backend.rs | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 399839af7b5b1..3108ba899467b 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -428,7 +428,7 @@ pub trait Backend: AuxStore + Send + Sync { fn finalize_block( &self, block: BlockId, - justifications: Option, + justification: Option, ) -> sp_blockchain::Result<()>; /// Append justification to the block with the given Id. diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 9b82dfa6aa38d..6b3def14f1ee6 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -272,7 +272,7 @@ impl Blockchain { Ok(()) } - fn finalize_header(&self, id: BlockId, justifications: Option) -> sp_blockchain::Result<()> { + fn finalize_header(&self, id: BlockId, justification: Option) -> sp_blockchain::Result<()> { let hash = match self.header(id)? { Some(h) => h.hash(), None => return Err(sp_blockchain::Error::UnknownBlock(format!("{}", id))), @@ -281,7 +281,7 @@ impl Blockchain { let mut storage = self.storage.write(); storage.finalized_hash = hash; - if justifications.is_some() { + if justification.is_some() { let block = storage.blocks.get_mut(&hash) .expect("hash was fetched from a block in the db; qed"); @@ -289,7 +289,7 @@ impl Blockchain { StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j }; - *block_justifications = justifications; + *block_justifications = justification.map(Justifications::from); } Ok(()) @@ -520,7 +520,7 @@ pub struct BlockImportOperation { old_state: InMemoryBackend>, new_state: Option<> as StateBackend>>::Transaction>, aux: Vec<(Vec, Option>)>, - finalized_blocks: Vec<(BlockId, Option)>, + finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, } @@ -607,8 +607,7 @@ impl backend::BlockImportOperation for BlockImportOperatio block: BlockId, justification: Option, ) -> sp_blockchain::Result<()> { - let justifications = justification.map(Justifications::from); - self.finalized_blocks.push((block, justifications)); + self.finalized_blocks.push((block, justification)); Ok(()) } @@ -724,9 +723,9 @@ impl backend::Backend for Backend where Block::Hash fn finalize_block( &self, block: BlockId, - justifications: Option, + justification: Option, ) -> sp_blockchain::Result<()> { - self.blockchain.finalize_header(block, justifications) + self.blockchain.finalize_header(block, justification) } fn append_justification( diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index a1b43ea5dfc6b..3422bd91651fe 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -531,7 +531,7 @@ mod tests { }; use sp_blockchain::HeaderBackend as BlockchainHeaderBackend; use sp_core::H256; - use sp_runtime::{ConsensusEngineId, Justifications, testing::{Digest, Header}}; + use sp_runtime::{ConsensusEngineId, testing::{Digest, Header}}; use sp_runtime::traits::{Hash, BlakeTwo256}; use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage}; use crate::Backend; @@ -957,7 +957,7 @@ mod tests { let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); let config1 = Some(ChangesTrieConfiguration::new(2, 6)); let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - let just1 = Some(Justifications::from((CONS0_ENGINE_ID, vec![42]))); + let just1 = Some((CONS0_ENGINE_ID, vec![42])); backend.finalize_block(BlockId::Number(1), just1).unwrap(); let config2 = Some(ChangesTrieConfiguration::new(2, 7)); let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index e6ff30151fb89..424f0d6fc7bf8 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1647,7 +1647,7 @@ impl sc_client_api::backend::Backend for Backend { fn finalize_block( &self, block: BlockId, - justifications: Option, + justification: Option, ) -> ClientResult<()> { let mut transaction = Transaction::new(); let hash = self.blockchain.expect_block_hash_from_id(&block)?; @@ -1655,6 +1655,7 @@ impl sc_client_api::backend::Backend for Backend { let mut displaced = None; let mut changes_trie_cache_ops = None; + let justifications = justification.map(Justifications::from); let (hash, number, is_best, is_finalized) = self.finalize_block_with_transaction( &mut transaction, &hash, @@ -2563,12 +2564,12 @@ pub(crate) mod tests { let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); let _ = insert_header(&backend, 1, block0, None, Default::default()); - let justification = Some(Justifications::from((CONS0_ENGINE_ID, vec![1, 2, 3]))); + let justification = Some((CONS0_ENGINE_ID, vec![1, 2, 3])); backend.finalize_block(BlockId::Number(1), justification.clone()).unwrap(); assert_eq!( backend.blockchain().justifications(BlockId::Number(1)).unwrap(), - justification, + justification.map(Justifications::from), ); } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 026a151bbb1d1..536579e0b3749 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -194,7 +194,7 @@ impl ClientBackend for Backend> fn finalize_block( &self, block: BlockId, - _justification: Option, + _justification: Option, ) -> ClientResult<()> { self.blockchain.storage().finalize_header(block) } From 609bbe069659f5aa614c526a18feda44bee0f7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 14:29:22 +0100 Subject: [PATCH 40/58] backend: revert finalized_blocks --- client/db/src/lib.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 424f0d6fc7bf8..fc3a5797c1557 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -678,7 +678,7 @@ pub struct BlockImportOperation { changes_trie_config_update: Option>, pending_block: Option>, aux_ops: Vec<(Vec, Option>)>, - finalized_blocks: Vec<(BlockId, Option)>, + finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, commit_state: bool, } @@ -814,8 +814,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc block: BlockId, justification: Option, ) -> ClientResult<()> { - let justifications = justification.map(Justifications::from); - self.finalized_blocks.push((block, justifications)); + self.finalized_blocks.push((block, justification)); Ok(()) } @@ -1113,7 +1112,7 @@ impl Backend { hash: &Block::Hash, header: &Block::Header, last_finalized: Option, - justifications: Option, + justification: Option, changes_trie_cache_ops: &mut Option>, finalization_displaced: &mut Option>>, ) -> ClientResult<(Block::Hash, ::Number, bool, bool)> { @@ -1130,11 +1129,11 @@ impl Backend { finalization_displaced, )?; - if let Some(justifications) = justifications { + if let Some(justification) = justification { transaction.set_from_vec( columns::JUSTIFICATION, &utils::number_and_hash_to_lookup_key(number, hash)?, - justifications.encode(), + Justifications::from(justification).encode(), ); } Ok((*hash, number, false, true)) @@ -1655,13 +1654,12 @@ impl sc_client_api::backend::Backend for Backend { let mut displaced = None; let mut changes_trie_cache_ops = None; - let justifications = justification.map(Justifications::from); let (hash, number, is_best, is_finalized) = self.finalize_block_with_transaction( &mut transaction, &hash, &header, None, - justifications, + justification, &mut changes_trie_cache_ops, &mut displaced, )?; From 037e0a413e536a35939efd01b7aff6ca9ff227de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 15:39:50 +0100 Subject: [PATCH 41/58] db: add a quick early return for performance --- client/db/src/lib.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index fc3a5797c1557..43dd0cea36831 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1679,16 +1679,17 @@ impl sc_client_api::backend::Backend for Backend { let header = self.blockchain.expect_header(block)?; let number = *header.number(); - // Check if block is finalized first + // Check if the block is finalized first. let is_descendent_of = is_descendent_of(&self.blockchain, None); let last_finalized = self.blockchain.last_finalized()?; - if hash != last_finalized && !is_descendent_of(&hash, &last_finalized)? { - return Err(ClientError::BadJustification( - "Can't append Justification to unfinalized block".into(), - )); + + // We can do a quick check first, before doing a proper but more expensive check + if number > self.blockchain.info().finalized_number + || (hash != last_finalized && !is_descendent_of(&hash, &last_finalized)?) + { + return Err(ClientError::NotInFinalizedChain); } - // Read and merge Justification use sp_blockchain::Backend; let justifications = if let Some(mut stored_justifications) = self.blockchain.justifications(block)? { @@ -1702,14 +1703,12 @@ impl sc_client_api::backend::Backend for Backend { Justifications::from(justification) }; - // Set Justification in Transaction in the same way as in finalize_block_with_transaction transaction.set_from_vec( columns::JUSTIFICATION, &utils::number_and_hash_to_lookup_key(number, hash)?, justifications.encode(), ); - // Commit to storage self.storage.db.commit(transaction)?; Ok(()) From c5fe8f2afbfbe6623535ca837a59601d2d0a1518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 15:40:19 +0100 Subject: [PATCH 42/58] client: minor reviewer comments --- client/consensus/manual-seal/src/lib.rs | 2 +- client/db/src/changes_tries_storage.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 80742e0f7d497..48fea4be554ec 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -52,7 +52,7 @@ use sp_api::{ProvideRuntimeApi, TransactionFor}; /// The `ConsensusEngineId` of Manual Seal. // We should consider creating a new crate primitives/manual-seal for this, if it ends up being used // outside of this crate. -pub const MANUAL_SEAL_ENGINE_ID: ConsensusEngineId = [b'm', b'a', b's', b'e']; +pub const MANUAL_SEAL_ENGINE_ID: ConsensusEngineId = [b'm', b'a', b'n', b'l']; /// The verifier for the manual seal engine; instantly finalizes. struct ManualSealVerifier; diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 3422bd91651fe..523f6a7d89ac8 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -538,7 +538,7 @@ mod tests { use crate::tests::{Block, insert_header, prepare_changes}; use super::*; - const CONS0_ENGINE_ID: ConsensusEngineId = *b"CON0"; + const TEST_ENGINE_ID: ConsensusEngineId = *b"TEST"; fn changes(number: u64) -> Option, Vec)>> { Some(vec![(number.to_le_bytes().to_vec(), number.to_le_bytes().to_vec())]) @@ -957,7 +957,7 @@ mod tests { let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); let config1 = Some(ChangesTrieConfiguration::new(2, 6)); let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - let just1 = Some((CONS0_ENGINE_ID, vec![42])); + let just1 = Some((TEST_ENGINE_ID, vec![42])); backend.finalize_block(BlockId::Number(1), just1).unwrap(); let config2 = Some(ChangesTrieConfiguration::new(2, 7)); let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); From 5c4a735e85629b6239e27b855b50336681ed3c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 16:19:24 +0100 Subject: [PATCH 43/58] service/test: use local ConsensusEngineId --- Cargo.lock | 1 - client/service/test/Cargo.toml | 3 --- client/service/test/src/client/mod.rs | 12 +++++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6072db09c845..3a260678a441c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7586,7 +7586,6 @@ dependencies = [ "sp-consensus", "sp-core", "sp-externalities", - "sp-finality-grandpa", "sp-panic-handler", "sp-runtime", "sp-state-machine", diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index 026092224465c..e55320d6c5fb7 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -42,6 +42,3 @@ sc-executor = { version = "0.9.0", path = "../../executor" } sp-panic-handler = { version = "3.0.0", path = "../../../primitives/panic-handler" } parity-scale-codec = "2.0.0" sp-tracing = { version = "3.0.0", path = "../../../primitives/tracing" } - -[dev-dependencies] -sp-finality-grandpa = { version = "3.0.0", path = "../../../primitives/finality-grandpa" } diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 11cfed7bc9eea..1fb497ec680f3 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -36,8 +36,9 @@ use sc_client_db::{ }; use sc_block_builder::BlockBuilderProvider; use sc_service::client::{self, Client, LocalCallExecutor, new_in_mem}; -use sp_runtime::traits::{ - BlakeTwo256, Block as BlockT, Header as HeaderT, +use sp_runtime::{ + ConsensusEngineId, + traits::{BlakeTwo256, Block as BlockT, Header as HeaderT}, }; use substrate_test_runtime::TestAPI; use sp_state_machine::backend::Backend as _; @@ -49,7 +50,6 @@ use sp_consensus::{ BlockOrigin, SelectChain, BlockImport, Error as ConsensusError, BlockCheckParams, ImportResult, BlockStatus, BlockImportParams, ForkChoiceStrategy, }; -use sp_finality_grandpa::GRANDPA_ENGINE_ID; use sp_storage::StorageKey; use sp_trie::{TrieConfiguration, trie_types::Layout}; use sp_runtime::{generic::BlockId, DigestItem, Justifications}; @@ -58,6 +58,8 @@ use hex_literal::hex; mod light; mod db; +const TEST_ENGINE_ID: ConsensusEngineId = *b"TEST"; + native_executor_instance!( Executor, substrate_test_runtime_client::runtime::api::dispatch, @@ -1014,7 +1016,7 @@ fn import_with_justification() { client.import(BlockOrigin::Own, a2.clone()).unwrap(); // A2 -> A3 - let justification = Justifications::from((GRANDPA_ENGINE_ID, vec![1, 2, 3])); + let justification = Justifications::from((TEST_ENGINE_ID, vec![1, 2, 3])); let a3 = client.new_block_at( &BlockId::Hash(a2.hash()), Default::default(), @@ -1086,7 +1088,7 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { ); // importing B1 as finalized should trigger a re-org and set it as new best - let justification = Justifications::from((GRANDPA_ENGINE_ID, vec![1, 2, 3])); + let justification = Justifications::from((TEST_ENGINE_ID, vec![1, 2, 3])); client.import_justified(BlockOrigin::Own, b1.clone(), justification).unwrap(); assert_eq!( From 1667e41278483a2fbd40cf1f03a7d4c892d5e600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 22 Feb 2021 16:44:52 +0100 Subject: [PATCH 44/58] network: add link to issue for sending multiple Justifications --- client/network/src/block_request_handler.rs | 2 ++ client/network/src/lib.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 1d6edda23977b..cff27987a78ec 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -148,6 +148,8 @@ impl BlockRequestHandler { }; // To keep compatibility we only send the grandpa justification + // We want to replace this by sending all Justifications if the possible. See + // https://github.com/paritytech/substrate/issues/8172 for the issue tracking this work. let justification = justifications .map(|just| just.into_justification(GRANDPA_ENGINE_ID)); diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 2e26681497c56..77309f31a56a4 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -292,6 +292,7 @@ const MAX_CONNECTIONS_ESTABLISHED_INCOMING: u32 = 10_000; // Redefine this locally to avoid depending on the GRANDPA crate. // NOTE: This is purely during a backwards compatible transitionary period and should be removed // once we can assume all nodes can send and receive multiple Justifications +// See https://github.com/paritytech/substrate/issues/8172 const GRANDPA_ENGINE_ID: ConsensusEngineId = *b"FRNK"; /// Minimum Requirements for a Hash within Networking From d7ea7bd1157a5cb0212beae5396ecfcf3ee0cca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 23 Feb 2021 23:24:33 +0100 Subject: [PATCH 45/58] Apply suggestions from code review Co-authored-by: Pierre Krieger --- client/network/src/protocol/sync.rs | 5 +---- client/network/test/src/sync.rs | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 1bd5992d9c0c9..7067144370eea 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -1899,7 +1899,6 @@ mod test { use sc_block_builder::BlockBuilderProvider; use sp_blockchain::HeaderBackend; use sp_consensus::block_validation::DefaultBlockAnnounceValidator; - use sp_runtime::ConsensusEngineId; use substrate_test_runtime_client::{ runtime::{Block, Hash, Header}, ClientBlockImportExt, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, @@ -1907,8 +1906,6 @@ mod test { }; use futures::{future::poll_fn, executor::block_on}; - const ID: ConsensusEngineId = *b"TEST"; - #[test] fn processes_empty_response_on_justification_request_for_unknown_block() { // if we ask for a justification for a given block to a peer that doesn't know that block @@ -2399,7 +2396,7 @@ mod test { ); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); - let just = (ID, Vec::new()); + let just = (b"TEST", Vec::new()); client.finalize_block(BlockId::Hash(finalized_block.hash()), Some(just)).unwrap(); sync.update_chain_info(&info.best_hash, info.best_number); diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index e93e329e1584e..190b87e52bf8c 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -255,7 +255,7 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justifications(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - let just = (ID, Vec::new()); + let just = (b"FRNK", Vec::new()); net.peer(0).client().finalize_block(BlockId::Number(10), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(15), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(20), Some(just.clone()), true).unwrap(); @@ -309,7 +309,7 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - let just = (ID, Vec::new()); + let just = (b"FRNK", Vec::new()); net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(just), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); @@ -719,7 +719,7 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - let just = Some((ID, Vec::new())); + let just = Some((b"FRNK", Vec::new())); net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); From 87cf0d31c21bfcbf972ac94c9f5684301115f1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 23 Feb 2021 23:33:49 +0100 Subject: [PATCH 46/58] Apply suggestions from code review Co-authored-by: Pierre Krieger --- client/network/src/protocol.rs | 4 ++-- client/network/test/src/sync.rs | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 124465bb6623c..c2c76d8fc5cfd 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -657,10 +657,10 @@ impl Protocol { // For compatibility we assume that the incoming Justifications is an // `EncodedJustification`, as before, and that it can only be for GRANDPA. let justification: EncodedJustification = block_data.justification; - let justification = (GRANDPA_ENGINE_ID, justification); + let justification = (b"FRNK", justification); Some(Justifications::from(justification)) } else if block_data.is_empty_justification { - Some(Justifications::from((GRANDPA_ENGINE_ID, Vec::new()))) + Some(Justifications::from((b"FRNK", Vec::new()))) } else { None }, diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 190b87e52bf8c..075d1d2b52f0b 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -22,9 +22,7 @@ use futures::{Future, executor::block_on}; use super::*; use sp_consensus::block_validation::Validation; use substrate_test_runtime::Header; -use sp_runtime::{ConsensusEngineId, Justifications}; - -const ID: ConsensusEngineId = *b"FRNK"; +use sp_runtime::Justifications; fn test_ancestor_search_when_common_is(n: usize) { sp_tracing::try_init_simple(); @@ -277,7 +275,7 @@ fn sync_justifications() { .peer(0) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((ID, Vec::new()))) + .unwrap() != Some(Justifications::from((b"FRNK", Vec::new()))) { return Poll::Pending; } @@ -285,7 +283,7 @@ fn sync_justifications() { .peer(1) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((ID, Vec::new()))) + .unwrap() != Some(Justifications::from((b"FRNK", Vec::new()))) { return Poll::Pending; } @@ -322,12 +320,12 @@ fn sync_justifications_across_forks() { .peer(0) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((ID, Vec::new()))) + .unwrap() == Some(Justifications::from((b"FRNK", Vec::new()))) && net .peer(1) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((ID, Vec::new()))) + .unwrap() == Some(Justifications::from((b"FRNK", Vec::new()))) { Poll::Ready(()) } else { From b266679058263d30f530b9049af31bebcf249ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 23 Feb 2021 23:25:03 +0100 Subject: [PATCH 47/58] network: tweaks to review suggestions --- client/network/src/block_request_handler.rs | 15 ++++++++------ client/network/src/lib.rs | 8 +------- client/network/src/protocol.rs | 22 +++++++++++---------- client/network/src/protocol/sync.rs | 2 +- client/network/test/src/lib.rs | 3 ++- client/network/test/src/sync.rs | 14 ++++++------- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index cff27987a78ec..f3b7ec181ad9d 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -24,7 +24,6 @@ use crate::protocol::{message::BlockAttributes}; use crate::request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig}; use crate::schema::v1::block_request::FromBlock; use crate::schema::v1::{BlockResponse, Direction}; -use crate::GRANDPA_ENGINE_ID; use futures::channel::{mpsc, oneshot}; use futures::stream::StreamExt; use log::debug; @@ -147,11 +146,15 @@ impl BlockRequestHandler { None }; - // To keep compatibility we only send the grandpa justification - // We want to replace this by sending all Justifications if the possible. See - // https://github.com/paritytech/substrate/issues/8172 for the issue tracking this work. - let justification = justifications - .map(|just| just.into_justification(GRANDPA_ENGINE_ID)); + // In a follow up PR tracked by https://github.com/paritytech/substrate/issues/8172 + // we want to send/receive all Justifications if possible. + // For now we keep compatibility by selecting precisely the GRANDPA one, and not just + // the first one. When sending we could have just taken the first one, since we don't + // expect there to be any other kind currently, but when receiving we need to add the + // engine ID tag. + // The ID tag is hardcoded here to avoid depending on the GRANDPA crate, and will be + // removed when resolving the above issue. + let justification = justifications.map(|just| just.into_justification(*b"FRNK")); let is_empty_justification = justification .as_ref() diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 77309f31a56a4..556e71da23831 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -275,7 +275,7 @@ pub use service::{ }; pub use sc_peerset::ReputationChange; -use sp_runtime::{ConsensusEngineId, traits::{Block as BlockT, NumberFor}}; +use sp_runtime::traits::{Block as BlockT, NumberFor}; /// The maximum allowed number of established connections per peer. /// @@ -289,12 +289,6 @@ const MAX_CONNECTIONS_PER_PEER: usize = 2; /// The maximum number of concurrent established connections that were incoming. const MAX_CONNECTIONS_ESTABLISHED_INCOMING: u32 = 10_000; -// Redefine this locally to avoid depending on the GRANDPA crate. -// NOTE: This is purely during a backwards compatible transitionary period and should be removed -// once we can assume all nodes can send and receive multiple Justifications -// See https://github.com/paritytech/substrate/issues/8172 -const GRANDPA_ENGINE_ID: ConsensusEngineId = *b"FRNK"; - /// Minimum Requirements for a Hash within Networking pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {} diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index c2c76d8fc5cfd..4385c1091ae0a 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -19,7 +19,7 @@ use crate::{ chain::Client, config::{self, ProtocolId}, - error, GRANDPA_ENGINE_ID, + error, request_responses::RequestFailure, utils::{interval, LruHashSet}, }; @@ -43,9 +43,10 @@ use sp_consensus::{ block_validation::BlockAnnounceValidator, import_queue::{BlockImportResult, BlockImportError, IncomingBlock, Origin} }; -use sp_runtime::{generic::BlockId, EncodedJustification, Justifications}; -use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub +use sp_runtime::{ + Justifications, + generic::BlockId, + traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub}, }; use sp_arithmetic::traits::SaturatedConversion; use sync::{ChainSync, SyncState}; @@ -654,13 +655,14 @@ impl Protocol { None }, justifications: if !block_data.justification.is_empty() { - // For compatibility we assume that the incoming Justifications is an - // `EncodedJustification`, as before, and that it can only be for GRANDPA. - let justification: EncodedJustification = block_data.justification; - let justification = (b"FRNK", justification); - Some(Justifications::from(justification)) + // For compatibility we assume that the incoming Justifications is from GRANDPA. + // The ID tag is hardcoded here to avoid depending on the GRANDPA crate. + // NOTE: This is purely during a backwards compatible transitionary period and should be removed + // once we can assume all nodes can send and receive multiple Justifications + // See https://github.com/paritytech/substrate/issues/8172 + Some(Justifications::from((*b"FRNK", block_data.justification))) } else if block_data.is_empty_justification { - Some(Justifications::from((b"FRNK", Vec::new()))) + Some(Justifications::from((*b"FRNK", Vec::new()))) } else { None }, diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 7067144370eea..96d5ba3ffcf90 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -2396,7 +2396,7 @@ mod test { ); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); - let just = (b"TEST", Vec::new()); + let just = (*b"TEST", Vec::new()); client.finalize_block(BlockId::Hash(finalized_block.hash()), Some(just)).unwrap(); sync.update_chain_info(&info.best_hash, info.best_number); diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index bb3c517069a92..0eddadb427a5b 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -1036,7 +1036,8 @@ impl JustificationImport for ForceFinalized { _number: NumberFor, justifications: Justifications, ) -> Result<(), Self::Error> { - // WIP(JON) + // Currently we just import the first Justification. It would be useful to extend this in + // the future to also append any additional Justifications. let justification = justifications.into_iter().next(); self.0.finalize_block(BlockId::Hash(hash), justification, true) .map_err(|_| ConsensusError::InvalidJustification.into()) diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 075d1d2b52f0b..a986f9f8853b8 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -253,7 +253,7 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justifications(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - let just = (b"FRNK", Vec::new()); + let just = (*b"FRNK", Vec::new()); net.peer(0).client().finalize_block(BlockId::Number(10), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(15), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(20), Some(just.clone()), true).unwrap(); @@ -275,7 +275,7 @@ fn sync_justifications() { .peer(0) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((b"FRNK", Vec::new()))) + .unwrap() != Some(Justifications::from((*b"FRNK", Vec::new()))) { return Poll::Pending; } @@ -283,7 +283,7 @@ fn sync_justifications() { .peer(1) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((b"FRNK", Vec::new()))) + .unwrap() != Some(Justifications::from((*b"FRNK", Vec::new()))) { return Poll::Pending; } @@ -307,7 +307,7 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - let just = (b"FRNK", Vec::new()); + let just = (*b"FRNK", Vec::new()); net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(just), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); @@ -320,12 +320,12 @@ fn sync_justifications_across_forks() { .peer(0) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((b"FRNK", Vec::new()))) + .unwrap() == Some(Justifications::from((*b"FRNK", Vec::new()))) && net .peer(1) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((b"FRNK", Vec::new()))) + .unwrap() == Some(Justifications::from((*b"FRNK", Vec::new()))) { Poll::Ready(()) } else { @@ -717,7 +717,7 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - let just = Some((b"FRNK", Vec::new())); + let just = Some((*b"FRNK", Vec::new())); net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); From 7f8e0b306180db99a872c13f387f97e0fe43133e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 24 Feb 2021 10:22:48 +0100 Subject: [PATCH 48/58] network: revert change to BlockData for backwards compatibility --- client/network/src/protocol.rs | 13 ++++------- client/network/src/protocol/message.rs | 6 ++--- client/network/src/protocol/sync.rs | 27 ++++++++++++++++------ client/network/src/protocol/sync/blocks.rs | 2 +- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 4385c1091ae0a..e58e9729a51bd 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -654,15 +654,10 @@ impl Protocol { } else { None }, - justifications: if !block_data.justification.is_empty() { - // For compatibility we assume that the incoming Justifications is from GRANDPA. - // The ID tag is hardcoded here to avoid depending on the GRANDPA crate. - // NOTE: This is purely during a backwards compatible transitionary period and should be removed - // once we can assume all nodes can send and receive multiple Justifications - // See https://github.com/paritytech/substrate/issues/8172 - Some(Justifications::from((*b"FRNK", block_data.justification))) + justification: if !block_data.justification.is_empty() { + Some(block_data.justification) } else if block_data.is_empty_justification { - Some(Justifications::from((*b"FRNK", Vec::new()))) + Some(Vec::new()) } else { None }, @@ -993,7 +988,7 @@ impl Protocol { body: None, receipt: None, message_queue: None, - justifications: None, + justification: None, }, ], }, diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 7fca4384d6d92..01e9a5d7215af 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -148,7 +148,7 @@ pub struct RemoteReadResponse { pub mod generic { use bitflags::bitflags; use codec::{Encode, Decode, Input, Output}; - use sp_runtime::Justifications; + use sp_runtime::EncodedJustification; use super::{ RemoteReadResponse, Transactions, Direction, RequestId, BlockAttributes, RemoteCallResponse, ConsensusEngineId, @@ -232,8 +232,8 @@ pub mod generic { pub receipt: Option>, /// Block message queue if requested. pub message_queue: Option>, - /// Justifications if requested. - pub justifications: Option, + /// Justification if requested. + pub justification: Option, } /// Identifies starting point of a block sequence. diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 96d5ba3ffcf90..52885b3292684 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -44,7 +44,7 @@ use extra_requests::ExtraRequests; use libp2p::PeerId; use log::{debug, trace, warn, info, error}; use sp_runtime::{ - Justifications, + EncodedJustification, Justifications, generic::BlockId, traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, CheckedSub, SaturatedConversion, @@ -823,11 +823,13 @@ impl ChainSync { .drain(self.best_queued_number + One::one()) .into_iter() .map(|block_data| { + let justifications = + legacy_justification_mapping(block_data.block.justification); IncomingBlock { hash: block_data.block.hash, header: block_data.block.header, body: block_data.block.body, - justifications: block_data.block.justifications, + justifications, origin: block_data.origin, allow_missing_state: true, import_existing: false, @@ -846,7 +848,7 @@ impl ChainSync { hash: b.hash, header: b.header, body: b.body, - justifications: b.justifications, + justifications: legacy_justification_mapping(b.justification), origin: Some(who.clone()), allow_missing_state: true, import_existing: false, @@ -955,7 +957,7 @@ impl ChainSync { hash: b.hash, header: b.header, body: b.body, - justifications: b.justifications, + justifications: legacy_justification_mapping(b.justification), origin: Some(who.clone()), allow_missing_state: true, import_existing: false, @@ -1026,7 +1028,7 @@ impl ChainSync { return Err(BadPeer(who, rep::BAD_JUSTIFICATION)); } - block.justifications + block.justification } else { // we might have asked the peer for a justification on a block that we assumed it // had but didn't (regardless of whether it had a justification for it or not). @@ -1039,7 +1041,10 @@ impl ChainSync { None }; - if let Some((peer, hash, number, j)) = self.extra_justifications.on_response(who, justification) { + if let Some((peer, hash, number, j)) = self + .extra_justifications + .on_response(who, legacy_justification_mapping(justification)) + { return Ok(OnBlockJustification::Import { peer, hash, number, justifications: j }) } } @@ -1597,6 +1602,14 @@ impl ChainSync { } } +// This is purely during a backwards compatible transitionary period and should be removed +// once we can assume all nodes can send and receive multiple Justifications +// The ID tag is hardcoded here to avoid depending on the GRANDPA crate. +// See https://github.com/paritytech/substrate/issues/8172 +fn legacy_justification_mapping(justification: Option) -> Option { + justification.map(|just| (*b"FRNK", just).into()) +} + #[derive(Debug)] pub(crate) struct Metrics { pub(crate) queued_blocks: u32, @@ -2091,7 +2104,7 @@ mod test { body: Some(b.deconstruct().1), receipt: None, message_queue: None, - justifications: None, + justification: None, } ).collect(), } diff --git a/client/network/src/protocol/sync/blocks.rs b/client/network/src/protocol/sync/blocks.rs index a277a1524f773..60492f24ed8c3 100644 --- a/client/network/src/protocol/sync/blocks.rs +++ b/client/network/src/protocol/sync/blocks.rs @@ -227,7 +227,7 @@ mod test { body: None, message_queue: None, receipt: None, - justifications: None, + justification: None, }).collect() } From ad0599812ba5aca6f07f0d6fe6c017456117b35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Wed, 24 Feb 2021 12:50:19 +0100 Subject: [PATCH 49/58] Apply suggestion from code review Co-authored-by: Pierre Krieger --- client/finality-grandpa/src/finality_proof.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index b5892be399681..f3f12614d2dcb 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -132,7 +132,7 @@ pub struct FinalityProof { /// The hash of block F for which justification is provided. pub block: Header::Hash, /// Justification of the block F. - pub justification: EncodedJustification, + pub justification: Vec, /// The set of headers in the range (B; F] that we believe are unknown to the caller. Ordered. pub unknown_headers: Vec
, } From 3af19629fc10ff2c0bdc4d1e3c97674ae0e31275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Thu, 25 Feb 2021 14:47:14 +0100 Subject: [PATCH 50/58] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com> --- client/api/src/client.rs | 2 +- client/network/src/block_request_handler.rs | 2 +- client/network/src/protocol/sync.rs | 2 +- primitives/blockchain/src/backend.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 1cbe87e25d14f..97fe77c8d81e1 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -90,7 +90,7 @@ pub trait BlockBackend { /// Get block status. fn block_status(&self, id: &BlockId) -> sp_blockchain::Result; - /// Get block justifications set by id. + /// Get block justifications for the block with the given id. fn justifications(&self, id: &BlockId) -> sp_blockchain::Result>; /// Get block hash by number. diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index f3b7ec181ad9d..7742aec2e8835 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -146,7 +146,7 @@ impl BlockRequestHandler { None }; - // In a follow up PR tracked by https://github.com/paritytech/substrate/issues/8172 + // TODO: In a follow up PR tracked by https://github.com/paritytech/substrate/issues/8172 // we want to send/receive all Justifications if possible. // For now we keep compatibility by selecting precisely the GRANDPA one, and not just // the first one. When sending we could have just taken the first one, since we don't diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 52885b3292684..37f9a451b67d3 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -1605,7 +1605,7 @@ impl ChainSync { // This is purely during a backwards compatible transitionary period and should be removed // once we can assume all nodes can send and receive multiple Justifications // The ID tag is hardcoded here to avoid depending on the GRANDPA crate. -// See https://github.com/paritytech/substrate/issues/8172 +// TODO: https://github.com/paritytech/substrate/issues/8172 fn legacy_justification_mapping(justification: Option) -> Option { justification.map(|just| (*b"FRNK", just).into()) } diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index bbdfa1c41338e..6ee836acb6441 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -84,7 +84,7 @@ pub trait HeaderBackend: Send + Sync { pub trait Backend: HeaderBackend + HeaderMetadata { /// Get block body. Returns `None` if block is not found. fn body(&self, id: BlockId) -> Result::Extrinsic>>>; - /// Get block justifications. Returns `None` if justification does not exist. + /// Get block justifications. Returns `None` if no justification exists. fn justifications(&self, id: BlockId) -> Result>; /// Get last finalized block hash. fn last_finalized(&self) -> Result; From 3f5e07cacbc3b6cd209a83fdfc7920ff130c7b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 26 Feb 2021 14:27:52 +0100 Subject: [PATCH 51/58] primitives: update doc comment for Justifications --- primitives/runtime/src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index d078fd6e036df..801437cd99e6e 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -95,11 +95,13 @@ pub use either::Either; /// the block itself would allow swapping justifications to change the block's hash /// (and thus fork the chain). Sending a `Justification` alongside a block instead /// bypasses this problem. -pub type EncodedJustification = Vec; - -/// Justification together with an ID to tag its origin. +/// Each Justifications comes encoded, and together with an ID tag since we might have multiple +/// Justifications per block. pub type Justification = (ConsensusEngineId, EncodedJustification); +/// The Justification specific to a consensus engine, and encoded. +pub type EncodedJustification = Vec; + /// Collection of Justifications, since we might have more than one stored per block. #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] From a82979120b31f0dfdf9871e47115117234fef472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Fri, 26 Feb 2021 14:51:49 +0100 Subject: [PATCH 52/58] client/db/upgrade: avoid grandpa crate dependency --- Cargo.lock | 1 - client/db/Cargo.toml | 1 - client/db/src/upgrade.rs | 7 ++----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1c8a47e397d4..a65ca56cee339 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6829,7 +6829,6 @@ dependencies = [ "sp-consensus", "sp-core", "sp-database", - "sp-finality-grandpa", "sp-keyring", "sp-runtime", "sp-state-machine", diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 771d112e2bb06..72c26fead1c1c 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -33,7 +33,6 @@ sc-executor = { version = "0.9.0", path = "../executor" } sc-state-db = { version = "0.9.0", path = "../state-db" } sp-trie = { version = "3.0.0", path = "../../primitives/trie" } sp-consensus = { version = "0.9.0", path = "../../primitives/consensus/common" } -sp-finality-grandpa = { version = "3.0.0", path = "../../primitives/finality-grandpa" } sp-blockchain = { version = "3.0.0", path = "../../primitives/blockchain" } sp-database = { version = "3.0.0", path = "../../primitives/database" } parity-db = { version = "0.2.2", optional = true } diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index b9d74653b0826..0ef38a702a6e6 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -83,11 +83,8 @@ fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_b let mut transaction = db.transaction(); for key in keys { if let Some(justification) = db.get(columns::JUSTIFICATION, &key).map_err(db_err)? { - let justifications = sp_runtime::Justifications::from(( - sp_finality_grandpa::GRANDPA_ENGINE_ID, - justification, - )); - + // Tag each Justification with the hardcoded ID for GRANDPA. Avoid the dependency on the GRANDPA crate + let justifications = sp_runtime::Justifications::from((*b"FRNK", justification)); transaction.put_vec(columns::JUSTIFICATION, &key, justifications.encode()); } } From dda0b2d11afb8cd073c0b17ea717b17e99fded28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Mon, 1 Mar 2021 16:46:59 +0100 Subject: [PATCH 53/58] consensus: revert to single Justification for import_justification --- client/finality-grandpa/src/import.rs | 29 ++++++++--------- client/network/src/service.rs | 4 +-- client/network/test/src/lib.rs | 7 ++-- .../consensus/common/src/block_import.rs | 4 +-- .../consensus/common/src/import_queue.rs | 4 +-- .../common/src/import_queue/basic_queue.rs | 32 ++++++++++--------- 6 files changed, 38 insertions(+), 42 deletions(-) diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 70f6e2a52650a..312c9e9401a03 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -33,7 +33,7 @@ use sp_consensus::{ SelectChain, }; use sp_finality_grandpa::{ConsensusLog, ScheduledChange, SetId, GRANDPA_ENGINE_ID}; -use sp_runtime::Justifications; +use sp_runtime::Justification; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{ Block as BlockT, DigestFor, Header as HeaderT, NumberFor, Zero, @@ -128,14 +128,14 @@ impl JustificationImport &mut self, hash: Block::Hash, number: NumberFor, - justifications: Justifications, + justification: Justification, ) -> Result<(), Self::Error> { // this justification was requested by the sync service, therefore we // are not sure if it should enact a change or not. it could have been a // request made as part of initial sync but that means the justification // wasn't part of the block and was requested asynchronously, probably // makes sense to log in that case. - GrandpaBlockImport::import_justification(self, hash, number, justifications, false, false) + GrandpaBlockImport::import_justification(self, hash, number, justification, false, false) } } @@ -502,12 +502,15 @@ impl BlockImport _ => {}, } - match justifications { - Some(justifications) => { + let grandpa_justification = justifications + .map(|just| just.into_justification(GRANDPA_ENGINE_ID)) + .flatten(); + match grandpa_justification { + Some(justification) => { let import_res = self.import_justification( hash, number, - justifications, + (GRANDPA_ENGINE_ID, justification), needs_justification, initial_sync, ); @@ -615,24 +618,18 @@ where &mut self, hash: Block::Hash, number: NumberFor, - justifications: Justifications, + justification: Justification, enacts_change: bool, initial_sync: bool, ) -> Result<(), ConsensusError> { - if justifications.iter().filter(|j| j.0 == GRANDPA_ENGINE_ID).count() > 1 { + if justification.0 != GRANDPA_ENGINE_ID { return Err(ConsensusError::ClientImport( - "Received multiple GRANDPA Justifications for the same block.".into(), + "GRANDPA can only import GRANDPA Justifications.".into(), )); } - let grandpa_justification = justifications - .into_justification(GRANDPA_ENGINE_ID) - .ok_or(ConsensusError::ClientImport( - "GRANDPA can only import GRANDPA Justifications.".into(), - ))?; - let justification = GrandpaJustification::decode_and_verify_finalizes( - &grandpa_justification, + &justification.1, (hash, number), self.authority_set.set_id(), &self.authority_set.current_authorities(), diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 74ce9316fc41c..54a5559d2eaf9 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1452,11 +1452,11 @@ impl Future for NetworkWorker { } this.import_queue.import_blocks(origin, blocks); }, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::JustificationImport(origin, hash, nb, justification))) => { + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::JustificationImport(origin, hash, nb, justifications))) => { if let Some(metrics) = this.metrics.as_ref() { metrics.import_queue_justifications_submitted.inc(); } - this.import_queue.import_justification(origin, hash, nb, justification); + this.import_queue.import_justifications(origin, hash, nb, justifications); }, Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::InboundRequest { protocol, result, .. })) => { if let Some(metrics) = this.metrics.as_ref() { diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 0eddadb427a5b..afeb2b4dd330b 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -1034,12 +1034,9 @@ impl JustificationImport for ForceFinalized { &mut self, hash: H256, _number: NumberFor, - justifications: Justifications, + justification: Justification, ) -> Result<(), Self::Error> { - // Currently we just import the first Justification. It would be useful to extend this in - // the future to also append any additional Justifications. - let justification = justifications.into_iter().next(); - self.0.finalize_block(BlockId::Hash(hash), justification, true) + self.0.finalize_block(BlockId::Hash(hash), Some(justification), true) .map_err(|_| ConsensusError::InvalidJustification.into()) } } diff --git a/primitives/consensus/common/src/block_import.rs b/primitives/consensus/common/src/block_import.rs index 5ec66286f0982..9ecdbb39c9ca5 100644 --- a/primitives/consensus/common/src/block_import.rs +++ b/primitives/consensus/common/src/block_import.rs @@ -18,7 +18,7 @@ //! Block import helpers. use sp_runtime::traits::{Block as BlockT, DigestItemFor, Header as HeaderT, NumberFor, HashFor}; -use sp_runtime::Justifications; +use sp_runtime::{Justification, Justifications}; use serde::{Serialize, Deserialize}; use std::borrow::Cow; use std::collections::HashMap; @@ -345,6 +345,6 @@ pub trait JustificationImport { &mut self, hash: B::Hash, number: NumberFor, - justifications: Justifications, + justification: Justification, ) -> Result<(), Self::Error>; } diff --git a/primitives/consensus/common/src/import_queue.rs b/primitives/consensus/common/src/import_queue.rs index 9bdc20009b8a0..e88c059a6a7fc 100644 --- a/primitives/consensus/common/src/import_queue.rs +++ b/primitives/consensus/common/src/import_queue.rs @@ -102,8 +102,8 @@ pub trait Verifier: Send + Sync { pub trait ImportQueue: Send { /// Import bunch of blocks. fn import_blocks(&mut self, origin: BlockOrigin, blocks: Vec>); - /// Import a block justification. - fn import_justification( + /// Import block justifications. + fn import_justifications( &mut self, who: Origin, hash: B::Hash, diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index 8cef510fea37a..b7715d696e087 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -18,7 +18,7 @@ use std::{pin::Pin, time::Duration, marker::PhantomData}; use futures::{prelude::*, task::Context, task::Poll}; use futures_timer::Delay; -use sp_runtime::{Justifications, traits::{Block as BlockT, Header as HeaderT, NumberFor}}; +use sp_runtime::{Justification, Justifications, traits::{Block as BlockT, Header as HeaderT, NumberFor}}; use sp_utils::mpsc::{TracingUnboundedSender, tracing_unbounded, TracingUnboundedReceiver}; use prometheus_endpoint::Registry; @@ -112,22 +112,24 @@ impl ImportQueue for BasicQueue } } - fn import_justification( + fn import_justifications( &mut self, who: Origin, hash: B::Hash, number: NumberFor, justifications: Justifications, ) { - let res = self.justification_sender.unbounded_send( - worker_messages::ImportJustification(who, hash, number, justifications), - ); - - if res.is_err() { - log::error!( - target: "sync", - "import_justification: Background import task is no longer alive" + for justification in justifications { + let res = self.justification_sender.unbounded_send( + worker_messages::ImportJustification(who, hash, number, justification), ); + + if res.is_err() { + log::error!( + target: "sync", + "import_justification: Background import task is no longer alive" + ); + } } } @@ -143,7 +145,7 @@ mod worker_messages { use super::*; pub struct ImportBlocks(pub BlockOrigin, pub Vec>); - pub struct ImportJustification(pub Origin, pub B::Hash, pub NumberFor, pub Justifications); + pub struct ImportJustification(pub Origin, pub B::Hash, pub NumberFor, pub Justification); } /// The process of importing blocks. @@ -281,11 +283,11 @@ impl BlockImportWorker { who: Origin, hash: B::Hash, number: NumberFor, - justifications: Justifications + justification: Justification, ) { let started = wasm_timer::Instant::now(); let success = self.justification_import.as_mut().map(|justification_import| { - justification_import.import_justification(hash, number, justifications) + justification_import.import_justification(hash, number, justification) .map_err(|e| { debug!( target: "sync", @@ -476,7 +478,7 @@ mod tests { &mut self, _hash: Hash, _number: BlockNumber, - _justification: Justifications, + _justification: Justification, ) -> Result<(), Self::Error> { Ok(()) } @@ -560,7 +562,7 @@ mod tests { libp2p::PeerId::random(), hash, 1, - sp_runtime::Justifications::from((ENGINE_ID, Vec::new())), + (ENGINE_ID, Vec::new()), ))) .unwrap(); From e405ce92a8c3a815e48755061bc9d937dc8c8b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Mon, 15 Mar 2021 22:36:05 +0000 Subject: [PATCH 54/58] primitives: improve justifications docs --- primitives/runtime/src/lib.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 8f1899680d165..4508f84eefc38 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -96,26 +96,30 @@ pub use either::Either; /// the block itself would allow swapping justifications to change the block's hash /// (and thus fork the chain). Sending a `Justification` alongside a block instead /// bypasses this problem. -/// Each Justifications comes encoded, and together with an ID tag since we might have multiple -/// Justifications per block. +/// +/// Each justification is provided as an encoded blob, and is tagged with an ID +/// to identify the consensus engine that generated the proof (we might have +/// multiple justifications from different engines for the same block). pub type Justification = (ConsensusEngineId, EncodedJustification); -/// The Justification specific to a consensus engine, and encoded. +/// The encoded justification specific to a consensus engine. pub type EncodedJustification = Vec; -/// Collection of Justifications, since we might have more than one stored per block. +/// Collection of justifications for a given block, multiple justifications may +/// be provided by different consensus engines for the same block. #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] pub struct Justifications(Vec); impl Justifications { - /// Return an iterator over the Justifications. + /// Return an iterator over the justifications. pub fn iter(&self) -> impl Iterator { self.0.iter() } - /// Append a justification. Returns false if a Justification with the same ConsensusEngineId - /// already exists, in which case the Justification is not inserted. + /// Append a justification. Returns false if a justification with the same + /// `ConsensusEngineId` already exists, in which case the justification is + /// not inserted. pub fn append(&mut self, justification: Justification) -> bool { if self.get(justification.0).is_some() { return false; @@ -124,12 +128,14 @@ impl Justifications { true } - /// Return the encoded Justification for the given consensus engine, if it exists. + /// Return the encoded justification for the given consensus engine, if it + /// exists. pub fn get(&self, engine_id: ConsensusEngineId) -> Option<&EncodedJustification> { self.iter().find(|j| j.0 == engine_id).map(|j| &j.1) } - /// Return the encoded Justification + /// Return a copy of the encoded justification for the given consensus + /// engine, if it exists. pub fn into_justification(self, engine_id: ConsensusEngineId) -> Option { self.into_iter().find(|j| j.0 == engine_id).map(|j| j.1) } From aaac8c574c3aea6207e1a01f209c1c8b525749ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Mon, 15 Mar 2021 22:37:12 +0000 Subject: [PATCH 55/58] style cleanups --- client/api/src/in_mem.rs | 8 ++++---- client/consensus/babe/src/lib.rs | 2 +- client/consensus/manual-seal/src/lib.rs | 2 -- client/db/src/changes_tries_storage.rs | 6 ++---- client/light/src/backend.rs | 2 +- client/network/test/src/sync.rs | 14 +++++++------- client/service/test/src/client/light.rs | 9 ++++----- primitives/consensus/common/src/block_import.rs | 2 +- primitives/consensus/common/src/import_queue.rs | 8 ++++---- .../common/src/import_queue/basic_queue.rs | 6 ++---- test-utils/client/src/client_ext.rs | 7 +++---- 11 files changed, 29 insertions(+), 37 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 5a046420a5f79..c3d266954278e 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -306,18 +306,18 @@ impl Blockchain { .get_mut(&hash) .expect("hash was fetched from a block in the db; qed"); - let block_justification = match block { + let block_justifications = match block { StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j }; - if let Some(stored_justifications) = block_justification { + if let Some(stored_justifications) = block_justifications { if !stored_justifications.append(justification) { return Err(sp_blockchain::Error::BadJustification( "Duplicate consensus engine ID".into() )); } } else { - *block_justification = Some(Justifications::from(justification)); + *block_justifications = Some(Justifications::from(justification)); }; Ok(()) @@ -836,7 +836,7 @@ mod tests { } #[test] - fn append_and_retreive_justifications() { + fn append_and_retrieve_justifications() { let blockchain = test_blockchain(); let last_finalized = blockchain.last_finalized().unwrap(); let block = BlockId::Hash(last_finalized); diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 9a79bfda1cc52..727ee29221b22 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -1102,7 +1102,7 @@ where ) -> Result<(BlockImportParams, Option)>>), String> { trace!( target: "babe", - "Verifying origin: {:?} header: {:?} justification: {:?} body: {:?}", + "Verifying origin: {:?} header: {:?} justification(s): {:?} body: {:?}", origin, header, justifications, diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index f3e0cd2ee8e42..870640c1f2012 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -50,8 +50,6 @@ pub use self::{ use sp_api::{ProvideRuntimeApi, TransactionFor}; /// The `ConsensusEngineId` of Manual Seal. -// We should consider creating a new crate primitives/manual-seal for this, if it ends up being used -// outside of this crate. pub const MANUAL_SEAL_ENGINE_ID: ConsensusEngineId = [b'm', b'a', b'n', b'l']; /// The verifier for the manual seal engine; instantly finalizes. diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 523f6a7d89ac8..8051adc1832bc 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -531,15 +531,13 @@ mod tests { }; use sp_blockchain::HeaderBackend as BlockchainHeaderBackend; use sp_core::H256; - use sp_runtime::{ConsensusEngineId, testing::{Digest, Header}}; + use sp_runtime::testing::{Digest, Header}; use sp_runtime::traits::{Hash, BlakeTwo256}; use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage}; use crate::Backend; use crate::tests::{Block, insert_header, prepare_changes}; use super::*; - const TEST_ENGINE_ID: ConsensusEngineId = *b"TEST"; - fn changes(number: u64) -> Option, Vec)>> { Some(vec![(number.to_le_bytes().to_vec(), number.to_le_bytes().to_vec())]) } @@ -957,7 +955,7 @@ mod tests { let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); let config1 = Some(ChangesTrieConfiguration::new(2, 6)); let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - let just1 = Some((TEST_ENGINE_ID, vec![42])); + let just1 = Some((*b"TEST", vec![42])); backend.finalize_block(BlockId::Number(1), just1).unwrap(); let config2 = Some(ChangesTrieConfiguration::new(2, 7)); let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 536579e0b3749..52ace4fd94753 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -286,7 +286,7 @@ impl BlockImportOperation for ImportOperation &mut self, header: Block::Header, _body: Option>, - _justification: Option, + _justifications: Option, state: NewBlockState, ) -> ClientResult<()> { self.leaf_state = state; diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 37b13da7ebf7c..73f21961fbcc8 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -253,7 +253,7 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justifications(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - let just = (*b"FRNK", Vec::new()); + let just = (*b"TEST", Vec::new()); net.peer(0).client().finalize_block(BlockId::Number(10), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(15), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(20), Some(just.clone()), true).unwrap(); @@ -275,7 +275,7 @@ fn sync_justifications() { .peer(0) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((*b"FRNK", Vec::new()))) + .unwrap() != Some(Justifications::from((*b"TEST", Vec::new()))) { return Poll::Pending; } @@ -283,7 +283,7 @@ fn sync_justifications() { .peer(1) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((*b"FRNK", Vec::new()))) + .unwrap() != Some(Justifications::from((*b"TEST", Vec::new()))) { return Poll::Pending; } @@ -307,7 +307,7 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - let just = (*b"FRNK", Vec::new()); + let just = (*b"TEST", Vec::new()); net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(just), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); @@ -320,12 +320,12 @@ fn sync_justifications_across_forks() { .peer(0) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((*b"FRNK", Vec::new()))) + .unwrap() == Some(Justifications::from((*b"TEST", Vec::new()))) && net .peer(1) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((*b"FRNK", Vec::new()))) + .unwrap() == Some(Justifications::from((*b"TEST", Vec::new()))) { Poll::Ready(()) } else { @@ -717,7 +717,7 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - let just = Some((*b"FRNK", Vec::new())); + let just = Some((*b"TEST", Vec::new())); net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 0be8199e56aea..02d54a24c3135 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -28,9 +28,9 @@ use sc_light::{ }; use std::sync::Arc; use sp_runtime::{ - traits::{BlakeTwo256, HashFor, NumberFor}, - generic::BlockId, traits::{Block as _, Header as HeaderT}, Digest, - Justifications, + generic::BlockId, + traits::{BlakeTwo256, Block as _, HashFor, Header as HeaderT, NumberFor}, + Digest, Justifications, }; use std::collections::HashMap; use parking_lot::Mutex; @@ -372,14 +372,13 @@ fn execution_proof_is_generated_and_checked() { // prepare remote client let mut remote_client = substrate_test_runtime_client::new(); - const ID: sp_runtime::ConsensusEngineId = *b"TEST"; for i in 1u32..3u32 { let mut digest = Digest::default(); digest.push(sp_runtime::generic::DigestItem::Other::(i.to_le_bytes().to_vec())); remote_client.import_justified( BlockOrigin::Own, remote_client.new_block(digest).unwrap().build().unwrap().block, - Justifications::from((ID, Default::default())), + Justifications::from((*b"TEST", Default::default())), ).unwrap(); } diff --git a/primitives/consensus/common/src/block_import.rs b/primitives/consensus/common/src/block_import.rs index 9ecdbb39c9ca5..9b7995a2b00bd 100644 --- a/primitives/consensus/common/src/block_import.rs +++ b/primitives/consensus/common/src/block_import.rs @@ -128,7 +128,7 @@ pub struct BlockImportParams { /// re-executed in a runtime that checks digest equivalence -- the /// post-runtime digests are pushed back on after. pub header: Block::Header, - /// Justification provided for this block from the outside. + /// Justification(s) provided for this block from the outside. pub justifications: Option, /// Digest items that have been added after the runtime for external /// work, like a consensus signature. diff --git a/primitives/consensus/common/src/import_queue.rs b/primitives/consensus/common/src/import_queue.rs index e88c059a6a7fc..b6067645a8920 100644 --- a/primitives/consensus/common/src/import_queue.rs +++ b/primitives/consensus/common/src/import_queue.rs @@ -68,7 +68,7 @@ pub struct IncomingBlock { pub header: Option<::Header>, /// Block body if requested. pub body: Option::Extrinsic>>, - /// Justification if requested. + /// Justification(s) if requested. pub justifications: Option, /// The peer, we received this from pub origin: Option, @@ -182,8 +182,8 @@ pub(crate) fn import_single_block_metered, Transaction ) -> Result>, BlockImportError> { let peer = block.origin; - let (header, justification) = match (block.header, block.justifications) { - (Some(header), justification) => (header, justification), + let (header, justifications) = match (block.header, block.justifications) { + (Some(header), justifications) => (header, justifications), (None, _) => { if let Some(ref peer) = peer { debug!(target: "sync", "Header {} was not provided by {} ", block.hash, peer); @@ -238,7 +238,7 @@ pub(crate) fn import_single_block_metered, Transaction } let started = wasm_timer::Instant::now(); - let (mut import_block, maybe_keys) = verifier.verify(block_origin, header, justification, block.body) + let (mut import_block, maybe_keys) = verifier.verify(block_origin, header, justifications, block.body) .map_err(|msg| { if let Some(ref peer) = peer { trace!(target: "sync", "Verifying {}({}) from {} failed: {}", number, hash, peer, msg); diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index b7715d696e087..eb2b4b1fa7fcd 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -444,7 +444,7 @@ mod tests { &mut self, origin: BlockOrigin, header: Header, - _justification: Option, + _justifications: Option, _body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { Ok((BlockImportParams::new(origin, header), None)) @@ -556,13 +556,11 @@ mod tests { let mut import_justification = || { let hash = Hash::random(); - const ENGINE_ID: sp_runtime::ConsensusEngineId = *b"TEST"; - block_on(finality_sender.send(worker_messages::ImportJustification( libp2p::PeerId::random(), hash, 1, - (ENGINE_ID, Vec::new()), + (*b"TEST", Vec::new()), ))) .unwrap(); diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index 2b9c1d787d264..aa4856f6baf66 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -51,15 +51,14 @@ pub trait ClientBlockImportExt: Sized { fn import_as_best(&mut self, origin: BlockOrigin, block: Block) -> Result<(), ConsensusError>; /// Import a block and finalize it. - fn import_as_final(&mut self, origin: BlockOrigin, block: Block) - -> Result<(), ConsensusError>; + fn import_as_final(&mut self, origin: BlockOrigin, block: Block) -> Result<(), ConsensusError>; - /// Import block with justification, finalizes block. + /// Import block with justification(s), finalizes block. fn import_justified( &mut self, origin: BlockOrigin, block: Block, - justifications: Justifications + justifications: Justifications, ) -> Result<(), ConsensusError>; } From 3e4443f402762529295f56fcc3ab537a3410af18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Mon, 15 Mar 2021 22:37:33 +0000 Subject: [PATCH 56/58] use and_then --- client/finality-grandpa-warp-sync/src/proof.rs | 5 ++--- client/finality-grandpa/src/finality_proof.rs | 16 ++++------------ client/finality-grandpa/src/import.rs | 4 ++-- client/network/src/block_request_handler.rs | 8 ++++---- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/client/finality-grandpa-warp-sync/src/proof.rs b/client/finality-grandpa-warp-sync/src/proof.rs index 55ccffe31ea2b..e6fb989abc9d8 100644 --- a/client/finality-grandpa-warp-sync/src/proof.rs +++ b/client/finality-grandpa-warp-sync/src/proof.rs @@ -110,8 +110,7 @@ impl WarpSyncProof { let justification = backend .justifications(BlockId::Number(*last_block))? - .map(|just| just.into_justification(GRANDPA_ENGINE_ID)) - .flatten() + .and_then(|just| just.into_justification(GRANDPA_ENGINE_ID)) .expect( "header is last in set and contains standard change signal; \ must have justification; \ @@ -176,7 +175,7 @@ mod tests { use sp_blockchain::HeaderBackend; use sp_consensus::BlockOrigin; use sp_finality_grandpa::GRANDPA_ENGINE_ID; - use sp_keyring::Ed25519Keyring; + use sp_keyring::Ed25519Keyring; use sp_runtime::{generic::BlockId, traits::Header as _}; use std::sync::Arc; use substrate_test_runtime_client::{ diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index e636ffcce0f7d..80ba8cee9101e 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -190,18 +190,10 @@ where // Get the Justification stored at the last block of the set let last_block_for_set_id = BlockId::Number(last_block_for_set); let justification = - if let Some(justifications) = blockchain.justifications(last_block_for_set_id)? { - if let Some(grandpa_justification) = justifications.into_justification(GRANDPA_ENGINE_ID) { - grandpa_justification - } else { - trace!( - target: "afg", - "Justification found, but none for GRANDPA, \ - when making finality proof for {}. Returning empty proof.", - block, - ); - return Ok(None); - } + if let Some(grandpa_justification) = blockchain.justifications(last_block_for_set_id)? + .and_then(|justifications| justifications.into_justification(GRANDPA_ENGINE_ID)) + { + grandpa_justification } else { trace!( target: "afg", diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 74b82ad37a886..6814d5dfb6195 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -519,8 +519,8 @@ impl BlockImport } let grandpa_justification = justifications - .map(|just| just.into_justification(GRANDPA_ENGINE_ID)) - .flatten(); + .and_then(|just| just.into_justification(GRANDPA_ENGINE_ID)); + match grandpa_justification { Some(justification) => { let import_res = self.import_justification( diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 70a4579b4c0c7..2cc888c220f62 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -282,21 +282,21 @@ impl BlockRequestHandler { }; // TODO: In a follow up PR tracked by https://github.com/paritytech/substrate/issues/8172 - // we want to send/receive all Justifications if possible. + // we want to send/receive all justifications. // For now we keep compatibility by selecting precisely the GRANDPA one, and not just // the first one. When sending we could have just taken the first one, since we don't // expect there to be any other kind currently, but when receiving we need to add the // engine ID tag. // The ID tag is hardcoded here to avoid depending on the GRANDPA crate, and will be // removed when resolving the above issue. - let justification = justifications.map(|just| just.into_justification(*b"FRNK")); + let justification = justifications.and_then(|just| just.into_justification(*b"FRNK")); let is_empty_justification = justification .as_ref() - .map(|j| j.as_ref().map(|j| j.is_empty()).unwrap_or(false)) + .map(|j| j.is_empty()) .unwrap_or(false); - let justification = justification.unwrap_or_default().unwrap_or_default(); + let justification = justification.unwrap_or_default(); let body = if get_body { match self.client.block_body(&BlockId::Hash(hash))? { From 340e092a88e07da998ef613d8073f7aa0f2d0361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Mon, 15 Mar 2021 22:37:47 +0000 Subject: [PATCH 57/58] client: rename JUSTIFICATIONS db column --- client/db/src/lib.rs | 15 +++++++-------- client/db/src/upgrade.rs | 6 +++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 43dd0cea36831..acda057938e92 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -61,7 +61,7 @@ use sc_client_api::{ }; use sp_blockchain::{ Result as ClientResult, Error as ClientError, - well_known_cache_keys, HeaderBackend, + well_known_cache_keys, Backend as _, HeaderBackend, }; use codec::{Decode, Encode}; use hash_db::Prefix; @@ -352,7 +352,7 @@ pub(crate) mod columns { pub const KEY_LOOKUP: u32 = 3; pub const HEADER: u32 = 4; pub const BODY: u32 = 5; - pub const JUSTIFICATION: u32 = 6; + pub const JUSTIFICATIONS: u32 = 6; pub const CHANGES_TRIE: u32 = 7; pub const AUX: u32 = 8; /// Offchain workers local storage @@ -537,7 +537,7 @@ impl sc_client_api::blockchain::Backend for BlockchainDb) -> ClientResult> { - match read_db(&*self.db, columns::KEY_LOOKUP, columns::JUSTIFICATION, id)? { + match read_db(&*self.db, columns::KEY_LOOKUP, columns::JUSTIFICATIONS, id)? { Some(justification) => match Decode::decode(&mut &justification[..]) { Ok(justification) => Ok(Some(justification)), Err(err) => return Err(sp_blockchain::Error::Backend( @@ -1131,7 +1131,7 @@ impl Backend { if let Some(justification) = justification { transaction.set_from_vec( - columns::JUSTIFICATION, + columns::JUSTIFICATIONS, &utils::number_and_hash_to_lookup_key(number, hash)?, Justifications::from(justification).encode(), ); @@ -1242,8 +1242,8 @@ impl Backend { }, } } - if let Some(justification) = pending_block.justifications { - transaction.set_from_vec(columns::JUSTIFICATION, &lookup_key, justification.encode()); + if let Some(justifications) = pending_block.justifications { + transaction.set_from_vec(columns::JUSTIFICATIONS, &lookup_key, justifications.encode()); } if number.is_zero() { @@ -1690,7 +1690,6 @@ impl sc_client_api::backend::Backend for Backend { return Err(ClientError::NotInFinalizedChain); } - use sp_blockchain::Backend; let justifications = if let Some(mut stored_justifications) = self.blockchain.justifications(block)? { if !stored_justifications.append(justification) { @@ -1704,7 +1703,7 @@ impl sc_client_api::backend::Backend for Backend { }; transaction.set_from_vec( - columns::JUSTIFICATION, + columns::JUSTIFICATIONS, &utils::number_and_hash_to_lookup_key(number, hash)?, justifications.encode(), ); diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 0ef38a702a6e6..6c7cbbb4a1af6 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -77,15 +77,15 @@ fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_b let db = Database::open(&db_cfg, db_path).map_err(db_err)?; // Get all the keys we need to update - let keys: Vec<_> = db.iter(columns::JUSTIFICATION).map(|entry| entry.0).collect(); + let keys: Vec<_> = db.iter(columns::JUSTIFICATIONS).map(|entry| entry.0).collect(); // Read and update each entry let mut transaction = db.transaction(); for key in keys { - if let Some(justification) = db.get(columns::JUSTIFICATION, &key).map_err(db_err)? { + if let Some(justification) = db.get(columns::JUSTIFICATIONS, &key).map_err(db_err)? { // Tag each Justification with the hardcoded ID for GRANDPA. Avoid the dependency on the GRANDPA crate let justifications = sp_runtime::Justifications::from((*b"FRNK", justification)); - transaction.put_vec(columns::JUSTIFICATION, &key, justifications.encode()); + transaction.put_vec(columns::JUSTIFICATIONS, &key, justifications.encode()); } } db.write(transaction).map_err(db_err)?; From 5c5449e2e7585a303229bd7722b0688c24e4e673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20H=C3=A4ggblad?= Date: Tue, 16 Mar 2021 12:21:03 +0100 Subject: [PATCH 58/58] network: revert to using FRNK in network-test --- client/network/test/src/sync.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 73f21961fbcc8..953639dcc0e22 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -253,7 +253,7 @@ fn sync_justifications() { assert_eq!(net.peer(1).client().justifications(&BlockId::Number(10)).unwrap(), None); // we finalize block #10, #15 and #20 for peer 0 with a justification - let just = (*b"TEST", Vec::new()); + let just = (*b"FRNK", Vec::new()); net.peer(0).client().finalize_block(BlockId::Number(10), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(15), Some(just.clone()), true).unwrap(); net.peer(0).client().finalize_block(BlockId::Number(20), Some(just.clone()), true).unwrap(); @@ -275,7 +275,7 @@ fn sync_justifications() { .peer(0) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((*b"TEST", Vec::new()))) + .unwrap() != Some(Justifications::from((*b"FRNK", Vec::new()))) { return Poll::Pending; } @@ -283,7 +283,7 @@ fn sync_justifications() { .peer(1) .client() .justifications(&BlockId::Number(height)) - .unwrap() != Some(Justifications::from((*b"TEST", Vec::new()))) + .unwrap() != Some(Justifications::from((*b"FRNK", Vec::new()))) { return Poll::Pending; } @@ -307,7 +307,7 @@ fn sync_justifications_across_forks() { // for both and finalize the small fork instead. net.block_until_sync(); - let just = (*b"TEST", Vec::new()); + let just = (*b"FRNK", Vec::new()); net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(just), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); @@ -320,12 +320,12 @@ fn sync_justifications_across_forks() { .peer(0) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((*b"TEST", Vec::new()))) + .unwrap() == Some(Justifications::from((*b"FRNK", Vec::new()))) && net .peer(1) .client() .justifications(&BlockId::Number(10)) - .unwrap() == Some(Justifications::from((*b"TEST", Vec::new()))) + .unwrap() == Some(Justifications::from((*b"FRNK", Vec::new()))) { Poll::Ready(()) } else { @@ -717,7 +717,7 @@ fn can_sync_to_peers_with_wrong_common_block() { net.block_until_connected(); // both peers re-org to the same fork without notifying each other - let just = Some((*b"TEST", Vec::new())); + let just = Some((*b"FRNK", Vec::new())); net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), just.clone(), true).unwrap(); net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), just, true).unwrap(); let final_hash = net.peer(0).push_blocks(1, false); @@ -991,7 +991,7 @@ fn multiple_requests_are_accepted_as_long_as_they_are_not_fulfilled() { // Finalize the block and make the justification available. net.peer(0).client().finalize_block( BlockId::Number(10), - Some((*b"TEST", Vec::new())), + Some((*b"FRNK", Vec::new())), true, ).unwrap(); @@ -1002,7 +1002,7 @@ fn multiple_requests_are_accepted_as_long_as_they_are_not_fulfilled() { .peer(1) .client() .justifications(&BlockId::Number(10)) - .unwrap() != Some(Justifications::from((*b"TEST", Vec::new()))) + .unwrap() != Some(Justifications::from((*b"FRNK", Vec::new()))) { return Poll::Pending; }