From 9e3a67b686018f539a325eda55645bc4df90624a Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 12 Dec 2017 17:24:28 +0100 Subject: [PATCH 01/40] candidate statement importing --- Cargo.lock | 8 ++ Cargo.toml | 1 + candidate-agreement/Cargo.toml | 8 ++ candidate-agreement/src/lib.rs | 44 ++++++ candidate-agreement/src/table.rs | 234 +++++++++++++++++++++++++++++++ 5 files changed, 295 insertions(+) create mode 100644 candidate-agreement/Cargo.toml create mode 100644 candidate-agreement/src/lib.rs create mode 100644 candidate-agreement/src/table.rs diff --git a/Cargo.lock b/Cargo.lock index 2956ffe63d1d9..9b23da8a9478b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,6 +612,14 @@ dependencies = [ "polkadot-cli 0.1.0", ] +[[package]] +name = "polkadot-candidate-agreement" +version = "0.1.0" +dependencies = [ + "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "polkadot-primitives 0.1.0", +] + [[package]] name = "polkadot-cli" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 84e987c436ece..c2c7ef9e363f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ polkadot-cli = { path = "cli", version = "0.1" } [workspace] members = [ + "candidate-agreement", "client", "contracts", "primitives", diff --git a/candidate-agreement/Cargo.toml b/candidate-agreement/Cargo.toml new file mode 100644 index 0000000000000..842127d1167b4 --- /dev/null +++ b/candidate-agreement/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "polkadot-candidate-agreement" +version = "0.1.0" +authors = ["Robert Habermeier "] + +[dependencies] +futures = "0.1" +polkadot-primitives = { path = "../primitives" } diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs new file mode 100644 index 0000000000000..0e5c39844216f --- /dev/null +++ b/candidate-agreement/src/lib.rs @@ -0,0 +1,44 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Propagation and agreement of candidates. +//! +//! This is parameterized by 3 numbers: +//! N: the number of validators total +//! P: the number of parachains +//! F: the number of faulty nodes (s.t. 3F + 1 <= N) +//! We also define G as the number of validators per parachain (N/P) +//! +//! Validators are split into groups by parachain, and each validator might come +//! up its own candidate for their parachain. Within groups, validators pass around +//! their candidates and produce statements of validity. +//! +//! Any candidate that receives majority approval by the validators in a group +//! may be subject to inclusion. + +extern crate futures; +extern crate polkadot_primitives as primitives; + +use primitives::parachain; + +mod table; + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + } +} diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs new file mode 100644 index 0000000000000..894cb0cfbdb2b --- /dev/null +++ b/candidate-agreement/src/table.rs @@ -0,0 +1,234 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! The statement table. +//! +//! This stores messages other validators issue about candidates. +//! +//! These messages are used to create a proposal submitted to a BFT consensus process. +//! +//! Proposals can either be sets of candidates for inclusion or a special value indicating +//! that some unspecified misbehavior has occurred. +//! +//! Each parachain is associated with two sets of validators: those which can +//! propose and attest to validity of candidates, and those who can only attest +//! to availability. + +use std::collections::{HashSet, HashMap}; +use std::hash::Hash; + +/// Statements circulated among peers. +pub enum Statement { + /// Broadcast by a validator to indicate that this is his candidate for + /// inclusion. + /// + /// Broadcasting two different candidate messages per round is not allowed. + Candidate(C::Candidate), + /// Broadcast by a validator to attest that the candidate with given digest + /// is valid. + Valid(C::Digest), + /// Broadcast by a validator to attest that the auxiliary data for a candidate + /// with given digest is available. + Available(C::Digest), + /// Broadcast by a validator to attest that the candidate with given digest + /// is invalid. + Invalid(C::Digest), +} + +/// A signed statement. +pub struct SignedStatement { + /// The statement. + pub statement: Statement, + /// The signature. + pub signature: C::Signature, +} + +/// Context for the statement table. +pub trait Context { + /// A validator ID + type ValidatorId: Hash + Eq + Clone; + /// The digest (hash or other unique attribute) of a candidate. + type Digest: Hash + Eq + Clone; + /// Candidate type. + type Candidate: Ord + Clone; + /// The group ID type + type GroupId: Hash + Eq + Clone; + /// A signature type. + type Signature: Clone; + + /// get the digest of a candidate. + fn candidate_digest(&self, candidate: &Self::Candidate) -> Self::Digest; + + /// get the group of a candidate. + fn candidate_group(&self, candidate: &Self::Candidate) -> Self::GroupId; + + /// Whether a validator is a member of a group. + /// Members are meant to submit candidates and vote on validity. + fn is_member_of(&self, validator: &Self::ValidatorId, group: &Self::GroupId) -> bool; + + /// Whether a validator is an availability guarantor of a group. + /// Guarantors are meant to vote on availability for candidates submitted + /// in a group. + fn is_availability_guarantor_of( + &self, + validator: &Self::ValidatorId, + group: &Self::GroupId, + ) -> bool; + + // recover signer of statement. + fn statement_signer( + &self, + statement: &SignedStatement, + ) -> Option; + + // sign a statement with the local key. + fn sign_statement( + &self, + statement: Statement, + ) -> SignedStatement; +} + +/// Misbehavior: voting both ways on candidate validity. +pub struct ValidityDoubleVote { + /// The candidate digest + pub digest: C::Digest, + /// The signature on the true vote. + pub t_signature: C::Signature, + /// The signature on the false vote. + pub f_signature: C::Signature, +} + +/// Misbehavior: declaring multiple candidates. +pub struct MultipleCandidates { + /// The first candidate seen. + pub first: (C::Candidate, C::Signature), + /// The second candidate seen. + pub second: (C::Candidate, C::Signature), +} + +/// Misbehavior: submitted statement for wrong group. +pub struct UnauthorizedStatement { + /// A signed statement which was submitted without proper authority. + pub statement: SignedStatement, +} + +/// Different kinds of misbehavior. All of these kinds of malicious misbehavior +/// are easily provable and extremely disincentivized. +pub enum Misbehavior { + /// Voted invalid and valid on validity. + ValidityDoubleVote(ValidityDoubleVote), + /// Submitted multiple candidates. + MultipleCandidates(MultipleCandidates), + /// Submitted a message withou + UnauthorizedStatement(UnauthorizedStatement), +} + +// Votes on a specific candidate. +struct CandidateData { + group_id: C::GroupId, + candidate: C::Candidate, + validity_votes: HashMap, + indicated_bad_by: Vec, +} + +/// Stores votes +pub struct Table { + proposed_candidates: HashMap, + detected_misbehavior: HashMap>, + candidate_votes: HashMap>, +} + +impl Table { + /// Import a signed statement + pub fn import_statement(&mut self, context: &C, statement: SignedStatement) { + let signer = match context.statement_signer(&statement) { + None => return, + Some(signer) => signer, + }; + + let maybe_misbehavior = match statement.statement { + Statement::Candidate(candidate) => self.import_candidate( + context, + signer.clone(), + candidate, + statement.signature + ), + Statement::Valid(digest) => unimplemented!(), + _ => unimplemented!(), + }; + + if let Some(misbehavior) = maybe_misbehavior { + // all misbehavior in agreement is provable and actively malicious. + // punishments are not cumulative. + self.detected_misbehavior.insert(validator, misbehavior); + } + } + + fn import_candidate( + &mut self, + context: &C, + from: C::ValidatorId, + candidate: C::Candidate, + signature: C::Signature, + ) -> Option> { + use std::collections::hash_map::Entry; + + let group = context.candidate_group(&candidate); + if !context.is_member_of(&from, &group) { + return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + signature, + statement: Statement::Candidate(candidate), + }, + })); + } + + // check that validator hasn't already specified another candidate. + let digest = context.candidate_digest(&candidate); + + match self.proposed_candidates.entry(from.clone()) { + Entry::Occupied(occ) => { + // if digest is different, fetch candidate and + // note misbehavior. + let old_digest = &occ.get().0; + if old_digest != &digest { + let old_candidate = self.candidate_votes.get(old_digest) + .expect("proposed digest implies existence of votes entry; qed") + .candidate + .clone(); + + return Some(Misbehavior::MultipleCandidates(MultipleCandidates { + first: (old_candidate, occ.get().1.clone()), + second: (candidate, signature), + })); + } + } + Entry::Vacant(vacant) => { + vacant.insert((digest.clone(), signature)); + + // TODO: seed validity votes with issuer here? + self.candidate_votes.entry(digest).or_insert_with(move || CandidateData { + group_id: group, + candidate: candidate, + validity_votes: HashMap::new(), + indicated_bad_by: Vec::new(), + }); + } + } + + None + } +} From 6ee2205d8510839df4259801b1e29387927e03a9 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 12 Dec 2017 17:48:13 +0100 Subject: [PATCH 02/40] import votes on validity --- candidate-agreement/src/table.rs | 75 ++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 894cb0cfbdb2b..eb77166840e89 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -28,6 +28,7 @@ //! to availability. use std::collections::{HashSet, HashMap}; +use std::collections::hash_map::Entry; use std::hash::Hash; /// Statements circulated among peers. @@ -166,14 +167,27 @@ impl Table { candidate, statement.signature ), - Statement::Valid(digest) => unimplemented!(), + Statement::Valid(digest) => self.validity_vote( + context, + signer.clone(), + digest, + true, + statement.signature, + ), + Statement::Invalid(digest) => self.validity_vote( + context, + signer.clone(), + digest, + false, + statement.signature, + ), _ => unimplemented!(), }; if let Some(misbehavior) = maybe_misbehavior { // all misbehavior in agreement is provable and actively malicious. // punishments are not cumulative. - self.detected_misbehavior.insert(validator, misbehavior); + self.detected_misbehavior.insert(signer, misbehavior); } } @@ -184,8 +198,6 @@ impl Table { candidate: C::Candidate, signature: C::Signature, ) -> Option> { - use std::collections::hash_map::Entry; - let group = context.candidate_group(&candidate); if !context.is_member_of(&from, &group) { return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { @@ -231,4 +243,59 @@ impl Table { None } + + fn validity_vote( + &mut self, + context: &C, + from: C::ValidatorId, + digest: C::Digest, + valid: bool, + signature: C::Signature, + ) -> Option> { + let statement = SignedStatement { + signature: signature.clone(), + statement: if valid { + Statement::Valid(digest.clone()) + } else { + Statement::Invalid(digest.clone()) + } + }; + + let votes = match self.candidate_votes.get_mut(&digest) { + None => return None, // TODO: queue up but don't get DoS'ed + Some(votes) => votes, + }; + + // check that this validator actually can vote in this group. + if !context.is_member_of(&from, &votes.group_id) { + return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement + })); + } + + // check for double votes. + match votes.validity_votes.entry(from.clone()) { + Entry::Occupied(occ) => { + if occ.get().0 != valid { + let (t_signature, f_signature) = if valid { + (signature, occ.get().1.clone()) + } else { + (occ.get().1.clone(), signature) + }; + + return Some(Misbehavior::ValidityDoubleVote(ValidityDoubleVote { + digest: digest, + t_signature, + f_signature, + })); + } + } + Entry::Vacant(vacant) => { + vacant.insert((valid, signature)); + votes.indicated_bad_by.push(from); + } + } + + None + } } From abe2e674be8400ec0875d24a2a7e794045e92a59 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 12 Dec 2017 17:57:03 +0100 Subject: [PATCH 03/40] import availability votes --- candidate-agreement/src/table.rs | 46 +++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index eb77166840e89..31bc805fe2326 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -142,6 +142,7 @@ struct CandidateData { group_id: C::GroupId, candidate: C::Candidate, validity_votes: HashMap, + availability_votes: HashSet, indicated_bad_by: Vec, } @@ -236,6 +237,7 @@ impl Table { group_id: group, candidate: candidate, validity_votes: HashMap::new(), + availability_votes: HashSet::new(), indicated_bad_by: Vec::new(), }); } @@ -252,15 +254,6 @@ impl Table { valid: bool, signature: C::Signature, ) -> Option> { - let statement = SignedStatement { - signature: signature.clone(), - statement: if valid { - Statement::Valid(digest.clone()) - } else { - Statement::Invalid(digest.clone()) - } - }; - let votes = match self.candidate_votes.get_mut(&digest) { None => return None, // TODO: queue up but don't get DoS'ed Some(votes) => votes, @@ -269,7 +262,14 @@ impl Table { // check that this validator actually can vote in this group. if !context.is_member_of(&from, &votes.group_id) { return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { - statement + statement: SignedStatement { + signature: signature.clone(), + statement: if valid { + Statement::Valid(digest.clone()) + } else { + Statement::Invalid(digest.clone()) + } + } })); } @@ -298,4 +298,30 @@ impl Table { None } + + fn availability_vote( + &mut self, + context: &C, + from: C::ValidatorId, + digest: C::Digest, + signature: C::Signature, + ) -> Option> { + let votes = match self.candidate_votes.get_mut(&digest) { + None => return None, // TODO: queue up but don't get DoS'ed + Some(votes) => votes, + }; + + // check that this validator actually can vote in this group. + if !context.is_availability_guarantor_of(&from, &votes.group_id) { + return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + signature: signature.clone(), + statement: Statement::Available(digest), + } + })); + } + + votes.availability_votes.insert(from); + None + } } From 41dd470a3a3e803a0adbd82a64c7da503d14b647 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 12 Dec 2017 18:09:07 +0100 Subject: [PATCH 04/40] candidate receipt type --- primitives/src/parachain.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 7e931bed84342..223acf4c51579 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -49,6 +49,25 @@ pub struct Candidate { pub block: BlockData, } +/// Candidate receipt type. +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct CandidateReceipt { + /// The ID of the parachain this is a candidate for. + pub parachain_inex: Id, + /// The collator's account ID + pub collator: ::Address, + /// The head-data + pub head_data: HeadData, + /// Balance uploads to the relay chain. + pub balance_uploads: Vec<(::Address, ::uint::U256)>, + /// Egress queue roots. + pub egress_queue_roots: Vec<(Id, ::hash::H256)>, + /// Fees paid from the chain to the relay chain validators + pub fees: ::uint::U256, +} + /// Parachain ingress queue message. #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct Message(#[serde(with="bytes")] pub Vec); @@ -64,7 +83,7 @@ pub struct BlockData(#[serde(with="bytes")] pub Vec); pub struct Header(#[serde(with="bytes")] pub Vec); /// Parachain head data included in the chain. -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct HeadData(#[serde(with="bytes")] pub Vec); /// Parachain validation code. From 68b7b12b880b1bb28af743dbf59dbf56a480c3e8 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 12 Dec 2017 18:13:08 +0100 Subject: [PATCH 05/40] make table mod public --- candidate-agreement/src/lib.rs | 2 +- candidate-agreement/src/table.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs index 0e5c39844216f..6a1640f699afa 100644 --- a/candidate-agreement/src/lib.rs +++ b/candidate-agreement/src/lib.rs @@ -34,7 +34,7 @@ extern crate polkadot_primitives as primitives; use primitives::parachain; -mod table; +pub mod table; #[cfg(test)] mod tests { diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 31bc805fe2326..ae284f8bf29a1 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -20,8 +20,8 @@ //! //! These messages are used to create a proposal submitted to a BFT consensus process. //! -//! Proposals can either be sets of candidates for inclusion or a special value indicating -//! that some unspecified misbehavior has occurred. +//! Proposals are formed of sets of candidates which have the requisite number of +//! validity and availability votes. //! //! Each parachain is associated with two sets of validators: those which can //! propose and attest to validity of candidates, and those who can only attest @@ -182,7 +182,12 @@ impl Table { false, statement.signature, ), - _ => unimplemented!(), + Statement::Available(digest) => self.availability_vote( + context, + signer.clone(), + digest, + statement.signature, + ) }; if let Some(misbehavior) = maybe_misbehavior { From 773cecba2a9fec87193508a221df5fd9d7514320 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 12 Dec 2017 18:27:24 +0100 Subject: [PATCH 06/40] test context for table --- candidate-agreement/src/table.rs | 72 +++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index ae284f8bf29a1..1a97d32b8c011 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -94,12 +94,6 @@ pub trait Context { &self, statement: &SignedStatement, ) -> Option; - - // sign a statement with the local key. - fn sign_statement( - &self, - statement: Statement, - ) -> SignedStatement; } /// Misbehavior: voting both ways on candidate validity. @@ -330,3 +324,69 @@ impl Table { None } } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[derive(Copy, Clone, Hash, PartialEq, Eq)] + struct ValidatorId(usize); + + #[derive(Copy, Clone, Hash, PartialEq, Eq)] + struct GroupId(usize); + + // group, body + #[derive(Copy, Clone, Hash, PartialOrd, Ord, PartialEq, Eq)] + struct Candidate(usize, usize); + + #[derive(Copy, Clone, Hash)] + struct Signature(usize); + + #[derive(Copy, Clone, Hash, PartialEq, Eq)] + struct Digest(usize); + + struct TestContext { + // v -> (validity, availability) + validators: HashMap + } + + impl Context for TestContext { + type ValidatorId = ValidatorId; + type Digest = Digest; + type Candidate = Candidate; + type GroupId = GroupId; + type Signature = Signature; + + fn candidate_digest(&self, candidate: &Candidate) -> Digest { + Digest(candidate.1) + } + + fn candidate_group(&self, candidate: &Candidate) -> GroupId { + GroupId(candidate.0) + } + + fn is_member_of( + &self, + validator: &ValidatorId, + group: &GroupId + ) -> bool { + self.validators.get(validator).map(|v| &v.0 == group).unwrap_or(false) + } + + fn is_availability_guarantor_of( + &self, + validator: &ValidatorId, + group: &GroupId + ) -> bool { + self.validators.get(validator).map(|v| &v.1 == group).unwrap_or(false) + } + + fn statement_signer( + &self, + statement: &SignedStatement, + ) -> Option { + Some(ValidatorId(statement.signature.0)) + } + } +} From f0cd75153c3148ef7d7989048cd07261c8de89bc Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 13 Dec 2017 10:11:41 +0100 Subject: [PATCH 07/40] add harness for tests --- candidate-agreement/src/table.rs | 72 +++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 1a97d32b8c011..ba9ad1546974c 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -30,8 +30,10 @@ use std::collections::{HashSet, HashMap}; use std::collections::hash_map::Entry; use std::hash::Hash; +use std::fmt::Debug; /// Statements circulated among peers. +#[derive(PartialEq, Eq, Debug)] pub enum Statement { /// Broadcast by a validator to indicate that this is his candidate for /// inclusion. @@ -50,6 +52,7 @@ pub enum Statement { } /// A signed statement. +#[derive(PartialEq, Eq, Debug)] pub struct SignedStatement { /// The statement. pub statement: Statement, @@ -60,15 +63,15 @@ pub struct SignedStatement { /// Context for the statement table. pub trait Context { /// A validator ID - type ValidatorId: Hash + Eq + Clone; + type ValidatorId: Hash + Eq + Clone + Debug; /// The digest (hash or other unique attribute) of a candidate. - type Digest: Hash + Eq + Clone; + type Digest: Hash + Eq + Clone + Debug; /// Candidate type. - type Candidate: Ord + Clone; + type Candidate: Ord + Clone + Eq + Debug; /// The group ID type - type GroupId: Hash + Eq + Clone; + type GroupId: Hash + Eq + Clone + Debug; /// A signature type. - type Signature: Clone; + type Signature: Clone + Eq + Debug; /// get the digest of a candidate. fn candidate_digest(&self, candidate: &Self::Candidate) -> Self::Digest; @@ -97,6 +100,7 @@ pub trait Context { } /// Misbehavior: voting both ways on candidate validity. +#[derive(PartialEq, Eq, Debug)] pub struct ValidityDoubleVote { /// The candidate digest pub digest: C::Digest, @@ -107,6 +111,7 @@ pub struct ValidityDoubleVote { } /// Misbehavior: declaring multiple candidates. +#[derive(PartialEq, Eq, Debug)] pub struct MultipleCandidates { /// The first candidate seen. pub first: (C::Candidate, C::Signature), @@ -115,6 +120,7 @@ pub struct MultipleCandidates { } /// Misbehavior: submitted statement for wrong group. +#[derive(PartialEq, Eq, Debug)] pub struct UnauthorizedStatement { /// A signed statement which was submitted without proper authority. pub statement: SignedStatement, @@ -122,6 +128,7 @@ pub struct UnauthorizedStatement { /// Different kinds of misbehavior. All of these kinds of malicious misbehavior /// are easily provable and extremely disincentivized. +#[derive(PartialEq, Eq, Debug)] pub enum Misbehavior { /// Voted invalid and valid on validity. ValidityDoubleVote(ValidityDoubleVote), @@ -140,7 +147,17 @@ struct CandidateData { indicated_bad_by: Vec, } +/// Create a new, empty statement table. +pub fn create() -> Table { + Table { + proposed_candidates: HashMap::default(), + detected_misbehavior: HashMap::default(), + candidate_votes: HashMap::default(), + } +} + /// Stores votes +#[derive(Default)] pub struct Table { proposed_candidates: HashMap, detected_misbehavior: HashMap>, @@ -330,22 +347,23 @@ mod tests { use super::*; use std::collections::HashMap; - #[derive(Copy, Clone, Hash, PartialEq, Eq)] + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] struct ValidatorId(usize); - #[derive(Copy, Clone, Hash, PartialEq, Eq)] + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] struct GroupId(usize); // group, body - #[derive(Copy, Clone, Hash, PartialOrd, Ord, PartialEq, Eq)] + #[derive(Debug, Copy, Clone, Hash, PartialOrd, Ord, PartialEq, Eq)] struct Candidate(usize, usize); - #[derive(Copy, Clone, Hash)] + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] struct Signature(usize); - #[derive(Copy, Clone, Hash, PartialEq, Eq)] + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] struct Digest(usize); + #[derive(Debug, PartialEq, Eq)] struct TestContext { // v -> (validity, availability) validators: HashMap @@ -389,4 +407,38 @@ mod tests { Some(ValidatorId(statement.signature.0)) } } + + #[test] + fn submitting_two_candidates_is_misbehavior() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(2), GroupId(455))); + map + } + }; + + let mut table = create(); + let statement_a = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + + let statement_b = SignedStatement { + statement: Statement::Candidate(Candidate(2, 999)), + signature: Signature(1), + }; + + table.import_statement(&context, statement_a); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); + + table.import_statement(&context, statement_b); + assert_eq!( + table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), + &Misbehavior::MultipleCandidates(MultipleCandidates { + first: (Candidate(2, 100), Signature(1)), + second: (Candidate(2, 999), Signature(1)), + }) + ); + } } From f510e3d879c4c9c3e6a390118fbcb0ada29ce99b Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 13 Dec 2017 10:39:33 +0100 Subject: [PATCH 08/40] some tests for misbehavior --- candidate-agreement/src/lib.rs | 2 - candidate-agreement/src/table.rs | 94 ++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs index 6a1640f699afa..d90a424b7d817 100644 --- a/candidate-agreement/src/lib.rs +++ b/candidate-agreement/src/lib.rs @@ -32,8 +32,6 @@ extern crate futures; extern crate polkadot_primitives as primitives; -use primitives::parachain; - pub mod table; #[cfg(test)] diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index ba9ad1546974c..82895d93e2c99 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -441,4 +441,98 @@ mod tests { }) ); } + + #[test] + fn submitting_candidate_from_wrong_group_is_misbehavior() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(3), GroupId(455))); + map + } + }; + + let mut table = create(); + let statement = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + + table.import_statement(&context, statement); + + assert_eq!( + table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), + &Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }, + }) + ); + } + + #[test] + fn unauthorized_votes() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(2), GroupId(455))); + map.insert(ValidatorId(2), (GroupId(3), GroupId(222))); + map + } + }; + + let mut table = create(); + + let candidate_a = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + let candidate_a_digest = Digest(100); + + let candidate_b = SignedStatement { + statement: Statement::Candidate(Candidate(3, 987)), + signature: Signature(2), + }; + let candidate_b_digest = Digest(987); + + table.import_statement(&context, candidate_a); + table.import_statement(&context, candidate_b); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(2))); + + // validator 1 votes for availability on 2's candidate. + let bad_availability_vote = SignedStatement { + statement: Statement::Available(candidate_b_digest.clone()), + signature: Signature(1), + }; + table.import_statement(&context, bad_availability_vote); + + assert_eq!( + table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), + &Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + statement: Statement::Available(candidate_b_digest), + signature: Signature(1), + }, + }) + ); + + // validator 2 votes for validity on 1's candidate. + let bad_validity_vote = SignedStatement { + statement: Statement::Valid(candidate_a_digest.clone()), + signature: Signature(2), + }; + table.import_statement(&context, bad_validity_vote); + + assert_eq!( + table.detected_misbehavior.get(&ValidatorId(2)).unwrap(), + &Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + statement: Statement::Valid(candidate_a_digest), + signature: Signature(2), + }, + }) + ); + } } From ee2a86509138e1396c8fe1594bc3d300867cd4e8 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 13 Dec 2017 12:12:26 +0100 Subject: [PATCH 09/40] produce proposal from table --- candidate-agreement/src/table.rs | 129 ++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 82895d93e2c99..a06c149b61451 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -69,7 +69,7 @@ pub trait Context { /// Candidate type. type Candidate: Ord + Clone + Eq + Debug; /// The group ID type - type GroupId: Hash + Eq + Clone + Debug; + type GroupId: Hash + Eq + Clone + Debug + Ord; /// A signature type. type Signature: Clone + Eq + Debug; @@ -97,6 +97,9 @@ pub trait Context { &self, statement: &SignedStatement, ) -> Option; + + // requisite number of votes for validity and availability respectively from a group. + fn requisite_votes(&self, group: &Self::GroupId) -> (usize, usize); } /// Misbehavior: voting both ways on candidate validity. @@ -147,6 +150,17 @@ struct CandidateData { indicated_bad_by: Vec, } +impl CandidateData { + // Candidate data can be included in a proposal + // if it has enough validity and availability votes + // and no validators have called it bad. + fn can_be_included(&self, validity_threshold: usize, availability_threshold: usize) -> bool { + self.indicated_bad_by.is_empty() + && self.validity_votes.len() >= validity_threshold + && self.availability_votes.len() >= availability_threshold + } +} + /// Create a new, empty statement table. pub fn create() -> Table { Table { @@ -165,6 +179,35 @@ pub struct Table { } impl Table { + /// Produce a set of proposed candidates. + /// + /// This will be at most one per group, consisting of the + /// best candidate for each group with requisite votes for inclusion. + pub fn proposed_candidates(&self, context: &C) -> Vec { + use std::collections::BTreeMap; + use std::collections::btree_map::Entry as BTreeEntry; + + let mut best_candidates = BTreeMap::new(); + for candidate_data in self.candidate_votes.values() { + let group_id = &candidate_data.group_id; + let (validity_t, availability_t) = context.requisite_votes(group_id); + + if !candidate_data.can_be_included(validity_t, availability_t) { continue } + let candidate = &candidate_data.candidate; + match best_candidates.entry(group_id.clone()) { + BTreeEntry::Occupied(mut occ) => { + let mut candidate_ref = occ.get_mut(); + if *candidate_ref < candidate { + *candidate_ref = candidate; + } + } + BTreeEntry::Vacant(vacant) => { vacant.insert(candidate); }, + } + } + + best_candidates.values().map(|v| C::Candidate::clone(v)).collect::>() + } + /// Import a signed statement pub fn import_statement(&mut self, context: &C, statement: SignedStatement) { let signer = match context.statement_signer(&statement) { @@ -350,7 +393,7 @@ mod tests { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] struct ValidatorId(usize); - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + #[derive(Debug, Copy, Clone, Hash, PartialOrd, Ord, PartialEq, Eq)] struct GroupId(usize); // group, body @@ -406,6 +449,10 @@ mod tests { ) -> Option { Some(ValidatorId(statement.signature.0)) } + + fn requisite_votes(&self, _id: &GroupId) -> (usize, usize) { + (6, 34) + } } #[test] @@ -535,4 +582,82 @@ mod tests { }) ); } + + #[test] + fn validity_double_vote_is_misbehavior() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(2), GroupId(455))); + map.insert(ValidatorId(2), (GroupId(2), GroupId(246))); + map + } + }; + + let mut table = create(); + let statement = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + let candidate_digest = Digest(100); + + table.import_statement(&context, statement); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); + + let valid_statement = SignedStatement { + statement: Statement::Valid(candidate_digest.clone()), + signature: Signature(2), + }; + + let invalid_statement = SignedStatement { + statement: Statement::Invalid(candidate_digest.clone()), + signature: Signature(2), + }; + + table.import_statement(&context, valid_statement); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(2))); + + table.import_statement(&context, invalid_statement); + + assert_eq!( + table.detected_misbehavior.get(&ValidatorId(2)).unwrap(), + &Misbehavior::ValidityDoubleVote(ValidityDoubleVote { + digest: candidate_digest, + f_signature: Signature(2), + t_signature: Signature(2), + }) + ); + } + + #[test] + fn candidate_can_be_included() { + let validity_threshold = 6; + let availability_threshold = 34; + + let mut candidate = CandidateData:: { + group_id: GroupId(4), + candidate: Candidate(4, 12345), + validity_votes: HashMap::new(), + availability_votes: HashSet::new(), + indicated_bad_by: Vec::new(), + }; + + assert!(!candidate.can_be_included(validity_threshold, availability_threshold)); + + for i in 0..validity_threshold { + candidate.validity_votes.insert(ValidatorId(i + 100), (true, Signature(i + 100))); + } + + assert!(!candidate.can_be_included(validity_threshold, availability_threshold)); + + for i in 0..availability_threshold { + candidate.availability_votes.insert(ValidatorId(i + 255)); + } + + assert!(candidate.can_be_included(validity_threshold, availability_threshold)); + + candidate.indicated_bad_by.push(ValidatorId(1024)); + + assert!(!candidate.can_be_included(validity_threshold, availability_threshold)); + } } From 9f88742be2b29b097de173387587884473c1faff Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 13 Dec 2017 16:41:58 +0100 Subject: [PATCH 10/40] count candidate issuance as implicit vote --- candidate-agreement/src/table.rs | 196 +++++++++++++++++++++++-------- 1 file changed, 146 insertions(+), 50 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index a06c149b61451..6fa4b1d14c8c9 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -27,8 +27,7 @@ //! propose and attest to validity of candidates, and those who can only attest //! to availability. -use std::collections::{HashSet, HashMap}; -use std::collections::hash_map::Entry; +use std::collections::hash_map::{HashMap, Entry}; use std::hash::Hash; use std::fmt::Debug; @@ -102,15 +101,18 @@ pub trait Context { fn requisite_votes(&self, group: &Self::GroupId) -> (usize, usize); } -/// Misbehavior: voting both ways on candidate validity. +/// Misbehavior: voting more than one way on candidate validity. +/// +/// Since there are three possible ways to vote, a double vote is possible in +/// three possible combinations. #[derive(PartialEq, Eq, Debug)] -pub struct ValidityDoubleVote { - /// The candidate digest - pub digest: C::Digest, - /// The signature on the true vote. - pub t_signature: C::Signature, - /// The signature on the false vote. - pub f_signature: C::Signature, +pub enum ValidityDoubleVote { + /// Implicit vote by issuing and explicity voting validity. + IssuedAndValidity((C::Candidate, C::Signature), (C::Digest, C::Signature)), + /// Implicit vote by issuing and explicitly voting invalidity + IssuedAndInvalidity((C::Candidate, C::Signature), (C::Digest, C::Signature)), + /// Direct votes for validity and invalidity + ValidityAndInvalidity(C::Digest, C::Signature, C::Signature), } /// Misbehavior: declaring multiple candidates. @@ -141,16 +143,43 @@ pub enum Misbehavior { UnauthorizedStatement(UnauthorizedStatement), } -// Votes on a specific candidate. -struct CandidateData { +// kinds of votes for validity +#[derive(Clone, PartialEq, Eq)] +enum ValidityVote { + // implicit validity vote by issuing + Issued(S), + // direct validity vote + Valid(S), + // direct invalidity vote + Invalid(S), +} + +/// Stores votes and data about a candidate. +pub struct CandidateData { group_id: C::GroupId, candidate: C::Candidate, - validity_votes: HashMap, - availability_votes: HashSet, + validity_votes: HashMap>, + availability_votes: HashMap, indicated_bad_by: Vec, } impl CandidateData { + /// whether this has been indicated bad by anyone. + pub fn indicated_bad(&self) -> bool { + !self.indicated_bad_by.is_empty() + } + + /// Get an iterator over those who have indicated this candidate valid. + // TODO: impl trait + pub fn voted_valid_by<'a>(&'a self) -> Box + 'a> { + Box::new(self.validity_votes.iter().filter_map(|(v, vote)| { + match *vote { + ValidityVote::Issued(_) | ValidityVote::Valid(_) => Some(v.clone()), + ValidityVote::Invalid(_) => None, + } + })) + } + // Candidate data can be included in a proposal // if it has enough validity and availability votes // and no validators have called it bad. @@ -208,7 +237,20 @@ impl Table { best_candidates.values().map(|v| C::Candidate::clone(v)).collect::>() } - /// Import a signed statement + /// Get an iterator of all candidates with a given group. + // TODO: impl iterator + pub fn candidates_in_group<'a>(&'a self, group_id: C::GroupId) + -> Box> + 'a> + { + Box::new(self.candidate_votes.values().filter(move |c| c.group_id == group_id)) + } + + /// Drain all misbehavior observed up to this point. + pub fn drain_misbehavior(&mut self) -> HashMap> { + ::std::mem::replace(&mut self.detected_misbehavior, HashMap::new()) + } + + /// Import a signed statement. pub fn import_statement(&mut self, context: &C, statement: SignedStatement) { let signer = match context.statement_signer(&statement) { None => return, @@ -226,15 +268,13 @@ impl Table { context, signer.clone(), digest, - true, - statement.signature, + ValidityVote::Valid(statement.signature), ), Statement::Invalid(digest) => self.validity_vote( context, signer.clone(), digest, - false, - statement.signature, + ValidityVote::Invalid(statement.signature), ), Statement::Available(digest) => self.availability_vote( context, @@ -284,25 +324,30 @@ impl Table { return Some(Misbehavior::MultipleCandidates(MultipleCandidates { first: (old_candidate, occ.get().1.clone()), - second: (candidate, signature), + second: (candidate, signature.clone()), })); } } Entry::Vacant(vacant) => { - vacant.insert((digest.clone(), signature)); + vacant.insert((digest.clone(), signature.clone())); // TODO: seed validity votes with issuer here? - self.candidate_votes.entry(digest).or_insert_with(move || CandidateData { + self.candidate_votes.entry(digest.clone()).or_insert_with(move || CandidateData { group_id: group, candidate: candidate, validity_votes: HashMap::new(), - availability_votes: HashSet::new(), + availability_votes: HashMap::new(), indicated_bad_by: Vec::new(), }); } } - None + self.validity_vote( + context, + from, + digest, + ValidityVote::Issued(signature), + ) } fn validity_vote( @@ -310,8 +355,7 @@ impl Table { context: &C, from: C::ValidatorId, digest: C::Digest, - valid: bool, - signature: C::Signature, + vote: ValidityVote, ) -> Option> { let votes = match self.candidate_votes.get_mut(&digest) { None => return None, // TODO: queue up but don't get DoS'ed @@ -320,13 +364,20 @@ impl Table { // check that this validator actually can vote in this group. if !context.is_member_of(&from, &votes.group_id) { + let (sig, valid) = match vote { + ValidityVote::Valid(s) => (s, true), + ValidityVote::Invalid(s) => (s, false), + ValidityVote::Issued(_) => + panic!("implicit issuance vote only cast if the candidate entry already created successfully; qed"), + }; + return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { statement: SignedStatement { - signature: signature.clone(), + signature: sig, statement: if valid { - Statement::Valid(digest.clone()) + Statement::Valid(digest) } else { - Statement::Invalid(digest.clone()) + Statement::Invalid(digest) } } })); @@ -335,23 +386,33 @@ impl Table { // check for double votes. match votes.validity_votes.entry(from.clone()) { Entry::Occupied(occ) => { - if occ.get().0 != valid { - let (t_signature, f_signature) = if valid { - (signature, occ.get().1.clone()) - } else { - (occ.get().1.clone(), signature) + if occ.get() != &vote { + let double_vote_proof = match (occ.get().clone(), vote) { + (ValidityVote::Issued(iss), ValidityVote::Valid(good)) | + (ValidityVote::Valid(good), ValidityVote::Issued(iss)) => + ValidityDoubleVote::IssuedAndValidity((votes.candidate.clone(), iss), (digest, good)), + (ValidityVote::Issued(iss), ValidityVote::Invalid(bad)) | + (ValidityVote::Invalid(bad), ValidityVote::Issued(iss)) => + ValidityDoubleVote::IssuedAndInvalidity((votes.candidate.clone(), iss), (digest, bad)), + (ValidityVote::Valid(good), ValidityVote::Invalid(bad)) | + (ValidityVote::Invalid(bad), ValidityVote::Valid(good)) => + ValidityDoubleVote::ValidityAndInvalidity(digest, good, bad), + _ => { + // this would occur if two different but valid signatures + // on the same kind of vote occurred. + return None; + } }; - return Some(Misbehavior::ValidityDoubleVote(ValidityDoubleVote { - digest: digest, - t_signature, - f_signature, - })); + return Some(Misbehavior::ValidityDoubleVote(double_vote_proof)); } } Entry::Vacant(vacant) => { - vacant.insert((valid, signature)); - votes.indicated_bad_by.push(from); + if let ValidityVote::Invalid(_) = vote { + votes.indicated_bad_by.push(from); + } + + vacant.insert(vote); } } @@ -380,7 +441,7 @@ impl Table { })); } - votes.availability_votes.insert(from); + votes.availability_votes.insert(from, signature); None } } @@ -621,11 +682,46 @@ mod tests { assert_eq!( table.detected_misbehavior.get(&ValidatorId(2)).unwrap(), - &Misbehavior::ValidityDoubleVote(ValidityDoubleVote { - digest: candidate_digest, - f_signature: Signature(2), - t_signature: Signature(2), - }) + &Misbehavior::ValidityDoubleVote(ValidityDoubleVote::ValidityAndInvalidity( + candidate_digest, + Signature(2), + Signature(2), + )) + ); + } + + #[test] + fn issue_and_vote_is_misbehavior() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(2), GroupId(455))); + map + } + }; + + let mut table = create(); + let statement = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + let candidate_digest = Digest(100); + + table.import_statement(&context, statement); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); + + let extra_vote = SignedStatement { + statement: Statement::Valid(candidate_digest.clone()), + signature: Signature(1), + }; + + table.import_statement(&context, extra_vote); + assert_eq!( + table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), + &Misbehavior::ValidityDoubleVote(ValidityDoubleVote::IssuedAndValidity( + (Candidate(2, 100), Signature(1)), + (Digest(100), Signature(1)), + )) ); } @@ -638,20 +734,20 @@ mod tests { group_id: GroupId(4), candidate: Candidate(4, 12345), validity_votes: HashMap::new(), - availability_votes: HashSet::new(), + availability_votes: HashMap::new(), indicated_bad_by: Vec::new(), }; assert!(!candidate.can_be_included(validity_threshold, availability_threshold)); for i in 0..validity_threshold { - candidate.validity_votes.insert(ValidatorId(i + 100), (true, Signature(i + 100))); + candidate.validity_votes.insert(ValidatorId(i + 100), ValidityVote::Valid(Signature(i + 100))); } assert!(!candidate.can_be_included(validity_threshold, availability_threshold)); for i in 0..availability_threshold { - candidate.availability_votes.insert(ValidatorId(i + 255)); + candidate.availability_votes.insert(ValidatorId(i + 255), Signature(i + 255)); } assert!(candidate.can_be_included(validity_threshold, availability_threshold)); From 7b6436e455029759f53c87d865ac9275087abeff Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 13 Dec 2017 18:45:08 +0100 Subject: [PATCH 11/40] keep track of messages known by validators --- candidate-agreement/src/lib.rs | 14 ++-- candidate-agreement/src/table.rs | 127 ++++++++++++++++++++++--------- 2 files changed, 100 insertions(+), 41 deletions(-) diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs index d90a424b7d817..5e7e2bb4adfc1 100644 --- a/candidate-agreement/src/lib.rs +++ b/candidate-agreement/src/lib.rs @@ -16,18 +16,18 @@ //! Propagation and agreement of candidates. //! -//! This is parameterized by 3 numbers: -//! N: the number of validators total -//! P: the number of parachains -//! F: the number of faulty nodes (s.t. 3F + 1 <= N) -//! We also define G as the number of validators per parachain (N/P) -//! //! Validators are split into groups by parachain, and each validator might come //! up its own candidate for their parachain. Within groups, validators pass around //! their candidates and produce statements of validity. //! //! Any candidate that receives majority approval by the validators in a group -//! may be subject to inclusion. +//! may be subject to inclusion, unless any validators flag that candidate as invalid. +//! +//! Wrongly flagging as invalid should be strongly disincentivized, so that in the +//! equilibrium state it is not expected to happen. Likewise with the submission +//! of invalid blocks. +//! +//! Groups themselves may be compromised by malicious validators. extern crate futures; extern crate polkadot_primitives as primitives; diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 6fa4b1d14c8c9..209e43fd47af1 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -27,6 +27,7 @@ //! propose and attest to validity of candidates, and those who can only attest //! to availability. +use std::collections::HashSet; use std::collections::hash_map::{HashMap, Entry}; use std::hash::Hash; use std::fmt::Debug; @@ -59,6 +60,26 @@ pub struct SignedStatement { pub signature: C::Signature, } +// A unique trace for a class of valid statements issued by a validator. +// +// We keep track of which statements we have received or sent to other validators +// in order to prevent relaying the same data multiple times. +// +// The signature of the statement is replaced by the validator because the validator +// is unique while signatures are not (at least under common schemes like +// Schnorr or ECDSA). +#[derive(Hash, PartialEq, Eq, Clone)] +enum StatementTrace { + /// The candidate proposed by the validator. + Candidate(V), + /// A validity statement from that validator about the given digest. + Valid(V, D), + /// An invalidity statement from that validator about the given digest. + Invalid(V, D), + /// An availability statement from that validator about the given digest. + Available(V, D), +} + /// Context for the statement table. pub trait Context { /// A validator ID @@ -66,11 +87,11 @@ pub trait Context { /// The digest (hash or other unique attribute) of a candidate. type Digest: Hash + Eq + Clone + Debug; /// Candidate type. - type Candidate: Ord + Clone + Eq + Debug; + type Candidate: Ord + Eq + Clone + Debug; /// The group ID type - type GroupId: Hash + Eq + Clone + Debug + Ord; + type GroupId: Hash + Ord + Eq + Clone + Debug; /// A signature type. - type Signature: Clone + Eq + Debug; + type Signature: Eq + Clone + Debug; /// get the digest of a candidate. fn candidate_digest(&self, candidate: &Self::Candidate) -> Self::Digest; @@ -91,7 +112,8 @@ pub trait Context { group: &Self::GroupId, ) -> bool; - // recover signer of statement. + // recover signer of statement and ensure the signature corresponds to the + // statement. fn statement_signer( &self, statement: &SignedStatement, @@ -104,7 +126,7 @@ pub trait Context { /// Misbehavior: voting more than one way on candidate validity. /// /// Since there are three possible ways to vote, a double vote is possible in -/// three possible combinations. +/// three possible combinations (unordered) #[derive(PartialEq, Eq, Debug)] pub enum ValidityDoubleVote { /// Implicit vote by issuing and explicity voting validity. @@ -190,10 +212,16 @@ impl CandidateData { } } +// validator metadata +struct ValidatorData { + proposal: Option<(C::Digest, C::Signature)>, + known_statements: HashSet>, +} + /// Create a new, empty statement table. pub fn create() -> Table { Table { - proposed_candidates: HashMap::default(), + validator_data: HashMap::default(), detected_misbehavior: HashMap::default(), candidate_votes: HashMap::default(), } @@ -202,7 +230,7 @@ pub fn create() -> Table { /// Stores votes #[derive(Default)] pub struct Table { - proposed_candidates: HashMap, + validator_data: HashMap>, detected_misbehavior: HashMap>, candidate_votes: HashMap>, } @@ -251,12 +279,22 @@ impl Table { } /// Import a signed statement. - pub fn import_statement(&mut self, context: &C, statement: SignedStatement) { + /// + /// This can note the origin of the statement to indicate that he has + /// seen it already. + pub fn import_statement(&mut self, context: &C, statement: SignedStatement, from: Option) { let signer = match context.statement_signer(&statement) { None => return, Some(signer) => signer, }; + let trace = match statement.statement { + Statement::Candidate(_) => StatementTrace::Candidate(signer.clone()), + Statement::Valid(ref d) => StatementTrace::Valid(signer.clone(), d.clone()), + Statement::Invalid(ref d) => StatementTrace::Invalid(signer.clone(), d.clone()), + Statement::Available(ref d) => StatementTrace::Available(signer.clone(), d.clone()), + }; + let maybe_misbehavior = match statement.statement { Statement::Candidate(candidate) => self.import_candidate( context, @@ -288,9 +326,22 @@ impl Table { // all misbehavior in agreement is provable and actively malicious. // punishments are not cumulative. self.detected_misbehavior.insert(signer, misbehavior); + } else { + if let Some(from) = from { + self.note_trace(trace.clone(), from); + } + + self.note_trace(trace, signer); } } + fn note_trace(&mut self, trace: StatementTrace, known_by: C::ValidatorId) { + self.validator_data.entry(known_by).or_insert_with(|| ValidatorData { + proposal: None, + known_statements: HashSet::default(), + }).known_statements.insert(trace); + } + fn import_candidate( &mut self, context: &C, @@ -311,25 +362,33 @@ impl Table { // check that validator hasn't already specified another candidate. let digest = context.candidate_digest(&candidate); - match self.proposed_candidates.entry(from.clone()) { - Entry::Occupied(occ) => { + match self.validator_data.entry(from.clone()) { + Entry::Occupied(mut occ) => { // if digest is different, fetch candidate and // note misbehavior. - let old_digest = &occ.get().0; - if old_digest != &digest { - let old_candidate = self.candidate_votes.get(old_digest) - .expect("proposed digest implies existence of votes entry; qed") - .candidate - .clone(); - - return Some(Misbehavior::MultipleCandidates(MultipleCandidates { - first: (old_candidate, occ.get().1.clone()), - second: (candidate, signature.clone()), - })); + let existing = occ.get_mut(); + + if let Some((ref old_digest, ref old_sig)) = existing.proposal { + if old_digest != &digest { + let old_candidate = self.candidate_votes.get(old_digest) + .expect("proposed digest implies existence of votes entry; qed") + .candidate + .clone(); + + return Some(Misbehavior::MultipleCandidates(MultipleCandidates { + first: (old_candidate, old_sig.clone()), + second: (candidate, signature.clone()), + })); + } + } else { + existing.proposal = Some((digest.clone(), signature.clone())); } } Entry::Vacant(vacant) => { - vacant.insert((digest.clone(), signature.clone())); + vacant.insert(ValidatorData { + proposal: Some((digest.clone(), signature.clone())), + known_statements: HashSet::new(), + }); // TODO: seed validity votes with issuer here? self.candidate_votes.entry(digest.clone()).or_insert_with(move || CandidateData { @@ -537,10 +596,10 @@ mod tests { signature: Signature(1), }; - table.import_statement(&context, statement_a); + table.import_statement(&context, statement_a, None); assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); - table.import_statement(&context, statement_b); + table.import_statement(&context, statement_b, None); assert_eq!( table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), &Misbehavior::MultipleCandidates(MultipleCandidates { @@ -566,7 +625,7 @@ mod tests { signature: Signature(1), }; - table.import_statement(&context, statement); + table.import_statement(&context, statement, None); assert_eq!( table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), @@ -604,8 +663,8 @@ mod tests { }; let candidate_b_digest = Digest(987); - table.import_statement(&context, candidate_a); - table.import_statement(&context, candidate_b); + table.import_statement(&context, candidate_a, None); + table.import_statement(&context, candidate_b, None); assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); assert!(!table.detected_misbehavior.contains_key(&ValidatorId(2))); @@ -614,7 +673,7 @@ mod tests { statement: Statement::Available(candidate_b_digest.clone()), signature: Signature(1), }; - table.import_statement(&context, bad_availability_vote); + table.import_statement(&context, bad_availability_vote, None); assert_eq!( table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), @@ -631,7 +690,7 @@ mod tests { statement: Statement::Valid(candidate_a_digest.clone()), signature: Signature(2), }; - table.import_statement(&context, bad_validity_vote); + table.import_statement(&context, bad_validity_vote, None); assert_eq!( table.detected_misbehavior.get(&ValidatorId(2)).unwrap(), @@ -662,7 +721,7 @@ mod tests { }; let candidate_digest = Digest(100); - table.import_statement(&context, statement); + table.import_statement(&context, statement, None); assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); let valid_statement = SignedStatement { @@ -675,10 +734,10 @@ mod tests { signature: Signature(2), }; - table.import_statement(&context, valid_statement); + table.import_statement(&context, valid_statement, None); assert!(!table.detected_misbehavior.contains_key(&ValidatorId(2))); - table.import_statement(&context, invalid_statement); + table.import_statement(&context, invalid_statement, None); assert_eq!( table.detected_misbehavior.get(&ValidatorId(2)).unwrap(), @@ -707,7 +766,7 @@ mod tests { }; let candidate_digest = Digest(100); - table.import_statement(&context, statement); + table.import_statement(&context, statement, None); assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); let extra_vote = SignedStatement { @@ -715,7 +774,7 @@ mod tests { signature: Signature(1), }; - table.import_statement(&context, extra_vote); + table.import_statement(&context, extra_vote, None); assert_eq!( table.detected_misbehavior.get(&ValidatorId(1)).unwrap(), &Misbehavior::ValidityDoubleVote(ValidityDoubleVote::IssuedAndValidity( From 85b0bf0f1d06cd199956f1dbc203b5117c1462c0 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 14 Dec 2017 11:56:14 +0100 Subject: [PATCH 12/40] fix primitives compilation --- primitives/src/parachain.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 4431721f62740..29fc11e9b502b 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -76,7 +76,7 @@ pub struct Message(#[serde(with="bytes")] pub Vec); /// /// This is just an ordered vector of other parachains' egress queues, /// obtained according to the routing rules. -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct ConsolidatedIngress(pub Vec<(Id, Vec)>); /// Parachain block data. @@ -111,10 +111,10 @@ mod tests { assert_eq!(ser::to_string_pretty(&Candidate { parachain_index: 5.into(), collator_signature: 10.into(), - unprocessed_ingress: vec![ - (1, vec![Message(vec![2])]), - (2, vec![Message(vec![2]), Message(vec![3])]), - ], + unprocessed_ingress: ConsolidatedIngress(vec![ + (Id(1), vec![Message(vec![2])]), + (Id(2), vec![Message(vec![2]), Message(vec![3])]), + ]), block: BlockData(vec![1, 2, 3]), }), r#"{ "parachainIndex": 5, From 946f160e577ed997d6620da08f7e5da514fd6448 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 18 Dec 2017 11:08:29 +0100 Subject: [PATCH 13/40] simple BFT agreement --- candidate-agreement/Cargo.toml | 2 +- candidate-agreement/src/bft.rs | 92 ++++++++++++++++++++++++++++++++++ candidate-agreement/src/lib.rs | 9 +--- 3 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 candidate-agreement/src/bft.rs diff --git a/candidate-agreement/Cargo.toml b/candidate-agreement/Cargo.toml index 842127d1167b4..9a2dc0ffb77db 100644 --- a/candidate-agreement/Cargo.toml +++ b/candidate-agreement/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "polkadot-candidate-agreement" version = "0.1.0" -authors = ["Robert Habermeier "] +authors = ["Parity Technologies "] [dependencies] futures = "0.1" diff --git a/candidate-agreement/src/bft.rs b/candidate-agreement/src/bft.rs new file mode 100644 index 0000000000000..03c67fcb0fd68 --- /dev/null +++ b/candidate-agreement/src/bft.rs @@ -0,0 +1,92 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! BFT Agreement based on a proposal. +//! +//! This is based off of PBFT with an assumption that a proposal is already +//! known by each node. The proposals they have may differ, so the agreement +//! may never complete. + +use std::collections::HashSet; +use std::hash::Hash; + +use futures::{Future, Stream, Sink}; +use futures::future::{ok, loop_fn, Loop}; + +/// Messages over the proposal. +pub enum Message

{ + /// Prepare to vote for proposal P. + Prepare(P), +} + +/// A localized message, including the sender. +pub struct LocalizedMessage { + /// The message received. + pub message: Message

, + /// The sender of the message + pub sender: V, +} + +/// Reach BFT agreement. Input the local proposal, message input stream, message output stream, +/// and maximum number of faulty participants. +/// +/// Messages should only be yielded from the input stream if the sender is authorized +/// to send messages. +/// +/// The input stream also may never conclude or the agreement code will panic. +/// Duplicate messages are allowed. +/// +/// The output stream assumes that messages will eventually be delivered to all +/// honest participants, either by repropagation, gossip, or some reliable +/// broadcast mechanism. +/// +/// This will collect 2f + 1 "prepare" messages. Since this is all within a single +/// view, the commit phase is not necessary. +// TODO: consider cross-view committing? +// TODO: impl future. +pub fn agree<'a, P, I, O, V>(local_proposal: P, input: I, output: O, max_faulty: usize) + -> Box + 'a> + where + P: 'a + Eq + Clone, + V: 'a + Hash + Eq, + I: 'a + Stream>, + O: 'a + Sink,SinkError=I::Error>, +{ + let prepared = HashSet::new(); + + let broadcast_message = output.send(Message::Prepare(local_proposal.clone())); + + let wait_for_prepares = loop_fn((input, prepared), move |(input, mut prepared)| { + let local_proposal = local_proposal.clone(); + input.into_future().and_then(move |(msg, remainder)| { + let msg = msg.expect("input stream never concludes; qed"); + let LocalizedMessage { message: Message::Prepare(p), sender } = msg; + + if p == local_proposal { + prepared.insert(sender); + + // the threshold is 2f + 1, but this node makes up the one. + if prepared.len() >= max_faulty * 2 { + return ok(Loop::Break(p)) + } + } + + ok(Loop::Continue((remainder, prepared))) + }).map_err(|(e, _)| e) + }); + + Box::new(broadcast_message.and_then(move |_| wait_for_prepares)) +} diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs index 5e7e2bb4adfc1..b23b9c606cb07 100644 --- a/candidate-agreement/src/lib.rs +++ b/candidate-agreement/src/lib.rs @@ -29,14 +29,9 @@ //! //! Groups themselves may be compromised by malicious validators. +#[macro_use] extern crate futures; extern crate polkadot_primitives as primitives; +pub mod bft; pub mod table; - -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - } -} From 2f5e37532900e88bfaee7a3a55584f3a5a9e9b8d Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 18 Dec 2017 11:08:43 +0100 Subject: [PATCH 14/40] kill unused macro_use annotation --- candidate-agreement/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs index b23b9c606cb07..09dd56f5f0874 100644 --- a/candidate-agreement/src/lib.rs +++ b/candidate-agreement/src/lib.rs @@ -29,7 +29,6 @@ //! //! Groups themselves may be compromised by malicious validators. -#[macro_use] extern crate futures; extern crate polkadot_primitives as primitives; From 73ce6583117e9fcb714247fe575348aed8b52a3b Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 18 Dec 2017 11:47:56 +0100 Subject: [PATCH 15/40] tests for BFT agreement --- candidate-agreement/src/bft.rs | 119 +++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 5 deletions(-) diff --git a/candidate-agreement/src/bft.rs b/candidate-agreement/src/bft.rs index 03c67fcb0fd68..f41bfd46062da 100644 --- a/candidate-agreement/src/bft.rs +++ b/candidate-agreement/src/bft.rs @@ -58,12 +58,13 @@ pub struct LocalizedMessage { // TODO: consider cross-view committing? // TODO: impl future. pub fn agree<'a, P, I, O, V>(local_proposal: P, input: I, output: O, max_faulty: usize) - -> Box + 'a> + -> Box + Send + 'a> where - P: 'a + Eq + Clone, - V: 'a + Hash + Eq, - I: 'a + Stream>, - O: 'a + Sink,SinkError=I::Error>, + P: 'a + Send + Eq + Clone, + V: 'a + Send + Hash + Eq, + I: 'a + Send + Stream>, + O: 'a + Send + Sink,SinkError=I::Error>, + I::Error: Send { let prepared = HashSet::new(); @@ -90,3 +91,111 @@ pub fn agree<'a, P, I, O, V>(local_proposal: P, input: I, output: O, max_faulty: Box::new(broadcast_message.and_then(move |_| wait_for_prepares)) } + +#[cfg(test)] +mod tests { + use futures::{Future, Stream, Sink}; + use super::*; + + #[test] + fn broadcasts_message() { + let (i_tx, i_rx) = ::futures::sync::mpsc::channel::>(10); + let (o_tx, o_rx) = ::futures::sync::mpsc::channel(10); + let max_faulty = 3; + + let agreement = agree( + 100_000, + i_rx.map_err(|_| ()), + o_tx.sink_map_err(|_| ()), + max_faulty, + ); + + ::std::thread::spawn(move || { + let _i_tx = i_tx; + let _ = agreement.wait(); + }); + + let sent_message = o_rx.wait() + .next() + .expect("to have a next item") + .expect("not to have an error"); + let Message::Prepare(p) = sent_message; + + assert_eq!(p, 100_000); + } + + #[test] + fn concludes_on_2f_prepares() { + let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); + let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); + let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); + let max_faulty = 3; + + let agreement = agree( + 100_000, + i_rx.map_err(|_| ()), + o_tx.sink_map_err(|_| ()), + max_faulty, + ); + + let iter = (0..(max_faulty * 2)).map(|i| { + LocalizedMessage { + message: Message::Prepare(100_000), + sender: i, + } + }); + + let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); + + ::std::thread::spawn(move || { + ::std::thread::sleep(::std::time::Duration::from_secs(5)); + timeout_tx.send(None).unwrap(); + }); + + let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) + .wait() + .map(|(r, _)| r) + .map_err(|(e, _)| e) + .expect("not to have an error") + .expect("not to fail to agree"); + + assert_eq!(agreed_value, 100_000); + } + + #[test] + fn never_concludes_on_less_than_2f_prepares() { + let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); + let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); + let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); + let max_faulty = 3; + + let agreement = agree( + 100_000, + i_rx.map_err(|_| ()), + o_tx.sink_map_err(|_| ()), + max_faulty, + ); + + let iter = (1..(max_faulty * 2)).map(|i| { + LocalizedMessage { + message: Message::Prepare(100_000), + sender: i, + } + }); + + let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); + + ::std::thread::spawn(move || { + ::std::thread::sleep(::std::time::Duration::from_millis(250)); + timeout_tx.send(None).unwrap(); + }); + + let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) + .wait() + .map(|(r, _)| r) + .map_err(|(e, _)| e) + .expect("not to have an error"); + + assert!(agreed_value.is_none()); + } +} From adf8984a2153d9e3646e254fe8cd3a89b45564ca Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 18 Dec 2017 12:45:30 +0100 Subject: [PATCH 16/40] test for not concluding on different prepares --- candidate-agreement/src/bft.rs | 39 +++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/candidate-agreement/src/bft.rs b/candidate-agreement/src/bft.rs index f41bfd46062da..b536dc9f2f203 100644 --- a/candidate-agreement/src/bft.rs +++ b/candidate-agreement/src/bft.rs @@ -55,7 +55,7 @@ pub struct LocalizedMessage { /// /// This will collect 2f + 1 "prepare" messages. Since this is all within a single /// view, the commit phase is not necessary. -// TODO: consider cross-view committing? +// TODO: consider cross-view committing // TODO: impl future. pub fn agree<'a, P, I, O, V>(local_proposal: P, input: I, output: O, max_faulty: usize) -> Box + Send + 'a> @@ -198,4 +198,41 @@ mod tests { assert!(agreed_value.is_none()); } + + #[test] + fn never_concludes_on_2f_prepares_for_different_proposal() { + let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); + let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); + let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); + let max_faulty = 3; + + let agreement = agree( + 100_000, + i_rx.map_err(|_| ()), + o_tx.sink_map_err(|_| ()), + max_faulty, + ); + + let iter = (0..(max_faulty * 2)).map(|i| { + LocalizedMessage { + message: Message::Prepare(100_001), + sender: i, + } + }); + + let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); + + ::std::thread::spawn(move || { + ::std::thread::sleep(::std::time::Duration::from_millis(250)); + timeout_tx.send(None).unwrap(); + }); + + let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) + .wait() + .map(|(r, _)| r) + .map_err(|(e, _)| e) + .expect("not to have an error"); + + assert!(agreed_value.is_none()); + } } From 0112657a38eb6e531edda9a8326c28843bf19ea4 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 18 Dec 2017 14:17:22 +0100 Subject: [PATCH 17/40] return summary upon statement import --- candidate-agreement/src/table.rs | 225 +++++++++++++++++++++++++------ 1 file changed, 185 insertions(+), 40 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 209e43fd47af1..c072be3ee0c1a 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -176,6 +176,21 @@ enum ValidityVote { Invalid(S), } +/// A summary of import of a statement. +#[derive(Clone, PartialEq, Eq)] +pub struct Summary { + /// The digest of the candidate referenced. + pub candidate: D, + /// The group that candidate is in. + pub group_id: G, + /// How many validity votes are currently witnessed. + pub validity_votes: usize, + /// How many availability votes are currently witnessed. + pub availability_votes: usize, + /// Whether this has been signalled bad by at least one participant. + pub signalled_bad: bool, +} + /// Stores votes and data about a candidate. pub struct CandidateData { group_id: C::GroupId, @@ -210,6 +225,16 @@ impl CandidateData { && self.validity_votes.len() >= validity_threshold && self.availability_votes.len() >= availability_threshold } + + fn summary(&self, digest: C::Digest) -> Summary { + Summary { + candidate: digest, + group_id: self.group_id.clone(), + validity_votes: self.validity_votes.len() - self.indicated_bad_by.len(), + availability_votes: self.availability_votes.len(), + signalled_bad: self.indicated_bad(), + } + } } // validator metadata @@ -282,9 +307,11 @@ impl Table { /// /// This can note the origin of the statement to indicate that he has /// seen it already. - pub fn import_statement(&mut self, context: &C, statement: SignedStatement, from: Option) { + pub fn import_statement(&mut self, context: &C, statement: SignedStatement, from: Option) + -> Option> + { let signer = match context.statement_signer(&statement) { - None => return, + None => return None, Some(signer) => signer, }; @@ -295,7 +322,7 @@ impl Table { Statement::Available(ref d) => StatementTrace::Available(signer.clone(), d.clone()), }; - let maybe_misbehavior = match statement.statement { + let (maybe_misbehavior, maybe_summary) = match statement.statement { Statement::Candidate(candidate) => self.import_candidate( context, signer.clone(), @@ -328,14 +355,16 @@ impl Table { self.detected_misbehavior.insert(signer, misbehavior); } else { if let Some(from) = from { - self.note_trace(trace.clone(), from); + self.note_trace_seen(trace.clone(), from); } - self.note_trace(trace, signer); + self.note_trace_seen(trace, signer); } + + maybe_summary } - fn note_trace(&mut self, trace: StatementTrace, known_by: C::ValidatorId) { + fn note_trace_seen(&mut self, trace: StatementTrace, known_by: C::ValidatorId) { self.validator_data.entry(known_by).or_insert_with(|| ValidatorData { proposal: None, known_statements: HashSet::default(), @@ -348,15 +377,18 @@ impl Table { from: C::ValidatorId, candidate: C::Candidate, signature: C::Signature, - ) -> Option> { + ) -> (Option>, Option>) { let group = context.candidate_group(&candidate); if !context.is_member_of(&from, &group) { - return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { - statement: SignedStatement { - signature, - statement: Statement::Candidate(candidate), - }, - })); + return ( + Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + signature, + statement: Statement::Candidate(candidate), + }, + })), + None, + ); } // check that validator hasn't already specified another candidate. @@ -375,10 +407,13 @@ impl Table { .candidate .clone(); - return Some(Misbehavior::MultipleCandidates(MultipleCandidates { - first: (old_candidate, old_sig.clone()), - second: (candidate, signature.clone()), - })); + return ( + Some(Misbehavior::MultipleCandidates(MultipleCandidates { + first: (old_candidate, old_sig.clone()), + second: (candidate, signature.clone()), + })), + None, + ); } } else { existing.proposal = Some((digest.clone(), signature.clone())); @@ -415,9 +450,9 @@ impl Table { from: C::ValidatorId, digest: C::Digest, vote: ValidityVote, - ) -> Option> { + ) -> (Option>, Option>) { let votes = match self.candidate_votes.get_mut(&digest) { - None => return None, // TODO: queue up but don't get DoS'ed + None => return (None, None), // TODO: queue up but don't get DoS'ed Some(votes) => votes, }; @@ -430,16 +465,19 @@ impl Table { panic!("implicit issuance vote only cast if the candidate entry already created successfully; qed"), }; - return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { - statement: SignedStatement { - signature: sig, - statement: if valid { - Statement::Valid(digest) - } else { - Statement::Invalid(digest) + return ( + Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + signature: sig, + statement: if valid { + Statement::Valid(digest) + } else { + Statement::Invalid(digest) + } } - } - })); + })), + None, + ); } // check for double votes. @@ -459,12 +497,17 @@ impl Table { _ => { // this would occur if two different but valid signatures // on the same kind of vote occurred. - return None; + return (None, None); } }; - return Some(Misbehavior::ValidityDoubleVote(double_vote_proof)); + return ( + Some(Misbehavior::ValidityDoubleVote(double_vote_proof)), + None, + ) } + + return (None, None); } Entry::Vacant(vacant) => { if let ValidityVote::Invalid(_) = vote { @@ -475,7 +518,7 @@ impl Table { } } - None + (None, Some(votes.summary(digest))) } fn availability_vote( @@ -484,24 +527,27 @@ impl Table { from: C::ValidatorId, digest: C::Digest, signature: C::Signature, - ) -> Option> { + ) -> (Option>, Option>) { let votes = match self.candidate_votes.get_mut(&digest) { - None => return None, // TODO: queue up but don't get DoS'ed + None => return (None, None), // TODO: queue up but don't get DoS'ed Some(votes) => votes, }; // check that this validator actually can vote in this group. if !context.is_availability_guarantor_of(&from, &votes.group_id) { - return Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { - statement: SignedStatement { - signature: signature.clone(), - statement: Statement::Available(digest), - } - })); + return ( + Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { + statement: SignedStatement { + signature: signature.clone(), + statement: Statement::Available(digest), + } + })), + None + ); } votes.availability_votes.insert(from, signature); - None + (None, Some(votes.summary(digest))) } } @@ -815,4 +861,103 @@ mod tests { assert!(!candidate.can_be_included(validity_threshold, availability_threshold)); } + + #[test] + fn candidate_import_gives_summary() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(2), GroupId(455))); + map + } + }; + + let mut table = create(); + let statement = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + + let summary = table.import_statement(&context, statement, None) + .expect("candidate import to give summary"); + + assert_eq!(summary.candidate, Digest(100)); + assert_eq!(summary.group_id, GroupId(2)); + assert_eq!(summary.validity_votes, 1); + assert_eq!(summary.availability_votes, 0); + } + + #[test] + fn candidate_vote_gives_summary() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(2), GroupId(455))); + map.insert(ValidatorId(2), (GroupId(2), GroupId(455))); + map + } + }; + + let mut table = create(); + let statement = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + let candidate_digest = Digest(100); + + table.import_statement(&context, statement, None); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); + + let vote = SignedStatement { + statement: Statement::Valid(candidate_digest.clone()), + signature: Signature(2), + }; + + let summary = table.import_statement(&context, vote, None) + .expect("candidate vote to give summary"); + + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(2))); + + assert_eq!(summary.candidate, Digest(100)); + assert_eq!(summary.group_id, GroupId(2)); + assert_eq!(summary.validity_votes, 2); + assert_eq!(summary.availability_votes, 0); + } + + #[test] + fn availability_vote_gives_summary() { + let context = TestContext { + validators: { + let mut map = HashMap::new(); + map.insert(ValidatorId(1), (GroupId(2), GroupId(455))); + map.insert(ValidatorId(2), (GroupId(5), GroupId(2))); + map + } + }; + + let mut table = create(); + let statement = SignedStatement { + statement: Statement::Candidate(Candidate(2, 100)), + signature: Signature(1), + }; + let candidate_digest = Digest(100); + + table.import_statement(&context, statement, None); + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(1))); + + let vote = SignedStatement { + statement: Statement::Available(candidate_digest.clone()), + signature: Signature(2), + }; + + let summary = table.import_statement(&context, vote, None) + .expect("candidate vote to give summary"); + + assert!(!table.detected_misbehavior.contains_key(&ValidatorId(2))); + + assert_eq!(summary.candidate, Digest(100)); + assert_eq!(summary.group_id, GroupId(2)); + assert_eq!(summary.validity_votes, 1); + assert_eq!(summary.availability_votes, 1); + } } From 6a7c2c09ba88ac2271ae2f9dfaf2040ffbc3dd0b Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 18 Dec 2017 15:49:35 +0100 Subject: [PATCH 18/40] accept bft agreement on proposal not locally submitted --- candidate-agreement/src/bft.rs | 168 +++++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 27 deletions(-) diff --git a/candidate-agreement/src/bft.rs b/candidate-agreement/src/bft.rs index b536dc9f2f203..1cc9158127fc4 100644 --- a/candidate-agreement/src/bft.rs +++ b/candidate-agreement/src/bft.rs @@ -20,24 +20,37 @@ //! known by each node. The proposals they have may differ, so the agreement //! may never complete. -use std::collections::HashSet; +use std::collections::HashMap; use std::hash::Hash; use futures::{Future, Stream, Sink}; use futures::future::{ok, loop_fn, Loop}; /// Messages over the proposal. +#[derive(Debug, Clone)] pub enum Message

{ /// Prepare to vote for proposal P. Prepare(P), } /// A localized message, including the sender. -pub struct LocalizedMessage { +#[derive(Debug, Clone)] +pub struct LocalizedMessage { /// The message received. pub message: Message

, /// The sender of the message pub sender: V, + /// The signature of the message. + pub signature: S, +} + +/// The agreed-upon data. +#[derive(Debug, Clone)] +pub struct Agreed { + /// The agreed-upon proposal. + pub proposal: P, + /// The justification for the proposal. + pub justification: Vec>, } /// Reach BFT agreement. Input the local proposal, message input stream, message output stream, @@ -57,35 +70,83 @@ pub struct LocalizedMessage { /// view, the commit phase is not necessary. // TODO: consider cross-view committing // TODO: impl future. -pub fn agree<'a, P, I, O, V>(local_proposal: P, input: I, output: O, max_faulty: usize) - -> Box + Send + 'a> +pub fn agree<'a, P, V, S, F, I, O>( + local_proposal: P, + local_id: V, + mut sign_local: F, + input: I, + output: O, + max_faulty: usize, +) -> Box, Error=I::Error> + Send + 'a> where - P: 'a + Send + Eq + Clone, - V: 'a + Send + Hash + Eq, - I: 'a + Send + Stream>, - O: 'a + Send + Sink,SinkError=I::Error>, + P: 'a + Send + Hash + Eq + Clone, + V: 'a + Send + Hash + Eq + Clone, + S: 'a + Send + Eq + Clone, + F: 'a + Send + FnMut(&Message

) -> S, + I: 'a + Send + Stream>, + O: 'a + Send + Sink,SinkError=I::Error>, I::Error: Send { - let prepared = HashSet::new(); + use std::collections::hash_map::Entry; + + let voting_for = HashMap::new(); + let prepared = HashMap::new(); - let broadcast_message = output.send(Message::Prepare(local_proposal.clone())); + let local_prepare = { + let local_prepare = Message::Prepare(local_proposal); + let local_signature = sign_local(&local_prepare); - let wait_for_prepares = loop_fn((input, prepared), move |(input, mut prepared)| { - let local_proposal = local_proposal.clone(); + LocalizedMessage { + message: local_prepare, + sender: local_id, + signature: local_signature, + } + }; + + // broadcast out our local prepare message and shortcut it into our input + // stream. + let broadcast_message = output.send(local_prepare.clone()); + let input = ::futures::stream::once(Ok(local_prepare)).chain(input); + + let wait_for_prepares = loop_fn((input, voting_for, prepared), move |(input, mut voting_for, mut prepared)| { input.into_future().and_then(move |(msg, remainder)| { let msg = msg.expect("input stream never concludes; qed"); - let LocalizedMessage { message: Message::Prepare(p), sender } = msg; - - if p == local_proposal { - prepared.insert(sender); + let LocalizedMessage { message: Message::Prepare(p), sender, signature } = msg; - // the threshold is 2f + 1, but this node makes up the one. - if prepared.len() >= max_faulty * 2 { - return ok(Loop::Break(p)) + let is_complete = match voting_for.entry(sender) { + Entry::Occupied(_) => { + // TODO: handle double vote. + false } + Entry::Vacant(vacant) => { + vacant.insert((p.clone(), signature)); + let n = prepared.entry(p.clone()).or_insert(0); + *n += 1; + *n > max_faulty * 2 + } + }; + + if is_complete { + let justification = voting_for.into_iter().filter_map(|(v, (x, s))| { + if x == p { + Some(LocalizedMessage { + message: Message::Prepare(x), + sender: v, + signature: s, + }) + } else { + None + } + }).collect(); + + ok(Loop::Break(Agreed { + justification, + proposal: p, + })) + } else { + ok(Loop::Continue((remainder, voting_for, prepared))) } - ok(Loop::Continue((remainder, prepared))) }).map_err(|(e, _)| e) }); @@ -99,12 +160,14 @@ mod tests { #[test] fn broadcasts_message() { - let (i_tx, i_rx) = ::futures::sync::mpsc::channel::>(10); + let (i_tx, i_rx) = ::futures::sync::mpsc::channel::>(10); let (o_tx, o_rx) = ::futures::sync::mpsc::channel(10); let max_faulty = 3; let agreement = agree( 100_000, + 255, + |_msg| true, i_rx.map_err(|_| ()), o_tx.sink_map_err(|_| ()), max_faulty, @@ -119,13 +182,14 @@ mod tests { .next() .expect("to have a next item") .expect("not to have an error"); - let Message::Prepare(p) = sent_message; + let Message::Prepare(p) = sent_message.message; assert_eq!(p, 100_000); + assert_eq!(sent_message.sender, 255); } #[test] - fn concludes_on_2f_prepares() { + fn concludes_on_2f_prepares_for_local_proposal() { let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); @@ -133,6 +197,8 @@ mod tests { let agreement = agree( 100_000, + 255, + |_msg| true, i_rx.map_err(|_| ()), o_tx.sink_map_err(|_| ()), max_faulty, @@ -142,6 +208,7 @@ mod tests { LocalizedMessage { message: Message::Prepare(100_000), sender: i, + signature: true, } }); @@ -159,11 +226,11 @@ mod tests { .expect("not to have an error") .expect("not to fail to agree"); - assert_eq!(agreed_value, 100_000); + assert_eq!(agreed_value.proposal, 100_000); } #[test] - fn never_concludes_on_less_than_2f_prepares() { + fn concludes_on_2f_plus_one_prepares_for_alternate_proposal() { let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); @@ -171,6 +238,49 @@ mod tests { let agreement = agree( 100_000, + 255, + |_msg| true, + i_rx.map_err(|_| ()), + o_tx.sink_map_err(|_| ()), + max_faulty, + ); + + let iter = (0..(max_faulty * 2 + 1)).map(|i| { + LocalizedMessage { + message: Message::Prepare(100_001), + sender: i, + signature: true, + } + }); + + let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); + + ::std::thread::spawn(move || { + ::std::thread::sleep(::std::time::Duration::from_secs(5)); + timeout_tx.send(None).unwrap(); + }); + + let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) + .wait() + .map(|(r, _)| r) + .map_err(|(e, _)| e) + .expect("not to have an error") + .expect("not to fail to agree"); + + assert_eq!(agreed_value.proposal, 100_001); + } + + #[test] + fn never_concludes_on_less_than_2f_prepares_for_local() { + let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); + let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); + let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); + let max_faulty = 3; + + let agreement = agree( + 100_000, + 255, + |_msg| true, i_rx.map_err(|_| ()), o_tx.sink_map_err(|_| ()), max_faulty, @@ -180,6 +290,7 @@ mod tests { LocalizedMessage { message: Message::Prepare(100_000), sender: i, + signature: true, } }); @@ -200,7 +311,7 @@ mod tests { } #[test] - fn never_concludes_on_2f_prepares_for_different_proposal() { + fn never_concludes_on_less_than_2f_plus_one_prepares_for_alternate() { let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); @@ -208,15 +319,18 @@ mod tests { let agreement = agree( 100_000, + 255, + |_msg| true, i_rx.map_err(|_| ()), o_tx.sink_map_err(|_| ()), max_faulty, ); - let iter = (0..(max_faulty * 2)).map(|i| { + let iter = (1..(max_faulty * 2 + 1)).map(|i| { LocalizedMessage { message: Message::Prepare(100_001), sender: i, + signature: true, } }); From b617079df86bf5016ed33f39b24f28ee15d38a9d Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 18 Dec 2017 16:07:55 +0100 Subject: [PATCH 19/40] check justification set for BFT --- candidate-agreement/src/bft.rs | 69 +++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/candidate-agreement/src/bft.rs b/candidate-agreement/src/bft.rs index 1cc9158127fc4..ae240d524f8df 100644 --- a/candidate-agreement/src/bft.rs +++ b/candidate-agreement/src/bft.rs @@ -20,14 +20,14 @@ //! known by each node. The proposals they have may differ, so the agreement //! may never complete. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::hash::Hash; use futures::{Future, Stream, Sink}; use futures::future::{ok, loop_fn, Loop}; /// Messages over the proposal. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Message

{ /// Prepare to vote for proposal P. Prepare(P), @@ -53,6 +53,51 @@ pub struct Agreed { pub justification: Vec>, } +/// Check validity and compactness justification set for a proposal. +/// +/// Validity checks whether the set of signed messages is enough to justify +/// the agreement of the proposal by the validators. +/// +/// Compactness enforces that no extraneous messages are included. +/// +/// Provide the proposal, the justification set to check, and a closure for +/// extracting validator IDs from signatures. Should return true only if the +/// signature is valid and the signer was a validator at that time. +pub fn check_justification( + proposal: P, + justification: &[LocalizedMessage], + max_faulty: usize, + check_sig: C, +) -> bool + where + P: Eq, + V: Hash + Eq, + C: Fn(&Message

, &S) -> Option +{ + let mut prepared = HashSet::new(); + + for message in justification { + let signer = match check_sig(&message.message, &message.signature) { + Some(signer) => signer, + None => return false, // compactness. + }; + + if signer != message.sender { return false } + + match message.message { + Message::Prepare(ref p) if p == &proposal => {}, + _ => return false, + }; + + // compactness + if !prepared.insert(signer) { return false } + + if prepared.len() > max_faulty * 2 { return true } + } + + false +} + /// Reach BFT agreement. Input the local proposal, message input stream, message output stream, /// and maximum number of faulty participants. /// @@ -198,7 +243,7 @@ mod tests { let agreement = agree( 100_000, 255, - |_msg| true, + |msg| (msg.clone(), 255), i_rx.map_err(|_| ()), o_tx.sink_map_err(|_| ()), max_faulty, @@ -208,7 +253,7 @@ mod tests { LocalizedMessage { message: Message::Prepare(100_000), sender: i, - signature: true, + signature: (Message::Prepare(100_000), i), } }); @@ -227,6 +272,12 @@ mod tests { .expect("not to fail to agree"); assert_eq!(agreed_value.proposal, 100_000); + assert!(check_justification( + agreed_value.proposal, + &agreed_value.justification, + max_faulty, + |msg, sig| if msg == &sig.0 { Some(sig.1) } else { None } + )); } #[test] @@ -239,7 +290,7 @@ mod tests { let agreement = agree( 100_000, 255, - |_msg| true, + |msg| (msg.clone(), 255), i_rx.map_err(|_| ()), o_tx.sink_map_err(|_| ()), max_faulty, @@ -249,7 +300,7 @@ mod tests { LocalizedMessage { message: Message::Prepare(100_001), sender: i, - signature: true, + signature: (Message::Prepare(100_001), i), } }); @@ -268,6 +319,12 @@ mod tests { .expect("not to fail to agree"); assert_eq!(agreed_value.proposal, 100_001); + assert!(check_justification( + agreed_value.proposal, + &agreed_value.justification, + max_faulty, + |msg, sig| if msg == &sig.0 { Some(sig.1) } else { None } + )); } #[test] From e3a046db9dbb25e1fdbe0129b3f60c0c049f2feb Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 21 Dec 2017 22:53:17 +0100 Subject: [PATCH 20/40] BFT rewrite: vote accumulator with tests --- candidate-agreement/src/bft.rs | 409 ---------------- candidate-agreement/src/bft/accumulator.rs | 528 +++++++++++++++++++++ candidate-agreement/src/bft/mod.rs | 100 ++++ candidate-agreement/src/table.rs | 86 ++-- 4 files changed, 671 insertions(+), 452 deletions(-) delete mode 100644 candidate-agreement/src/bft.rs create mode 100644 candidate-agreement/src/bft/accumulator.rs create mode 100644 candidate-agreement/src/bft/mod.rs diff --git a/candidate-agreement/src/bft.rs b/candidate-agreement/src/bft.rs deleted file mode 100644 index ae240d524f8df..0000000000000 --- a/candidate-agreement/src/bft.rs +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2017 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! BFT Agreement based on a proposal. -//! -//! This is based off of PBFT with an assumption that a proposal is already -//! known by each node. The proposals they have may differ, so the agreement -//! may never complete. - -use std::collections::{HashMap, HashSet}; -use std::hash::Hash; - -use futures::{Future, Stream, Sink}; -use futures::future::{ok, loop_fn, Loop}; - -/// Messages over the proposal. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Message

{ - /// Prepare to vote for proposal P. - Prepare(P), -} - -/// A localized message, including the sender. -#[derive(Debug, Clone)] -pub struct LocalizedMessage { - /// The message received. - pub message: Message

, - /// The sender of the message - pub sender: V, - /// The signature of the message. - pub signature: S, -} - -/// The agreed-upon data. -#[derive(Debug, Clone)] -pub struct Agreed { - /// The agreed-upon proposal. - pub proposal: P, - /// The justification for the proposal. - pub justification: Vec>, -} - -/// Check validity and compactness justification set for a proposal. -/// -/// Validity checks whether the set of signed messages is enough to justify -/// the agreement of the proposal by the validators. -/// -/// Compactness enforces that no extraneous messages are included. -/// -/// Provide the proposal, the justification set to check, and a closure for -/// extracting validator IDs from signatures. Should return true only if the -/// signature is valid and the signer was a validator at that time. -pub fn check_justification( - proposal: P, - justification: &[LocalizedMessage], - max_faulty: usize, - check_sig: C, -) -> bool - where - P: Eq, - V: Hash + Eq, - C: Fn(&Message

, &S) -> Option -{ - let mut prepared = HashSet::new(); - - for message in justification { - let signer = match check_sig(&message.message, &message.signature) { - Some(signer) => signer, - None => return false, // compactness. - }; - - if signer != message.sender { return false } - - match message.message { - Message::Prepare(ref p) if p == &proposal => {}, - _ => return false, - }; - - // compactness - if !prepared.insert(signer) { return false } - - if prepared.len() > max_faulty * 2 { return true } - } - - false -} - -/// Reach BFT agreement. Input the local proposal, message input stream, message output stream, -/// and maximum number of faulty participants. -/// -/// Messages should only be yielded from the input stream if the sender is authorized -/// to send messages. -/// -/// The input stream also may never conclude or the agreement code will panic. -/// Duplicate messages are allowed. -/// -/// The output stream assumes that messages will eventually be delivered to all -/// honest participants, either by repropagation, gossip, or some reliable -/// broadcast mechanism. -/// -/// This will collect 2f + 1 "prepare" messages. Since this is all within a single -/// view, the commit phase is not necessary. -// TODO: consider cross-view committing -// TODO: impl future. -pub fn agree<'a, P, V, S, F, I, O>( - local_proposal: P, - local_id: V, - mut sign_local: F, - input: I, - output: O, - max_faulty: usize, -) -> Box, Error=I::Error> + Send + 'a> - where - P: 'a + Send + Hash + Eq + Clone, - V: 'a + Send + Hash + Eq + Clone, - S: 'a + Send + Eq + Clone, - F: 'a + Send + FnMut(&Message

) -> S, - I: 'a + Send + Stream>, - O: 'a + Send + Sink,SinkError=I::Error>, - I::Error: Send -{ - use std::collections::hash_map::Entry; - - let voting_for = HashMap::new(); - let prepared = HashMap::new(); - - let local_prepare = { - let local_prepare = Message::Prepare(local_proposal); - let local_signature = sign_local(&local_prepare); - - LocalizedMessage { - message: local_prepare, - sender: local_id, - signature: local_signature, - } - }; - - // broadcast out our local prepare message and shortcut it into our input - // stream. - let broadcast_message = output.send(local_prepare.clone()); - let input = ::futures::stream::once(Ok(local_prepare)).chain(input); - - let wait_for_prepares = loop_fn((input, voting_for, prepared), move |(input, mut voting_for, mut prepared)| { - input.into_future().and_then(move |(msg, remainder)| { - let msg = msg.expect("input stream never concludes; qed"); - let LocalizedMessage { message: Message::Prepare(p), sender, signature } = msg; - - let is_complete = match voting_for.entry(sender) { - Entry::Occupied(_) => { - // TODO: handle double vote. - false - } - Entry::Vacant(vacant) => { - vacant.insert((p.clone(), signature)); - let n = prepared.entry(p.clone()).or_insert(0); - *n += 1; - *n > max_faulty * 2 - } - }; - - if is_complete { - let justification = voting_for.into_iter().filter_map(|(v, (x, s))| { - if x == p { - Some(LocalizedMessage { - message: Message::Prepare(x), - sender: v, - signature: s, - }) - } else { - None - } - }).collect(); - - ok(Loop::Break(Agreed { - justification, - proposal: p, - })) - } else { - ok(Loop::Continue((remainder, voting_for, prepared))) - } - - }).map_err(|(e, _)| e) - }); - - Box::new(broadcast_message.and_then(move |_| wait_for_prepares)) -} - -#[cfg(test)] -mod tests { - use futures::{Future, Stream, Sink}; - use super::*; - - #[test] - fn broadcasts_message() { - let (i_tx, i_rx) = ::futures::sync::mpsc::channel::>(10); - let (o_tx, o_rx) = ::futures::sync::mpsc::channel(10); - let max_faulty = 3; - - let agreement = agree( - 100_000, - 255, - |_msg| true, - i_rx.map_err(|_| ()), - o_tx.sink_map_err(|_| ()), - max_faulty, - ); - - ::std::thread::spawn(move || { - let _i_tx = i_tx; - let _ = agreement.wait(); - }); - - let sent_message = o_rx.wait() - .next() - .expect("to have a next item") - .expect("not to have an error"); - - let Message::Prepare(p) = sent_message.message; - assert_eq!(p, 100_000); - assert_eq!(sent_message.sender, 255); - } - - #[test] - fn concludes_on_2f_prepares_for_local_proposal() { - let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); - let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); - let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); - let max_faulty = 3; - - let agreement = agree( - 100_000, - 255, - |msg| (msg.clone(), 255), - i_rx.map_err(|_| ()), - o_tx.sink_map_err(|_| ()), - max_faulty, - ); - - let iter = (0..(max_faulty * 2)).map(|i| { - LocalizedMessage { - message: Message::Prepare(100_000), - sender: i, - signature: (Message::Prepare(100_000), i), - } - }); - - let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); - - ::std::thread::spawn(move || { - ::std::thread::sleep(::std::time::Duration::from_secs(5)); - timeout_tx.send(None).unwrap(); - }); - - let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) - .wait() - .map(|(r, _)| r) - .map_err(|(e, _)| e) - .expect("not to have an error") - .expect("not to fail to agree"); - - assert_eq!(agreed_value.proposal, 100_000); - assert!(check_justification( - agreed_value.proposal, - &agreed_value.justification, - max_faulty, - |msg, sig| if msg == &sig.0 { Some(sig.1) } else { None } - )); - } - - #[test] - fn concludes_on_2f_plus_one_prepares_for_alternate_proposal() { - let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); - let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); - let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); - let max_faulty = 3; - - let agreement = agree( - 100_000, - 255, - |msg| (msg.clone(), 255), - i_rx.map_err(|_| ()), - o_tx.sink_map_err(|_| ()), - max_faulty, - ); - - let iter = (0..(max_faulty * 2 + 1)).map(|i| { - LocalizedMessage { - message: Message::Prepare(100_001), - sender: i, - signature: (Message::Prepare(100_001), i), - } - }); - - let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); - - ::std::thread::spawn(move || { - ::std::thread::sleep(::std::time::Duration::from_secs(5)); - timeout_tx.send(None).unwrap(); - }); - - let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) - .wait() - .map(|(r, _)| r) - .map_err(|(e, _)| e) - .expect("not to have an error") - .expect("not to fail to agree"); - - assert_eq!(agreed_value.proposal, 100_001); - assert!(check_justification( - agreed_value.proposal, - &agreed_value.justification, - max_faulty, - |msg, sig| if msg == &sig.0 { Some(sig.1) } else { None } - )); - } - - #[test] - fn never_concludes_on_less_than_2f_prepares_for_local() { - let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); - let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); - let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); - let max_faulty = 3; - - let agreement = agree( - 100_000, - 255, - |_msg| true, - i_rx.map_err(|_| ()), - o_tx.sink_map_err(|_| ()), - max_faulty, - ); - - let iter = (1..(max_faulty * 2)).map(|i| { - LocalizedMessage { - message: Message::Prepare(100_000), - sender: i, - signature: true, - } - }); - - let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); - - ::std::thread::spawn(move || { - ::std::thread::sleep(::std::time::Duration::from_millis(250)); - timeout_tx.send(None).unwrap(); - }); - - let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) - .wait() - .map(|(r, _)| r) - .map_err(|(e, _)| e) - .expect("not to have an error"); - - assert!(agreed_value.is_none()); - } - - #[test] - fn never_concludes_on_less_than_2f_plus_one_prepares_for_alternate() { - let (i_tx, i_rx) = ::futures::sync::mpsc::channel(10); - let (o_tx, _o_rx) = ::futures::sync::mpsc::channel(10); - let (timeout_tx, timeout_rx) = ::futures::sync::oneshot::channel(); - let max_faulty = 3; - - let agreement = agree( - 100_000, - 255, - |_msg| true, - i_rx.map_err(|_| ()), - o_tx.sink_map_err(|_| ()), - max_faulty, - ); - - let iter = (1..(max_faulty * 2 + 1)).map(|i| { - LocalizedMessage { - message: Message::Prepare(100_001), - sender: i, - signature: true, - } - }); - - let (_i_tx, _) = i_tx.send_all(::futures::stream::iter_ok(iter)).wait().unwrap(); - - ::std::thread::spawn(move || { - ::std::thread::sleep(::std::time::Duration::from_millis(250)); - timeout_tx.send(None).unwrap(); - }); - - let agreed_value = agreement.map(Some).select(timeout_rx.map_err(|_| ())) - .wait() - .map(|(r, _)| r) - .map_err(|(e, _)| e) - .expect("not to have an error"); - - assert!(agreed_value.is_none()); - } -} diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs new file mode 100644 index 0000000000000..69d6329f9b589 --- /dev/null +++ b/candidate-agreement/src/bft/accumulator.rs @@ -0,0 +1,528 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Message accumulator for each round of BFT consensus. + +use std::collections::{HashMap, HashSet}; +use std::collections::hash_map::Entry; +use std::hash::Hash; + +use super::{Message, LocalizedMessage}; + +/// Context necessary to execute a round of BFT. +pub trait Context { + /// A full candidate. + type Candidate: Clone; + /// Unique digest of a proposed candidate (think hash). + type Digest: Hash + Eq + Clone; + /// Validator ID. + type ValidatorId: Hash + Eq; + /// A signature. + type Signature: Eq + Clone; +} + +/// Justification at a given round. +#[derive(PartialEq, Eq, Debug)] +pub struct Justification { + /// The round. + pub round_number: usize, + /// The digest prepared for. + pub digest: D, + /// Signatures for the prepare messages. + pub signatures: Vec, +} + +impl Justification { + /// Fails if there are duplicate signatures or invalid. + /// + /// Provide a closure for checking whether the signature is valid on a + /// digest. + /// + /// The closure should return true iff the round number, digest, and signature + /// represent a valid prepare message and the signer was authorized to issue + /// it. + pub fn check(&self, max_faulty: usize, check_message: F) -> bool + where + F: Fn(usize, &D, &S) -> Option, + V: Hash + Eq, + { + let mut prepared = HashSet::new(); + + let mut good = false; + for signature in &self.signatures { + match check_message(self.round_number, &self.digest, signature) { + None => return false, + Some(v) => { + if !prepared.insert(v) { + return false; + } else if prepared.len() > max_faulty * 2 { + // don't return just yet since later signatures may be invalid. + good = true; + } + } + } + } + + good + } +} + +/// Type alias to represent a justification specifically for a prepare. +pub type PrepareJustification = Justification; + +/// The round's state, based on imported messages. +#[derive(PartialEq, Eq, Debug)] +pub enum State { + /// No proposal yet. + Begin, + /// Proposal received. + Proposed(C), + /// Seen 2f + 1 prepares for this digest. + Prepared(PrepareJustification), + /// Seen 2f + 1 commits for a digest. + Concluded(Justification), + /// Seen 2f + 1 round-advancement messages. + Advanced(Option>), +} + +/// Accumulates messages for a given round of BFT consensus. +pub struct Accumulator { + round_number: usize, + max_faulty: usize, + round_proposer: V, + proposal: Option, + prepares: HashMap, + commits: HashMap, + vote_counts: HashMap, + advance_round: HashSet, + state: State, +} + +impl Accumulator + where + C: Eq + Clone, + D: Hash + Clone + Eq, + V: Hash + Eq, + S: Eq + Clone, +{ + /// Create a new state accumulator. + pub fn new(round_number: usize, max_faulty: usize, round_proposer: V) -> Self { + Accumulator { + round_number, + max_faulty, + round_proposer, + proposal: None, + prepares: HashMap::new(), + commits: HashMap::new(), + vote_counts: HashMap::new(), + advance_round: HashSet::new(), + state: State::Begin, + } + } + + /// How advance votes we have seen. + pub fn advance_votes(&self) -> usize { + self.advance_round.len() + } + + /// Inspect the current consensus state. + pub fn state(&self) -> &State { + &self.state + } + + /// Import a message. Importing duplicates is fine, but the signature + /// and authorization should have already been checked. + pub fn import_message( + &mut self, + message: LocalizedMessage, + ) + { + // old message. + if message.message.round_number() != self.round_number { + return; + } + + let (sender, signature) = (message.sender, message.signature); + + match message.message { + Message::Propose(_, p) => self.import_proposal(p, sender, signature), + Message::Prepare(_, d) => self.import_prepare(d, sender, signature), + Message::Commit(_, d) => self.import_commit(d, sender, signature), + Message::AdvanceRound(_) => self.import_advance_round(sender), + } + } + + fn import_proposal( + &mut self, + proposal: C, + sender: V, + signature: S, + ) { + if sender != self.round_proposer || self.proposal.is_some() { return } + + self.proposal = Some(proposal.clone()); + self.state = State::Proposed(proposal); + } + + fn import_prepare( + &mut self, + candidate: D, + sender: V, + signature: S, + ) { + // ignore any subsequent prepares by the same sender. + // TODO: if digest is different, that's misbehavior. + let prepared_for = if let Entry::Vacant(vacant) = self.prepares.entry(sender) { + vacant.insert((candidate.clone(), signature)); + let count = self.vote_counts.entry(candidate.clone()).or_insert((0, 0)); + count.0 += 1; + + if count.0 == self.max_faulty * 2 + 1 { + Some(candidate) + } else { + None + } + } else { + None + }; + + // only allow transition to prepare from begin or proposed state. + let valid_transition = match self.state { + State::Begin | State::Proposed(_) => true, + _ => false, + }; + + if let (true, Some(prepared_for)) = (valid_transition, prepared_for) { + let signatures = self.prepares + .values() + .filter(|&&(ref d, _)| d == &prepared_for) + .map(|&(_, ref s)| s.clone()) + .collect(); + + self.state = State::Prepared(PrepareJustification { + round_number: self.round_number, + digest: prepared_for, + signatures: signatures, + }); + } + } + + fn import_commit( + &mut self, + candidate: D, + sender: V, + signature: S, + ) { + // ignore any subsequent commits by the same sender. + // TODO: if digest is different, that's misbehavior. + let committed_for = if let Entry::Vacant(vacant) = self.commits.entry(sender) { + vacant.insert((candidate.clone(), signature)); + let count = self.vote_counts.entry(candidate.clone()).or_insert((0, 0)); + count.1 += 1; + + if count.1 == self.max_faulty * 2 + 1 { + Some(candidate) + } else { + None + } + } else { + None + }; + + // transition to concluded state always valid. + // only weird case is if the prior state was "advanced", + // but technically it's the same behavior as if the order of receiving + // the last "advance round" and "commit" messages were reversed. + if let Some(committed_for) = committed_for { + let signatures = self.commits + .values() + .filter(|&&(ref d, _)| d == &committed_for) + .map(|&(_, ref s)| s.clone()) + .collect(); + + self.state = State::Concluded(Justification { + round_number: self.round_number, + digest: committed_for, + signatures: signatures, + }); + } + } + + fn import_advance_round( + &mut self, + sender: V, + ) { + self.advance_round.insert(sender); + + if self.advance_round.len() != self.max_faulty * 2 + 1 { return } + + // allow transition to new round only if we haven't produced a justification + // yet. + self.state = match ::std::mem::replace(&mut self.state, State::Begin) { + State::Concluded(j) => State::Concluded(j), + State::Prepared(j) => State::Advanced(Some(j)), + State::Advanced(j) => State::Advanced(j), + State::Begin | State::Proposed(_) => State::Advanced(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Clone, PartialEq, Eq, Debug)] + pub struct Candidate(usize); + + #[derive(Hash, PartialEq, Eq, Clone, Debug)] + pub struct Digest(usize); + + #[derive(Hash, PartialEq, Eq, Debug)] + pub struct ValidatorId(usize); + + #[derive(PartialEq, Eq, Clone, Debug)] + pub struct Signature(usize, usize); + + #[test] + fn justification_checks_out() { + let mut justification = Justification { + round_number: 2, + digest: Digest(600), + signatures: (0..10).map(|i| Signature(600, i)).collect(), + }; + + let check_message = |r, d: &Digest, s: &Signature| { + if r == 2 && d.0 == 600 && s.0 == 600 { + Some(ValidatorId(s.1)) + } else { + None + } + }; + + assert!(justification.check(3, &check_message)); + assert!(!justification.check(5, &check_message)); + + { + // one bad signature is enough to spoil it. + justification.signatures.push(Signature(1001, 255)); + assert!(!justification.check(3, &check_message)); + + justification.signatures.pop(); + } + // duplicates not allowed. + justification.signatures.extend((0..10).map(|i| Signature(600, i))); + assert!(!justification.check(3, &check_message)); + } + + #[test] + fn accepts_proposal_from_proposer_only() { + let mut accumulator = Accumulator::<_, Digest, _, _>::new(1, 3, ValidatorId(8)); + assert_eq!(accumulator.state(), &State::Begin); + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(5), + signature: Signature(999, 5), + message: Message::Propose(1, Candidate(999)), + }); + + assert_eq!(accumulator.state(), &State::Begin); + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(8), + signature: Signature(999, 8), + message: Message::Propose(1, Candidate(999)), + }); + + assert_eq!(accumulator.state(), &State::Proposed(Candidate(999))); + } + + #[test] + fn reaches_prepare_phase() { + let mut accumulator = Accumulator::new(1, 3, ValidatorId(8)); + assert_eq!(accumulator.state(), &State::Begin); + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(8), + signature: Signature(999, 8), + message: Message::Propose(1, Candidate(999)), + }); + + assert_eq!(accumulator.state(), &State::Proposed(Candidate(999))); + + for i in 0..6 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::Prepare(1, Digest(999)), + }); + + assert_eq!(accumulator.state(), &State::Proposed(Candidate(999))); + } + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(7), + signature: Signature(999, 7), + message: Message::Prepare(1, Digest(999)), + }); + + match accumulator.state() { + &State::Prepared(ref j) => assert_eq!(j.digest, Digest(999)), + s => panic!("wrong state: {:?}", s), + } + } + + #[test] + fn prepare_to_commit() { + let mut accumulator = Accumulator::new(1, 3, ValidatorId(8)); + assert_eq!(accumulator.state(), &State::Begin); + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(8), + signature: Signature(999, 8), + message: Message::Propose(1, Candidate(999)), + }); + + assert_eq!(accumulator.state(), &State::Proposed(Candidate(999))); + + for i in 0..6 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::Prepare(1, Digest(999)), + }); + + assert_eq!(accumulator.state(), &State::Proposed(Candidate(999))); + } + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(7), + signature: Signature(999, 7), + message: Message::Prepare(1, Digest(999)), + }); + + match accumulator.state() { + &State::Prepared(ref j) => assert_eq!(j.digest, Digest(999)), + s => panic!("wrong state: {:?}", s), + } + + for i in 0..6 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::Commit(1, Digest(999)), + }); + + match accumulator.state() { + &State::Prepared(_) => {}, + s => panic!("wrong state: {:?}", s), + } + } + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(7), + signature: Signature(999, 7), + message: Message::Commit(1, Digest(999)), + }); + + match accumulator.state() { + &State::Concluded(ref j) => assert_eq!(j.digest, Digest(999)), + s => panic!("wrong state: {:?}", s), + } + } + + #[test] + fn prepare_to_advance() { + let mut accumulator = Accumulator::new(1, 3, ValidatorId(8)); + assert_eq!(accumulator.state(), &State::Begin); + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(8), + signature: Signature(999, 8), + message: Message::Propose(1, Candidate(999)), + }); + + assert_eq!(accumulator.state(), &State::Proposed(Candidate(999))); + + for i in 0..7 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::Prepare(1, Digest(999)), + }); + } + + match accumulator.state() { + &State::Prepared(ref j) => assert_eq!(j.digest, Digest(999)), + s => panic!("wrong state: {:?}", s), + } + + for i in 0..6 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::AdvanceRound(1), + }); + + match accumulator.state() { + &State::Prepared(_) => {}, + s => panic!("wrong state: {:?}", s), + } + } + + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(7), + signature: Signature(999, 7), + message: Message::AdvanceRound(1), + }); + + match accumulator.state() { + &State::Advanced(Some(_)) => {}, + s => panic!("wrong state: {:?}", s), + } + } + + #[test] + fn conclude_different_than_proposed() { + let mut accumulator = Accumulator::::new(1, 3, ValidatorId(8)); + assert_eq!(accumulator.state(), &State::Begin); + + for i in 0..7 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::Prepare(1, Digest(999)), + }); + } + + match accumulator.state() { + &State::Prepared(ref j) => assert_eq!(j.digest, Digest(999)), + s => panic!("wrong state: {:?}", s), + } + + for i in 0..7 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::Commit(1, Digest(999)), + }); + } + + match accumulator.state() { + &State::Concluded(ref j) => assert_eq!(j.digest, Digest(999)), + s => panic!("wrong state: {:?}", s), + } + } +} diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs new file mode 100644 index 0000000000000..da4b6fe91da1b --- /dev/null +++ b/candidate-agreement/src/bft/mod.rs @@ -0,0 +1,100 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! BFT Agreement based on a rotating proposer in different rounds. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +use futures::{IntoFuture, Future, Stream, Sink}; +use futures::future::{ok, loop_fn, Loop}; + +mod accumulator; + +/// Messages over the proposal. +/// Each message carries an associated round number. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Message { + /// Send a full proposal. + Propose(usize, P), + /// Prepare to vote for proposal with digest D. + Prepare(usize, D), + /// Commit to proposal with digest D.. + Commit(usize, D), + /// Propose advancement to a new round. + AdvanceRound(usize), +} + +impl Message { + fn round_number(&self) -> usize { + match *self { + Message::Propose(round, _) => round, + Message::Prepare(round, _) => round, + Message::Commit(round, _) => round, + Message::AdvanceRound(round) => round, + } + } +} + +/// A localized message, including the sender. +#[derive(Debug, Clone)] +pub struct LocalizedMessage { + /// The message received. + pub message: Message, + /// The sender of the message + pub sender: V, + /// The signature of the message. + pub signature: S, +} + +/// The agreed-upon data. +#[derive(Debug, Clone)] +pub struct Agreed { + /// The agreed-upon proposal. + pub proposal: P, + /// The justification for the proposal. + pub justification: Vec>, +} + +/// Parameters to agreement. +pub struct Params< + Validator, + SignLocal, + Timeout, + CanInclude, + MessagesIn, + MessagesOut, +> { + /// The ID of the current view's primary. + pub primary: Validator, + /// The local ID. + pub local_id: Validator, + /// A closure for signing local messages. + pub sign_local: SignLocal, + /// A timeout that fires when the view change should begin. + pub begin_view_change: Timeout, + /// A function for checking if a proposal can be voted for. + pub can_include: CanInclude, + /// The input stream. Should never conclude, and should yield only messages + /// sent by validators and which have been authenticated properly. + pub input: MessagesIn, + /// The output message sink. This assumes that messages will eventually + /// be delivered to all honest participants, either by repropagation, gossip, + /// or some reliable broadcast mechanism. + pub output: MessagesOut, + /// The maximum number of faulty nodes. + pub max_faulty: usize, +} diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index c072be3ee0c1a..68025c3708946 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -32,6 +32,49 @@ use std::collections::hash_map::{HashMap, Entry}; use std::hash::Hash; use std::fmt::Debug; +/// Context for the statement table. +pub trait Context { + /// A validator ID + type ValidatorId: Hash + Eq + Clone + Debug; + /// The digest (hash or other unique attribute) of a candidate. + type Digest: Hash + Eq + Clone + Debug; + /// Candidate type. + type Candidate: Ord + Eq + Clone + Debug; + /// The group ID type + type GroupId: Hash + Ord + Eq + Clone + Debug; + /// A signature type. + type Signature: Eq + Clone + Debug; + + /// get the digest of a candidate. + fn candidate_digest(&self, candidate: &Self::Candidate) -> Self::Digest; + + /// get the group of a candidate. + fn candidate_group(&self, candidate: &Self::Candidate) -> Self::GroupId; + + /// Whether a validator is a member of a group. + /// Members are meant to submit candidates and vote on validity. + fn is_member_of(&self, validator: &Self::ValidatorId, group: &Self::GroupId) -> bool; + + /// Whether a validator is an availability guarantor of a group. + /// Guarantors are meant to vote on availability for candidates submitted + /// in a group. + fn is_availability_guarantor_of( + &self, + validator: &Self::ValidatorId, + group: &Self::GroupId, + ) -> bool; + + // recover signer of statement and ensure the signature corresponds to the + // statement. + fn statement_signer( + &self, + statement: &SignedStatement, + ) -> Option; + + // requisite number of votes for validity and availability respectively from a group. + fn requisite_votes(&self, group: &Self::GroupId) -> (usize, usize); +} + /// Statements circulated among peers. #[derive(PartialEq, Eq, Debug)] pub enum Statement { @@ -80,49 +123,6 @@ enum StatementTrace { Available(V, D), } -/// Context for the statement table. -pub trait Context { - /// A validator ID - type ValidatorId: Hash + Eq + Clone + Debug; - /// The digest (hash or other unique attribute) of a candidate. - type Digest: Hash + Eq + Clone + Debug; - /// Candidate type. - type Candidate: Ord + Eq + Clone + Debug; - /// The group ID type - type GroupId: Hash + Ord + Eq + Clone + Debug; - /// A signature type. - type Signature: Eq + Clone + Debug; - - /// get the digest of a candidate. - fn candidate_digest(&self, candidate: &Self::Candidate) -> Self::Digest; - - /// get the group of a candidate. - fn candidate_group(&self, candidate: &Self::Candidate) -> Self::GroupId; - - /// Whether a validator is a member of a group. - /// Members are meant to submit candidates and vote on validity. - fn is_member_of(&self, validator: &Self::ValidatorId, group: &Self::GroupId) -> bool; - - /// Whether a validator is an availability guarantor of a group. - /// Guarantors are meant to vote on availability for candidates submitted - /// in a group. - fn is_availability_guarantor_of( - &self, - validator: &Self::ValidatorId, - group: &Self::GroupId, - ) -> bool; - - // recover signer of statement and ensure the signature corresponds to the - // statement. - fn statement_signer( - &self, - statement: &SignedStatement, - ) -> Option; - - // requisite number of votes for validity and availability respectively from a group. - fn requisite_votes(&self, group: &Self::GroupId) -> (usize, usize); -} - /// Misbehavior: voting more than one way on candidate validity. /// /// Since there are three possible ways to vote, a double vote is possible in From 02ac220c3bf574f9aba0b816a2f5873707422700 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 21 Dec 2017 22:54:51 +0100 Subject: [PATCH 21/40] squash some warnings --- candidate-agreement/src/bft/accumulator.rs | 3 +-- candidate-agreement/src/bft/mod.rs | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs index 69d6329f9b589..a0c99698df83a 100644 --- a/candidate-agreement/src/bft/accumulator.rs +++ b/candidate-agreement/src/bft/accumulator.rs @@ -158,7 +158,7 @@ impl Accumulator let (sender, signature) = (message.sender, message.signature); match message.message { - Message::Propose(_, p) => self.import_proposal(p, sender, signature), + Message::Propose(_, p) => self.import_proposal(p, sender), Message::Prepare(_, d) => self.import_prepare(d, sender, signature), Message::Commit(_, d) => self.import_commit(d, sender, signature), Message::AdvanceRound(_) => self.import_advance_round(sender), @@ -169,7 +169,6 @@ impl Accumulator &mut self, proposal: C, sender: V, - signature: S, ) { if sender != self.round_proposer || self.proposal.is_some() { return } diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index da4b6fe91da1b..74476b98b02d3 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -16,12 +16,6 @@ //! BFT Agreement based on a rotating proposer in different rounds. -use std::collections::{HashMap, HashSet}; -use std::hash::Hash; - -use futures::{IntoFuture, Future, Stream, Sink}; -use futures::future::{ok, loop_fn, Loop}; - mod accumulator; /// Messages over the proposal. From a93b231b7c27e1fdcb239c6e2c27eb78376eada7 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 22 Dec 2017 17:11:21 +0100 Subject: [PATCH 22/40] a few more tests for the accumulator --- candidate-agreement/src/bft/accumulator.rs | 38 ++++++++++++++++++++++ candidate-agreement/src/bft/mod.rs | 3 -- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs index a0c99698df83a..ce54c55ea1649 100644 --- a/candidate-agreement/src/bft/accumulator.rs +++ b/candidate-agreement/src/bft/accumulator.rs @@ -524,4 +524,42 @@ mod tests { s => panic!("wrong state: {:?}", s), } } + + #[test] + fn begin_to_advance() { + let mut accumulator = Accumulator::::new(1, 3, ValidatorId(8)); + assert_eq!(accumulator.state(), &State::Begin); + + for i in 0..7 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(1, i), + message: Message::AdvanceRound(1), + }); + } + + match accumulator.state() { + &State::Advanced(ref j) => assert!(j.is_none()), + s => panic!("wrong state: {:?}", s), + } + } + + #[test] + fn conclude_without_prepare() { + let mut accumulator = Accumulator::::new(1, 3, ValidatorId(8)); + assert_eq!(accumulator.state(), &State::Begin); + + for i in 0..7 { + accumulator.import_message(LocalizedMessage { + sender: ValidatorId(i), + signature: Signature(999, i), + message: Message::Commit(1, Digest(999)), + }); + } + + match accumulator.state() { + &State::Concluded(ref j) => assert_eq!(j.digest, Digest(999)), + s => panic!("wrong state: {:?}", s), + } + } } diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index 74476b98b02d3..bcb35718428ba 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -67,7 +67,6 @@ pub struct Agreed { pub struct Params< Validator, SignLocal, - Timeout, CanInclude, MessagesIn, MessagesOut, @@ -78,8 +77,6 @@ pub struct Params< pub local_id: Validator, /// A closure for signing local messages. pub sign_local: SignLocal, - /// A timeout that fires when the view change should begin. - pub begin_view_change: Timeout, /// A function for checking if a proposal can be voted for. pub can_include: CanInclude, /// The input stream. Should never conclude, and should yield only messages From a16bfd28474c058baefc50681099cd7590c8932e Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 22 Dec 2017 17:42:10 +0100 Subject: [PATCH 23/40] add sender to table's signed statement --- candidate-agreement/src/table.rs | 59 ++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 68025c3708946..764f3a77dd5e7 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -64,13 +64,6 @@ pub trait Context { group: &Self::GroupId, ) -> bool; - // recover signer of statement and ensure the signature corresponds to the - // statement. - fn statement_signer( - &self, - statement: &SignedStatement, - ) -> Option; - // requisite number of votes for validity and availability respectively from a group. fn requisite_votes(&self, group: &Self::GroupId) -> (usize, usize); } @@ -101,6 +94,8 @@ pub struct SignedStatement { pub statement: Statement, /// The signature. pub signature: C::Signature, + /// The sender. + pub sender: C::ValidatorId, } // A unique trace for a class of valid statements issued by a validator. @@ -303,49 +298,47 @@ impl Table { ::std::mem::replace(&mut self.detected_misbehavior, HashMap::new()) } - /// Import a signed statement. + /// Import a signed statement. Signatures should be checked for validity, and the + /// sender should be checked to actually be a validator. /// /// This can note the origin of the statement to indicate that he has /// seen it already. pub fn import_statement(&mut self, context: &C, statement: SignedStatement, from: Option) -> Option> { - let signer = match context.statement_signer(&statement) { - None => return None, - Some(signer) => signer, - }; + let SignedStatement { statement, signature, sender: signer } = statement; - let trace = match statement.statement { + let trace = match statement { Statement::Candidate(_) => StatementTrace::Candidate(signer.clone()), Statement::Valid(ref d) => StatementTrace::Valid(signer.clone(), d.clone()), Statement::Invalid(ref d) => StatementTrace::Invalid(signer.clone(), d.clone()), Statement::Available(ref d) => StatementTrace::Available(signer.clone(), d.clone()), }; - let (maybe_misbehavior, maybe_summary) = match statement.statement { + let (maybe_misbehavior, maybe_summary) = match statement { Statement::Candidate(candidate) => self.import_candidate( context, signer.clone(), candidate, - statement.signature + signature ), Statement::Valid(digest) => self.validity_vote( context, signer.clone(), digest, - ValidityVote::Valid(statement.signature), + ValidityVote::Valid(signature), ), Statement::Invalid(digest) => self.validity_vote( context, signer.clone(), digest, - ValidityVote::Invalid(statement.signature), + ValidityVote::Invalid(signature), ), Statement::Available(digest) => self.availability_vote( context, signer.clone(), digest, - statement.signature, + signature, ) }; @@ -385,6 +378,7 @@ impl Table { statement: SignedStatement { signature, statement: Statement::Candidate(candidate), + sender: from, }, })), None, @@ -469,6 +463,7 @@ impl Table { Some(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { statement: SignedStatement { signature: sig, + sender: from, statement: if valid { Statement::Valid(digest) } else { @@ -540,6 +535,7 @@ impl Table { statement: SignedStatement { signature: signature.clone(), statement: Statement::Available(digest), + sender: from, } })), None @@ -609,13 +605,6 @@ mod tests { self.validators.get(validator).map(|v| &v.1 == group).unwrap_or(false) } - fn statement_signer( - &self, - statement: &SignedStatement, - ) -> Option { - Some(ValidatorId(statement.signature.0)) - } - fn requisite_votes(&self, _id: &GroupId) -> (usize, usize) { (6, 34) } @@ -635,11 +624,13 @@ mod tests { let statement_a = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; let statement_b = SignedStatement { statement: Statement::Candidate(Candidate(2, 999)), signature: Signature(1), + sender: ValidatorId(1), }; table.import_statement(&context, statement_a, None); @@ -669,6 +660,7 @@ mod tests { let statement = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; table.import_statement(&context, statement, None); @@ -679,6 +671,7 @@ mod tests { statement: SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }, }) ); @@ -700,12 +693,14 @@ mod tests { let candidate_a = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; let candidate_a_digest = Digest(100); let candidate_b = SignedStatement { statement: Statement::Candidate(Candidate(3, 987)), signature: Signature(2), + sender: ValidatorId(2), }; let candidate_b_digest = Digest(987); @@ -718,6 +713,7 @@ mod tests { let bad_availability_vote = SignedStatement { statement: Statement::Available(candidate_b_digest.clone()), signature: Signature(1), + sender: ValidatorId(1), }; table.import_statement(&context, bad_availability_vote, None); @@ -727,6 +723,7 @@ mod tests { statement: SignedStatement { statement: Statement::Available(candidate_b_digest), signature: Signature(1), + sender: ValidatorId(1), }, }) ); @@ -735,6 +732,7 @@ mod tests { let bad_validity_vote = SignedStatement { statement: Statement::Valid(candidate_a_digest.clone()), signature: Signature(2), + sender: ValidatorId(2), }; table.import_statement(&context, bad_validity_vote, None); @@ -744,6 +742,7 @@ mod tests { statement: SignedStatement { statement: Statement::Valid(candidate_a_digest), signature: Signature(2), + sender: ValidatorId(2), }, }) ); @@ -764,6 +763,7 @@ mod tests { let statement = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; let candidate_digest = Digest(100); @@ -773,11 +773,13 @@ mod tests { let valid_statement = SignedStatement { statement: Statement::Valid(candidate_digest.clone()), signature: Signature(2), + sender: ValidatorId(2), }; let invalid_statement = SignedStatement { statement: Statement::Invalid(candidate_digest.clone()), signature: Signature(2), + sender: ValidatorId(2), }; table.import_statement(&context, valid_statement, None); @@ -809,6 +811,7 @@ mod tests { let statement = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; let candidate_digest = Digest(100); @@ -818,6 +821,7 @@ mod tests { let extra_vote = SignedStatement { statement: Statement::Valid(candidate_digest.clone()), signature: Signature(1), + sender: ValidatorId(1), }; table.import_statement(&context, extra_vote, None); @@ -876,6 +880,7 @@ mod tests { let statement = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; let summary = table.import_statement(&context, statement, None) @@ -902,6 +907,7 @@ mod tests { let statement = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; let candidate_digest = Digest(100); @@ -911,6 +917,7 @@ mod tests { let vote = SignedStatement { statement: Statement::Valid(candidate_digest.clone()), signature: Signature(2), + sender: ValidatorId(2), }; let summary = table.import_statement(&context, vote, None) @@ -939,6 +946,7 @@ mod tests { let statement = SignedStatement { statement: Statement::Candidate(Candidate(2, 100)), signature: Signature(1), + sender: ValidatorId(1), }; let candidate_digest = Digest(100); @@ -948,6 +956,7 @@ mod tests { let vote = SignedStatement { statement: Statement::Available(candidate_digest.clone()), signature: Signature(2), + sender: ValidatorId(2), }; let summary = table.import_statement(&context, vote, None) From 28ef13db9388ad667550ccaf6377a9708c341aab Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 27 Dec 2017 17:46:41 +0100 Subject: [PATCH 24/40] implement honest node strategy for BFT --- candidate-agreement/src/bft/accumulator.rs | 81 +-- candidate-agreement/src/bft/mod.rs | 595 +++++++++++++++++++-- candidate-agreement/src/lib.rs | 1 + candidate-agreement/src/table.rs | 74 +-- 4 files changed, 657 insertions(+), 94 deletions(-) diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs index ce54c55ea1649..78f2958bc56c9 100644 --- a/candidate-agreement/src/bft/accumulator.rs +++ b/candidate-agreement/src/bft/accumulator.rs @@ -35,7 +35,7 @@ pub trait Context { } /// Justification at a given round. -#[derive(PartialEq, Eq, Debug)] +#[derive(PartialEq, Eq, Debug, Clone)] pub struct Justification { /// The round. pub round_number: usize, @@ -54,21 +54,21 @@ impl Justification { /// The closure should return true iff the round number, digest, and signature /// represent a valid prepare message and the signer was authorized to issue /// it. - pub fn check(&self, max_faulty: usize, check_message: F) -> bool + pub fn check(&self, threshold: usize, check_message: F) -> bool where F: Fn(usize, &D, &S) -> Option, V: Hash + Eq, { - let mut prepared = HashSet::new(); + let mut voted = HashSet::new(); let mut good = false; for signature in &self.signatures { match check_message(self.round_number, &self.digest, signature) { None => return false, Some(v) => { - if !prepared.insert(v) { + if !voted.insert(v) { return false; - } else if prepared.len() > max_faulty * 2 { + } else if voted.len() >= threshold { // don't return just yet since later signatures may be invalid. good = true; } @@ -93,15 +93,22 @@ pub enum State { /// Seen 2f + 1 prepares for this digest. Prepared(PrepareJustification), /// Seen 2f + 1 commits for a digest. - Concluded(Justification), + Committed(Justification), /// Seen 2f + 1 round-advancement messages. Advanced(Option>), } /// Accumulates messages for a given round of BFT consensus. -pub struct Accumulator { +#[derive(Debug)] +pub struct Accumulator + where + C: Eq + Clone, + D: Hash + Eq + Clone, + V: Hash + Eq, + S: Eq + Clone, +{ round_number: usize, - max_faulty: usize, + threshold: usize, round_proposer: V, proposal: Option, prepares: HashMap, @@ -114,15 +121,15 @@ pub struct Accumulator { impl Accumulator where C: Eq + Clone, - D: Hash + Clone + Eq, + D: Hash + Eq + Clone, V: Hash + Eq, S: Eq + Clone, { /// Create a new state accumulator. - pub fn new(round_number: usize, max_faulty: usize, round_proposer: V) -> Self { + pub fn new(round_number: usize, threshold: usize, round_proposer: V) -> Self { Accumulator { round_number, - max_faulty, + threshold, round_proposer, proposal: None, prepares: HashMap::new(), @@ -138,6 +145,20 @@ impl Accumulator self.advance_round.len() } + /// Get the round number. + pub fn round_number(&self) -> usize { + self.round_number.clone() + } + + /// Get the round proposer. + pub fn round_proposer(&self) -> &V { + &self.round_proposer + } + + pub fn proposal(&self) -> Option<&C> { + self.proposal.as_ref() + } + /// Inspect the current consensus state. pub fn state(&self) -> &State { &self.state @@ -189,7 +210,7 @@ impl Accumulator let count = self.vote_counts.entry(candidate.clone()).or_insert((0, 0)); count.0 += 1; - if count.0 == self.max_faulty * 2 + 1 { + if count.0 == self.threshold { Some(candidate) } else { None @@ -232,7 +253,7 @@ impl Accumulator let count = self.vote_counts.entry(candidate.clone()).or_insert((0, 0)); count.1 += 1; - if count.1 == self.max_faulty * 2 + 1 { + if count.1 == self.threshold { Some(candidate) } else { None @@ -252,7 +273,7 @@ impl Accumulator .map(|&(_, ref s)| s.clone()) .collect(); - self.state = State::Concluded(Justification { + self.state = State::Committed(Justification { round_number: self.round_number, digest: committed_for, signatures: signatures, @@ -266,12 +287,12 @@ impl Accumulator ) { self.advance_round.insert(sender); - if self.advance_round.len() != self.max_faulty * 2 + 1 { return } + if self.advance_round.len() != self.threshold { return } // allow transition to new round only if we haven't produced a justification // yet. self.state = match ::std::mem::replace(&mut self.state, State::Begin) { - State::Concluded(j) => State::Concluded(j), + State::Committed(j) => State::Committed(j), State::Prepared(j) => State::Advanced(Some(j)), State::Advanced(j) => State::Advanced(j), State::Begin | State::Proposed(_) => State::Advanced(None), @@ -311,24 +332,24 @@ mod tests { } }; - assert!(justification.check(3, &check_message)); - assert!(!justification.check(5, &check_message)); + assert!(justification.check(7, &check_message)); + assert!(!justification.check(11, &check_message)); { // one bad signature is enough to spoil it. justification.signatures.push(Signature(1001, 255)); - assert!(!justification.check(3, &check_message)); + assert!(!justification.check(7, &check_message)); justification.signatures.pop(); } // duplicates not allowed. justification.signatures.extend((0..10).map(|i| Signature(600, i))); - assert!(!justification.check(3, &check_message)); + assert!(!justification.check(11, &check_message)); } #[test] fn accepts_proposal_from_proposer_only() { - let mut accumulator = Accumulator::<_, Digest, _, _>::new(1, 3, ValidatorId(8)); + let mut accumulator = Accumulator::<_, Digest, _, _>::new(1, 7, ValidatorId(8)); assert_eq!(accumulator.state(), &State::Begin); accumulator.import_message(LocalizedMessage { @@ -350,7 +371,7 @@ mod tests { #[test] fn reaches_prepare_phase() { - let mut accumulator = Accumulator::new(1, 3, ValidatorId(8)); + let mut accumulator = Accumulator::new(1, 7, ValidatorId(8)); assert_eq!(accumulator.state(), &State::Begin); accumulator.import_message(LocalizedMessage { @@ -385,7 +406,7 @@ mod tests { #[test] fn prepare_to_commit() { - let mut accumulator = Accumulator::new(1, 3, ValidatorId(8)); + let mut accumulator = Accumulator::new(1, 7, ValidatorId(8)); assert_eq!(accumulator.state(), &State::Begin); accumulator.import_message(LocalizedMessage { @@ -437,14 +458,14 @@ mod tests { }); match accumulator.state() { - &State::Concluded(ref j) => assert_eq!(j.digest, Digest(999)), + &State::Committed(ref j) => assert_eq!(j.digest, Digest(999)), s => panic!("wrong state: {:?}", s), } } #[test] fn prepare_to_advance() { - let mut accumulator = Accumulator::new(1, 3, ValidatorId(8)); + let mut accumulator = Accumulator::new(1, 7, ValidatorId(8)); assert_eq!(accumulator.state(), &State::Begin); accumulator.import_message(LocalizedMessage { @@ -495,7 +516,7 @@ mod tests { #[test] fn conclude_different_than_proposed() { - let mut accumulator = Accumulator::::new(1, 3, ValidatorId(8)); + let mut accumulator = Accumulator::::new(1, 7, ValidatorId(8)); assert_eq!(accumulator.state(), &State::Begin); for i in 0..7 { @@ -520,14 +541,14 @@ mod tests { } match accumulator.state() { - &State::Concluded(ref j) => assert_eq!(j.digest, Digest(999)), + &State::Committed(ref j) => assert_eq!(j.digest, Digest(999)), s => panic!("wrong state: {:?}", s), } } #[test] fn begin_to_advance() { - let mut accumulator = Accumulator::::new(1, 3, ValidatorId(8)); + let mut accumulator = Accumulator::::new(1, 7, ValidatorId(8)); assert_eq!(accumulator.state(), &State::Begin); for i in 0..7 { @@ -546,7 +567,7 @@ mod tests { #[test] fn conclude_without_prepare() { - let mut accumulator = Accumulator::::new(1, 3, ValidatorId(8)); + let mut accumulator = Accumulator::::new(1, 7, ValidatorId(8)); assert_eq!(accumulator.state(), &State::Begin); for i in 0..7 { @@ -558,7 +579,7 @@ mod tests { } match accumulator.state() { - &State::Concluded(ref j) => assert_eq!(j.digest, Digest(999)), + &State::Committed(ref j) => assert_eq!(j.digest, Digest(999)), s => panic!("wrong state: {:?}", s), } } diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index bcb35718428ba..0c78029ecfb42 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -18,6 +18,15 @@ mod accumulator; +use std::collections::{HashMap, VecDeque}; +use std::hash::Hash; + +use futures::{future, Future, Stream, Sink, Poll, Async, AsyncSink}; + +use self::accumulator::State; + +pub use self::accumulator::{Accumulator, Justification, PrepareJustification}; + /// Messages over the proposal. /// Each message carries an associated round number. #[derive(Debug, Clone, PartialEq, Eq)] @@ -54,38 +63,556 @@ pub struct LocalizedMessage { pub signature: S, } -/// The agreed-upon data. -#[derive(Debug, Clone)] -pub struct Agreed { - /// The agreed-upon proposal. - pub proposal: P, - /// The justification for the proposal. - pub justification: Vec>, -} - -/// Parameters to agreement. -pub struct Params< - Validator, - SignLocal, - CanInclude, - MessagesIn, - MessagesOut, -> { - /// The ID of the current view's primary. - pub primary: Validator, - /// The local ID. - pub local_id: Validator, - /// A closure for signing local messages. - pub sign_local: SignLocal, - /// A function for checking if a proposal can be voted for. - pub can_include: CanInclude, - /// The input stream. Should never conclude, and should yield only messages - /// sent by validators and which have been authenticated properly. - pub input: MessagesIn, - /// The output message sink. This assumes that messages will eventually - /// be delivered to all honest participants, either by repropagation, gossip, - /// or some reliable broadcast mechanism. - pub output: MessagesOut, - /// The maximum number of faulty nodes. - pub max_faulty: usize, +/// Context necessary for agreement. +pub trait Context { + /// Candidate proposed. + type Candidate: Eq + Clone; + /// Candidate digest. + type Digest: Hash + Eq + Clone; + /// Validator ID. + type ValidatorId: Hash + Eq + Clone; + /// Signature. + type Signature: Eq + Clone; + /// A future that resolves when a round timeout is concluded. + type RoundTimeout: Future; + /// A future that resolves when a proposal is ready. + type Proposal: Future; + + /// Get the local validator ID. + fn local_id(&self) -> Self::ValidatorId; + + /// Get the best proposal. + fn proposal(&self) -> Self::Proposal; + + /// Get the digest of a candidate. + fn candidate_digest(&self, candidate: &Self::Candidate) -> Self::Digest; + + /// Sign a message using the local validator ID. + fn sign_local(&self, message: Message) + -> ContextLocalizedMessage; + + /// Get the proposer for a given round of consensus. + fn round_proposer(&self, round: usize) -> Self::ValidatorId; + + /// Whether the candidate is valid. + fn candidate_valid(&self, candidate: &Self::Candidate) -> bool; + + /// Create a round timeout. The context will determine the correct timeout + /// length, and create a future that will resolve when the timeout is + /// concluded. + fn begin_round_timeout(&self, round: usize) -> Self::RoundTimeout; +} + +/// Type alias for a localized message using only type parameters from `Context`. +// TODO: actual type alias when it's no longer a warning. +#[derive(Debug)] +pub struct ContextLocalizedMessage(pub LocalizedMessage); + +impl Clone for ContextLocalizedMessage + where LocalizedMessage: Clone +{ + fn clone(&self) -> Self { + ContextLocalizedMessage(self.0.clone()) + } +} + +#[derive(Debug)] +struct Sending { + items: VecDeque, + flushing: bool, +} + +impl Sending { + fn with_capacity(n: usize) -> Self { + Sending { + items: VecDeque::with_capacity(n), + flushing: false, + } + } + + fn push(&mut self, item: T) { + self.items.push_back(item); + self.flushing = false; + } + + // process all the sends into the sink. + fn process_all>(&mut self, sink: &mut S) -> Poll<(), S::SinkError> { + while let Some(item) = self.items.pop_front() { + match sink.start_send(item) { + Err(e) => return Err(e), + Ok(AsyncSink::NotReady(item)) => { + self.items.push_front(item); + return Ok(Async::NotReady); + } + Ok(AsyncSink::Ready) => { self.flushing = true; } + } + } + + while self.flushing { + match sink.poll_complete() { + Err(e) => return Err(e), + Ok(Async::NotReady) => return Ok(Async::NotReady), + Ok(Async::Ready(())) => { self.flushing = false; } + } + } + + Ok(Async::Ready(())) + } +} + +/// Error returned when the input stream concludes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InputStreamConcluded; + +impl ::std::fmt::Display for InputStreamConcluded { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + write!(f, "{}", ::std::error::Error::description(self)) + } +} + +impl ::std::error::Error for InputStreamConcluded { + fn description(&self) -> &str { + "input stream of messages concluded prematurely" + } +} + +// get the "full BFT" threshold based on an amount of nodes and +// a maximum faulty. if nodes == 3f + 1, then threshold == 2f + 1. +fn bft_threshold(nodes: usize, max_faulty: usize) -> usize { + nodes - max_faulty +} + +/// Committed successfully. +pub struct Committed { + /// The candidate committed for. This will be unknown if + /// we never witnessed the proposal of the last round. + pub candidate: Option, + /// A justification for the candidate. + pub justification: Justification, +} + +struct Locked { + justification: PrepareJustification, +} + +impl Locked { + fn digest(&self) -> &D { + &self.justification.digest + } +} + +// the state of the local node during the current state of consensus. +// +// behavior is different when locked on a proposal. +#[derive(Clone, Copy)] +enum LocalState { + Start, + Proposed, + Prepared, + Committed, + VoteAdvance, +} + +// This structure manages a single "view" of consensus. +// +// We maintain two message accumulators: one for the round we are currently in, +// and one for a future round. +// +// We also store notable candidates: any proposed or prepared for, as well as any +// with witnessed threshold-prepares. +// This ensures that threshold-prepares witnessed by even one honest participant +// will still have the candidate available for proposal. +// +// We advance the round accumulators when one of two conditions is met: +// - we witness consensus of advancement in the current round. in this case we +// advance by one. +// - a higher threshold-prepare is broadcast to us. in this case we can +// advance to the round of the threshold-prepare. this is an indication +// that we have experienced severe asynchrony/clock drift with the remainder +// of the other validators, and it is unlikely that we can assist in +// consensus meaningfully. nevertheless we make an attempt. +struct Strategy { + nodes: usize, + max_faulty: usize, + fetching_proposal: Option, + round_timeout: future::Fuse, + local_state: LocalState, + locked: Option>, + notable_candidates: HashMap, + current_accumulator: Accumulator, + future_accumulator: Accumulator, + local_id: C::ValidatorId, +} + +impl Strategy { + fn create(context: &C, nodes: usize, max_faulty: usize) -> Self { + let timeout = context.begin_round_timeout(0); + let threshold = bft_threshold(nodes, max_faulty); + + let current_accumulator = Accumulator::new( + 0, + threshold, + context.round_proposer(0), + ); + + let future_accumulator = Accumulator::new( + 1, + threshold, + context.round_proposer(1), + ); + + Strategy { + nodes, + max_faulty, + current_accumulator, + future_accumulator, + fetching_proposal: None, + local_state: LocalState::Start, + locked: None, + notable_candidates: HashMap::new(), + round_timeout: timeout.fuse(), + local_id: context.local_id(), + } + } + + fn import_message(&mut self, msg: ContextLocalizedMessage) { + let msg = msg.0; + let round_number = msg.message.round_number(); + + if round_number == self.current_accumulator.round_number() { + self.current_accumulator.import_message(msg); + } else if round_number == self.future_accumulator.round_number() { + self.future_accumulator.import_message(msg); + } + } + + // poll the strategy: this will queue messages to be sent and advance + // rounds if necessary. + // + // only call within the context of a `Task`. + fn poll(&mut self, context: &C, sending: &mut Sending>) + -> Poll, E> + where + C::RoundTimeout: Future, + C::Proposal: Future, + { + self.propose(context, sending)?; + self.prepare(context, sending); + self.commit(context, sending); + self.vote_advance(context, sending)?; + + let advance = match self.current_accumulator.state() { + &State::Advanced(ref p_just) => { + // lock to any witnessed prepare justification. + if let Some(p_just) = p_just.as_ref() { + self.locked = Some(Locked { justification: p_just.clone() }); + } + + let round_number = self.current_accumulator.round_number(); + Some(round_number + 1) + } + &State::Committed(ref just) => { + let candidate = self.notable_candidates.get(&just.digest).cloned(); + let committed = Committed { + candidate, + justification: just.clone() + }; + + return Ok(Async::Ready(committed)) + } + _ => None, + }; + + if let Some(new_round) = advance { + self.advance_to_round(context, new_round); + } + + Ok(Async::NotReady) + } + + fn propose(&mut self, context: &C, sending: &mut Sending>) + -> Result<(), ::Error> + { + if let LocalState::Start = self.local_state { + let mut propose = false; + if let &State::Begin = self.current_accumulator.state() { + let round_number = self.current_accumulator.round_number(); + let primary = context.round_proposer(round_number); + propose = self.local_id == primary; + }; + + if !propose { return Ok(()) } + + // obtain the proposal to broadcast. + let proposal = match self.locked { + Some(ref locked) => { + // TODO: it's possible but very unlikely that we don't have the + // corresponding proposal for what we are locked to. + // + // since this is an edge case on an edge case, it is fine + // to eat the round timeout for now, but it can be optimized by + // broadcasting an advance vote. + self.notable_candidates.get(locked.digest()).cloned() + } + None => { + let res = self.fetching_proposal + .get_or_insert_with(|| context.proposal()) + .poll()?; + + match res { + Async::Ready(p) => Some(p), + Async::NotReady => None, + } + } + }; + + if let Some(proposal) = proposal { + self.fetching_proposal = None; + + let message = Message::Propose( + self.current_accumulator.round_number(), + proposal + ); + + self.import_and_send_message(message, context, sending); + self.local_state = LocalState::Proposed; + } + + } + + Ok(()) + } + + fn prepare(&mut self, context: &C, sending: &mut Sending>) { + // prepare only upon start or having proposed. + match self.local_state { + LocalState::Start | LocalState::Proposed => {}, + _ => return + }; + + let mut prepare_for = None; + + // we can't prepare until something was proposed. + if let &State::Proposed(ref candidate) = self.current_accumulator.state() { + let digest = context.candidate_digest(candidate); + + // vote to prepare only if we believe the candidate to be valid and + // we are not locked on some other candidate. + match self.locked { + Some(ref locked) if locked.digest() != &digest => {} + Some(_) | None => { + if context.candidate_valid(candidate) { + prepare_for = Some(digest); + } + } + } + } + + if let Some(digest) = prepare_for { + let message = Message::Prepare( + self.current_accumulator.round_number(), + digest + ); + + self.import_and_send_message(message, context, sending); + self.local_state = LocalState::Prepared; + } + } + + fn commit(&mut self, context: &C, sending: &mut Sending>) { + // commit only if we haven't voted to advance or committed already + match self.local_state { + LocalState::Committed | LocalState::VoteAdvance => return, + _ => {} + } + + let mut commit_for = None; + + if let &State::Prepared(ref p_just) = self.current_accumulator.state() { + // we are now locked to this prepare justification. + let digest = p_just.digest.clone(); + self.locked = Some(Locked { justification: p_just.clone() }); + commit_for = Some(digest); + } + + if let Some(digest) = commit_for { + let message = Message::Commit( + self.current_accumulator.round_number(), + digest + ); + + self.import_and_send_message(message, context, sending); + self.local_state = LocalState::Committed; + } + } + + fn vote_advance(&mut self, context: &C, sending: &mut Sending>) + -> Result<(), ::Error> + { + // we can vote for advancement under all circumstances unless we have already. + if let LocalState::VoteAdvance = self.local_state { return Ok(()) } + + // if we got f + 1 advance votes, or the timeout has fired, and we haven't + // sent an AdvanceRound message yet, do so. + let mut attempt_advance = self.current_accumulator.advance_votes() > self.max_faulty; + + if let Async::Ready(_) = self.round_timeout.poll()? { + attempt_advance = true; + } + + // the other situation we attempt to advance is if there is a proposal + // that is not equal to the one we are locked to. + match (self.local_state, self.current_accumulator.state(), &self.locked) { + (LocalState::Start, &State::Proposed(ref candidate), &Some(ref locked)) => { + let candidate_digest = context.candidate_digest(candidate); + if &candidate_digest != locked.digest() { + attempt_advance = true; + } + } + _ => {} + } + + if attempt_advance { + let message = Message::AdvanceRound( + self.current_accumulator.round_number(), + ); + + self.import_and_send_message(message, context, sending); + self.local_state = LocalState::VoteAdvance; + } + + Ok(()) + } + + fn advance_to_round(&mut self, context: &C, round: usize) { + assert!(round > self.current_accumulator.round_number()); + + let threshold = self.nodes - self.max_faulty; + + self.fetching_proposal = None; + self.round_timeout = context.begin_round_timeout(round).fuse(); + self.local_state = LocalState::Start; + + let new_future = Accumulator::new( + round + 1, + threshold, + context.round_proposer(round + 1), + ); + + // when advancing from a round, store away the witnessed proposal. + // + // if we or other participants end up locked on that candidate, + // we will have it. + if let Some(proposal) = self.current_accumulator.proposal() { + let digest = context.candidate_digest(proposal); + self.notable_candidates.entry(digest).or_insert_with(|| proposal.clone()); + } + + // special case when advancing by a single round. + if self.future_accumulator.round_number() == round { + self.current_accumulator + = ::std::mem::replace(&mut self.future_accumulator, new_future); + } else { + self.future_accumulator = new_future; + self.current_accumulator = Accumulator::new( + round, + threshold, + context.round_proposer(round), + ); + } + } + + fn import_and_send_message( + &mut self, + message: Message, + context: &C, + sending: &mut Sending> + ) { + let signed_message = context.sign_local(message); + self.import_message(signed_message.clone()); + sending.push(signed_message); + } +} + +/// Future that resolves upon BFT agreement for a candidate. +#[must_use = "futures do nothing unless polled"] +pub struct Agreement { + context: C, + input: I, + output: O, + concluded: Option>, + sending: Sending>, + strategy: Strategy, +} + +impl Future for Agreement + where + C: Context, + C::RoundTimeout: Future, + C::Proposal: Future, + I: Stream,Error=E>, + O: Sink,SinkError=E>, + E: From, +{ + type Item = Committed; + type Error = E; + + fn poll(&mut self) -> Poll { + // even if we've observed the conclusion, wait until all + // pending outgoing messages are flushed. + if let Some(just) = self.concluded.take() { + return Ok(match self.sending.process_all(&mut self.output)? { + Async::Ready(()) => Async::Ready(just), + Async::NotReady => { + self.concluded = Some(just); + Async::NotReady + } + }) + } + + // make progress on flushing all pending messages. + let _ = self.sending.process_all(&mut self.output)?; + + // try to process timeouts. + if let Async::Ready(just) = self.strategy.poll(&self.context, &mut self.sending)? { + self.concluded = Some(just); + return self.poll(); + } + + let message = try_ready!(self.input.poll()).ok_or(InputStreamConcluded)?; + self.strategy.import_message(message); + + self.poll() + } +} + +/// Attempt to reach BFT agreement on a candidate. +/// +/// `nodes` is the number of nodes in the system. +/// `max_faulty` is the maximum number of faulty nodes. Should be less than +/// 1/3 of `nodes`, otherwise agreement may never be reached. +/// +/// The input stream should never logically conclude. The logic here assumes +/// that messages flushed to the output stream will eventually reach other nodes. +/// +/// Note that it is possible to witness agreement being reached without ever +/// seeing the candidate. Any candidates seen will be checked for validity. +/// +/// Although technically the agreement will always complete (given the eventual +/// delivery of messages), in practice it is possible for this future to +/// conclude without having witnessed the conclusion. +/// In general, this future should be pre-empted by the import of a justification +/// set for this block height. +pub fn agree(context: C, nodes: usize, max_faulty: usize, input: I, output: O) + -> Agreement +{ + let strategy = Strategy::create(&context, nodes, max_faulty); + Agreement { + context, + input, + output, + concluded: None, + sending: Sending::with_capacity(4), + strategy: strategy, + } } diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs index 09dd56f5f0874..b23b9c606cb07 100644 --- a/candidate-agreement/src/lib.rs +++ b/candidate-agreement/src/lib.rs @@ -29,6 +29,7 @@ //! //! Groups themselves may be compromised by malicious validators. +#[macro_use] extern crate futures; extern crate polkadot_primitives as primitives; diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 764f3a77dd5e7..cfcab5e1e4d95 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -70,32 +70,32 @@ pub trait Context { /// Statements circulated among peers. #[derive(PartialEq, Eq, Debug)] -pub enum Statement { +pub enum Statement { /// Broadcast by a validator to indicate that this is his candidate for /// inclusion. /// /// Broadcasting two different candidate messages per round is not allowed. - Candidate(C::Candidate), + Candidate(C), /// Broadcast by a validator to attest that the candidate with given digest /// is valid. - Valid(C::Digest), + Valid(D), /// Broadcast by a validator to attest that the auxiliary data for a candidate /// with given digest is available. - Available(C::Digest), + Available(D), /// Broadcast by a validator to attest that the candidate with given digest /// is invalid. - Invalid(C::Digest), + Invalid(D), } /// A signed statement. #[derive(PartialEq, Eq, Debug)] -pub struct SignedStatement { +pub struct SignedStatement { /// The statement. - pub statement: Statement, + pub statement: Statement, /// The signature. - pub signature: C::Signature, + pub signature: S, /// The sender. - pub sender: C::ValidatorId, + pub sender: V, } // A unique trace for a class of valid statements issued by a validator. @@ -123,41 +123,52 @@ enum StatementTrace { /// Since there are three possible ways to vote, a double vote is possible in /// three possible combinations (unordered) #[derive(PartialEq, Eq, Debug)] -pub enum ValidityDoubleVote { +pub enum ValidityDoubleVote { /// Implicit vote by issuing and explicity voting validity. - IssuedAndValidity((C::Candidate, C::Signature), (C::Digest, C::Signature)), + IssuedAndValidity((C, S), (D, S)), /// Implicit vote by issuing and explicitly voting invalidity - IssuedAndInvalidity((C::Candidate, C::Signature), (C::Digest, C::Signature)), + IssuedAndInvalidity((C, S), (D, S)), /// Direct votes for validity and invalidity - ValidityAndInvalidity(C::Digest, C::Signature, C::Signature), + ValidityAndInvalidity(D, S, S), } /// Misbehavior: declaring multiple candidates. #[derive(PartialEq, Eq, Debug)] -pub struct MultipleCandidates { +pub struct MultipleCandidates { /// The first candidate seen. - pub first: (C::Candidate, C::Signature), + pub first: (C, S), /// The second candidate seen. - pub second: (C::Candidate, C::Signature), + pub second: (C, S), } /// Misbehavior: submitted statement for wrong group. #[derive(PartialEq, Eq, Debug)] -pub struct UnauthorizedStatement { +pub struct UnauthorizedStatement { /// A signed statement which was submitted without proper authority. - pub statement: SignedStatement, + pub statement: SignedStatement, } /// Different kinds of misbehavior. All of these kinds of malicious misbehavior /// are easily provable and extremely disincentivized. #[derive(PartialEq, Eq, Debug)] -pub enum Misbehavior { +pub enum Misbehavior { /// Voted invalid and valid on validity. - ValidityDoubleVote(ValidityDoubleVote), + ValidityDoubleVote(ValidityDoubleVote), /// Submitted multiple candidates. - MultipleCandidates(MultipleCandidates), + MultipleCandidates(MultipleCandidates), /// Submitted a message withou - UnauthorizedStatement(UnauthorizedStatement), + UnauthorizedStatement(UnauthorizedStatement), +} + +/// Fancy work-around for a type alias of context-based misbehavior +/// without producing compiler warnings. +pub trait ResolveMisbehavior { + /// The misbehavior type. + type Misbehavior; +} + +impl ResolveMisbehavior for C { + type Misbehavior = Misbehavior; } // kinds of votes for validity @@ -251,7 +262,7 @@ pub fn create() -> Table { #[derive(Default)] pub struct Table { validator_data: HashMap>, - detected_misbehavior: HashMap>, + detected_misbehavior: HashMap::Misbehavior>, candidate_votes: HashMap>, } @@ -294,7 +305,7 @@ impl Table { } /// Drain all misbehavior observed up to this point. - pub fn drain_misbehavior(&mut self) -> HashMap> { + pub fn drain_misbehavior(&mut self) -> HashMap::Misbehavior> { ::std::mem::replace(&mut self.detected_misbehavior, HashMap::new()) } @@ -303,9 +314,12 @@ impl Table { /// /// This can note the origin of the statement to indicate that he has /// seen it already. - pub fn import_statement(&mut self, context: &C, statement: SignedStatement, from: Option) - -> Option> - { + pub fn import_statement( + &mut self, + context: &C, + statement: SignedStatement, + from: Option + ) -> Option> { let SignedStatement { statement, signature, sender: signer } = statement; let trace = match statement { @@ -370,7 +384,7 @@ impl Table { from: C::ValidatorId, candidate: C::Candidate, signature: C::Signature, - ) -> (Option>, Option>) { + ) -> (Option<::Misbehavior>, Option>) { let group = context.candidate_group(&candidate); if !context.is_member_of(&from, &group) { return ( @@ -444,7 +458,7 @@ impl Table { from: C::ValidatorId, digest: C::Digest, vote: ValidityVote, - ) -> (Option>, Option>) { + ) -> (Option<::Misbehavior>, Option>) { let votes = match self.candidate_votes.get_mut(&digest) { None => return (None, None), // TODO: queue up but don't get DoS'ed Some(votes) => votes, @@ -522,7 +536,7 @@ impl Table { from: C::ValidatorId, digest: C::Digest, signature: C::Signature, - ) -> (Option>, Option>) { + ) -> (Option<::Misbehavior>, Option>) { let votes = match self.candidate_votes.get_mut(&digest) { None => return (None, None), // TODO: queue up but don't get DoS'ed Some(votes) => votes, From a52569ea7065a87ab5111c9bf9bf27b8e0c4cd3c Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 27 Dec 2017 18:03:41 +0100 Subject: [PATCH 25/40] inex -> index --- primitives/src/parachain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 29fc11e9b502b..1d64f3c82429a 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -55,7 +55,7 @@ pub struct Candidate { #[serde(deny_unknown_fields)] pub struct CandidateReceipt { /// The ID of the parachain this is a candidate for. - pub parachain_inex: Id, + pub parachain_index: Id, /// The collator's account ID pub collator: ::Address, /// The head-data From dfa1c1259b91c2eedde881bad35aee7a8ec78d9f Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 27 Dec 2017 18:27:18 +0100 Subject: [PATCH 26/40] import and broadcast lock proofs --- candidate-agreement/src/bft/mod.rs | 98 +++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 21 deletions(-) diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index 0c78029ecfb42..e2322cc38bda4 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -89,7 +89,7 @@ pub trait Context { /// Sign a message using the local validator ID. fn sign_local(&self, message: Message) - -> ContextLocalizedMessage; + -> LocalizedMessage; /// Get the proposer for a given round of consensus. fn round_proposer(&self, round: usize) -> Self::ValidatorId; @@ -103,16 +103,26 @@ pub trait Context { fn begin_round_timeout(&self, round: usize) -> Self::RoundTimeout; } +/// Communication that can occur between participants in consensus. +#[derive(Debug, Clone)] +pub enum Communication { + /// A consensus message (proposal or vote) + Message(LocalizedMessage), + /// A proof-of-lock. + Locked(PrepareJustification), +} + /// Type alias for a localized message using only type parameters from `Context`. // TODO: actual type alias when it's no longer a warning. -#[derive(Debug)] -pub struct ContextLocalizedMessage(pub LocalizedMessage); +pub struct ContextCommunication(pub Communication); -impl Clone for ContextLocalizedMessage - where LocalizedMessage: Clone +impl Clone for ContextCommunication + where + LocalizedMessage: Clone, + PrepareJustification: Clone, { fn clone(&self) -> Self { - ContextLocalizedMessage(self.0.clone()) + ContextCommunication(self.0.clone()) } } @@ -275,8 +285,10 @@ impl Strategy { } } - fn import_message(&mut self, msg: ContextLocalizedMessage) { - let msg = msg.0; + fn import_message( + &mut self, + msg: LocalizedMessage + ) { let round_number = msg.message.round_number(); if round_number == self.current_accumulator.round_number() { @@ -286,11 +298,31 @@ impl Strategy { } } + fn import_lock_proof( + &mut self, + context: &C, + justification: PrepareJustification, + ) { + // TODO: find a way to avoid processing of the signatures if the sender is + // not the primary or the round number is low. + let current_round_number = self.current_accumulator.round_number(); + if justification.round_number < current_round_number { + return + } else if justification.round_number == current_round_number { + self.locked = Some(Locked { justification }); + } else { + // jump ahead to the prior round as this is an indication of a supermajority + // good nodes being at least on that round. + self.advance_to_round(context, justification.round_number); + self.locked = Some(Locked { justification }); + } + } + // poll the strategy: this will queue messages to be sent and advance // rounds if necessary. // // only call within the context of a `Task`. - fn poll(&mut self, context: &C, sending: &mut Sending>) + fn poll(&mut self, context: &C, sending: &mut Sending>) -> Poll, E> where C::RoundTimeout: Future, @@ -312,7 +344,20 @@ impl Strategy { Some(round_number + 1) } &State::Committed(ref just) => { - let candidate = self.notable_candidates.get(&just.digest).cloned(); + // fetch the agreed-upon candidate: + // - we may not have received the proposal in the first place + // - there is no guarantee that the proposal we got was agreed upon + // (can happen if faulty primary) + // - look in the candidates of prior rounds just in case. + let candidate = self.current_accumulator + .proposal() + .and_then(|c| if context.candidate_digest(c) == just.digest { + Some(c.clone()) + } else { + None + }) + .or_else(|| self.notable_candidates.get(&just.digest).cloned()); + let committed = Committed { candidate, justification: just.clone() @@ -330,7 +375,7 @@ impl Strategy { Ok(Async::NotReady) } - fn propose(&mut self, context: &C, sending: &mut Sending>) + fn propose(&mut self, context: &C, sending: &mut Sending>) -> Result<(), ::Error> { if let LocalState::Start = self.local_state { @@ -375,15 +420,22 @@ impl Strategy { ); self.import_and_send_message(message, context, sending); + + // broadcast the justification along with the proposal if we are locked. + if let Some(ref locked) = self.locked { + sending.push( + ContextCommunication(Communication::Locked(locked.justification.clone())) + ); + } + self.local_state = LocalState::Proposed; } - } Ok(()) } - fn prepare(&mut self, context: &C, sending: &mut Sending>) { + fn prepare(&mut self, context: &C, sending: &mut Sending>) { // prepare only upon start or having proposed. match self.local_state { LocalState::Start | LocalState::Proposed => {}, @@ -419,7 +471,7 @@ impl Strategy { } } - fn commit(&mut self, context: &C, sending: &mut Sending>) { + fn commit(&mut self, context: &C, sending: &mut Sending>) { // commit only if we haven't voted to advance or committed already match self.local_state { LocalState::Committed | LocalState::VoteAdvance => return, @@ -446,7 +498,7 @@ impl Strategy { } } - fn vote_advance(&mut self, context: &C, sending: &mut Sending>) + fn vote_advance(&mut self, context: &C, sending: &mut Sending>) -> Result<(), ::Error> { // we can vote for advancement under all circumstances unless we have already. @@ -526,11 +578,11 @@ impl Strategy { &mut self, message: Message, context: &C, - sending: &mut Sending> + sending: &mut Sending> ) { let signed_message = context.sign_local(message); self.import_message(signed_message.clone()); - sending.push(signed_message); + sending.push(ContextCommunication(Communication::Message(signed_message))); } } @@ -541,7 +593,7 @@ pub struct Agreement { input: I, output: O, concluded: Option>, - sending: Sending>, + sending: Sending>, strategy: Strategy, } @@ -550,8 +602,8 @@ impl Future for Agreement C: Context, C::RoundTimeout: Future, C::Proposal: Future, - I: Stream,Error=E>, - O: Sink,SinkError=E>, + I: Stream,Error=E>, + O: Sink,SinkError=E>, E: From, { type Item = Committed; @@ -580,7 +632,11 @@ impl Future for Agreement } let message = try_ready!(self.input.poll()).ok_or(InputStreamConcluded)?; - self.strategy.import_message(message); + + match message.0 { + Communication::Message(message) => self.strategy.import_message(message), + Communication::Locked(proof) => self.strategy.import_lock_proof(&self.context, proof), + } self.poll() } From 800ce6ffb84b3ba298b72389e6c76b550defd482 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 27 Dec 2017 18:35:21 +0100 Subject: [PATCH 27/40] poll repeatedly when state changes --- candidate-agreement/src/bft/mod.rs | 35 +++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index e2322cc38bda4..940a0a9e21dbc 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -214,7 +214,7 @@ impl Locked { // the state of the local node during the current state of consensus. // // behavior is different when locked on a proposal. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq, Eq)] enum LocalState { Start, Proposed, @@ -327,6 +327,39 @@ impl Strategy { where C::RoundTimeout: Future, C::Proposal: Future, + { + let mut last_watermark = ( + self.current_accumulator.round_number(), + self.local_state + ); + + // poll until either completion or state doesn't change. + loop { + match self.poll_once(context, sending)? { + Async::Ready(x) => return Ok(Async::Ready(x)), + Async::NotReady => { + let new_watermark = ( + self.current_accumulator.round_number(), + self.local_state + ); + + if new_watermark == last_watermark { + return Ok(Async::NotReady) + } else { + last_watermark = new_watermark; + } + } + } + } + } + + // perform one round of polling: attempt to broadcast messages and change the state. + // if the round or internal round-state changes, this should be called again. + fn poll_once(&mut self, context: &C, sending: &mut Sending>) + -> Poll, E> + where + C::RoundTimeout: Future, + C::Proposal: Future, { self.propose(context, sending)?; self.prepare(context, sending); From 92d8510b2e451421bb029f5cc52f812be917d5ab Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 27 Dec 2017 18:48:35 +0100 Subject: [PATCH 28/40] don't broadcast advance vote immediately if locked --- candidate-agreement/src/bft/mod.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index 940a0a9e21dbc..c82e739abc35b 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -545,18 +545,6 @@ impl Strategy { attempt_advance = true; } - // the other situation we attempt to advance is if there is a proposal - // that is not equal to the one we are locked to. - match (self.local_state, self.current_accumulator.state(), &self.locked) { - (LocalState::Start, &State::Proposed(ref candidate), &Some(ref locked)) => { - let candidate_digest = context.candidate_digest(candidate); - if &candidate_digest != locked.digest() { - attempt_advance = true; - } - } - _ => {} - } - if attempt_advance { let message = Message::AdvanceRound( self.current_accumulator.round_number(), From fefb5cab54ac054b33fd618af7dcbef4c04fe103 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 27 Dec 2017 19:14:55 +0100 Subject: [PATCH 29/40] do not check validity of locked candidate --- candidate-agreement/src/bft/mod.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index c82e739abc35b..819f7a5da3db1 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -228,11 +228,6 @@ enum LocalState { // We maintain two message accumulators: one for the round we are currently in, // and one for a future round. // -// We also store notable candidates: any proposed or prepared for, as well as any -// with witnessed threshold-prepares. -// This ensures that threshold-prepares witnessed by even one honest participant -// will still have the candidate available for proposal. -// // We advance the round accumulators when one of two conditions is met: // - we witness consensus of advancement in the current round. in this case we // advance by one. @@ -485,11 +480,14 @@ impl Strategy { // we are not locked on some other candidate. match self.locked { Some(ref locked) if locked.digest() != &digest => {} - Some(_) | None => { - if context.candidate_valid(candidate) { - prepare_for = Some(digest); - } + Some(_) => { + // don't check validity if we are locked. + // this is necessary to preserve the liveness property. + prepare_for = Some(digest) } + None => if context.candidate_valid(candidate) { + prepare_for = Some(digest); + }, } } From 0c7928d190494329ea2ec54431b45f9f6ccc0c7e Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 27 Dec 2017 21:35:19 +0100 Subject: [PATCH 30/40] basic tests for the strategy --- candidate-agreement/src/bft/mod.rs | 52 +++-- candidate-agreement/src/bft/tests.rs | 324 +++++++++++++++++++++++++++ candidate-agreement/src/lib.rs | 1 - 3 files changed, 358 insertions(+), 19 deletions(-) create mode 100644 candidate-agreement/src/bft/tests.rs diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index 819f7a5da3db1..bad684820eeab 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -18,7 +18,11 @@ mod accumulator; +#[cfg(test)] +mod tests; + use std::collections::{HashMap, VecDeque}; +use std::fmt::Debug; use std::hash::Hash; use futures::{future, Future, Stream, Sink, Poll, Async, AsyncSink}; @@ -66,13 +70,13 @@ pub struct LocalizedMessage { /// Context necessary for agreement. pub trait Context { /// Candidate proposed. - type Candidate: Eq + Clone; + type Candidate: Debug + Eq + Clone; /// Candidate digest. - type Digest: Hash + Eq + Clone; + type Digest: Debug + Hash + Eq + Clone; /// Validator ID. - type ValidatorId: Hash + Eq + Clone; + type ValidatorId: Debug + Hash + Eq + Clone; /// Signature. - type Signature: Eq + Clone; + type Signature: Debug + Eq + Clone; /// A future that resolves when a round timeout is concluded. type RoundTimeout: Future; /// A future that resolves when a proposal is ready. @@ -193,6 +197,7 @@ fn bft_threshold(nodes: usize, max_faulty: usize) -> usize { } /// Committed successfully. +#[derive(Debug, Clone)] pub struct Committed { /// The candidate committed for. This will be unknown if /// we never witnessed the proposal of the last round. @@ -214,7 +219,7 @@ impl Locked { // the state of the local node during the current state of consensus. // // behavior is different when locked on a proposal. -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum LocalState { Start, Proposed, @@ -486,7 +491,7 @@ impl Strategy { prepare_for = Some(digest) } None => if context.candidate_valid(candidate) { - prepare_for = Some(digest); + prepare_for = Some(digest); }, } } @@ -641,23 +646,34 @@ impl Future for Agreement }) } - // make progress on flushing all pending messages. - let _ = self.sending.process_all(&mut self.output)?; + loop { + let message = match self.input.poll()? { + Async::Ready(msg) => msg.ok_or(InputStreamConcluded)?, + Async::NotReady => break, + }; - // try to process timeouts. - if let Async::Ready(just) = self.strategy.poll(&self.context, &mut self.sending)? { - self.concluded = Some(just); - return self.poll(); + match message.0 { + Communication::Message(message) => self.strategy.import_message(message), + Communication::Locked(proof) => self.strategy.import_lock_proof(&self.context, proof), + } } - let message = try_ready!(self.input.poll()).ok_or(InputStreamConcluded)?; + // try to process timeouts. + let state_machine_res = self.strategy.poll(&self.context, &mut self.sending)?; - match message.0 { - Communication::Message(message) => self.strategy.import_message(message), - Communication::Locked(proof) => self.strategy.import_lock_proof(&self.context, proof), - } + // make progress on flushing all pending messages. + let _ = self.sending.process_all(&mut self.output)?; - self.poll() + match state_machine_res { + Async::Ready(just) => { + self.concluded = Some(just); + self.poll() + } + Async::NotReady => { + + Ok(Async::NotReady) + } + } } } diff --git a/candidate-agreement/src/bft/tests.rs b/candidate-agreement/src/bft/tests.rs new file mode 100644 index 0000000000000..2228ab2604540 --- /dev/null +++ b/candidate-agreement/src/bft/tests.rs @@ -0,0 +1,324 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the candidate agreement strategy. + +use super::*; + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use futures::prelude::*; +use futures::sync::{oneshot, mpsc}; +use futures::future::FutureResult; + +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +struct Candidate(usize); + +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +struct Digest(usize); + +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +struct ValidatorId(usize); + +#[derive(Debug, PartialEq, Eq, Clone)] +struct Signature(Message, ValidatorId); + +struct SharedContext { + node_count: usize, + current_round: usize, + awaiting_round_timeouts: HashMap>>, +} + +#[derive(Debug)] +struct Error; + +impl From for Error { + fn from(_: InputStreamConcluded) -> Error { + Error + } +} + +impl SharedContext { + fn new(node_count: usize) -> Self { + SharedContext { + node_count, + current_round: 0, + awaiting_round_timeouts: HashMap::new() + } + } + + fn round_timeout(&mut self, round: usize) -> Box> { + let (tx, rx) = oneshot::channel(); + if round < self.current_round { + tx.send(()).unwrap() + } else { + self.awaiting_round_timeouts + .entry(round) + .or_insert_with(Vec::new) + .push(tx); + } + + Box::new(rx.map_err(|_| Error)) + } + + fn bump_round(&mut self) { + let awaiting_timeout = self.awaiting_round_timeouts + .remove(&self.current_round) + .unwrap_or_else(Vec::new); + + for tx in awaiting_timeout { + let _ = tx.send(()); + } + + self.current_round += 1; + } + + fn round_proposer(&self, round: usize) -> ValidatorId { + ValidatorId(round % self.node_count) + } +} + +struct Ctx { + local_id: ValidatorId, + proposal: Mutex, + shared: Arc>, +} + +impl Context for Ctx { + type Candidate = Candidate; + type Digest = Digest; + type ValidatorId = ValidatorId; + type Signature = Signature; + type RoundTimeout = Box>; + type Proposal = FutureResult; + + fn local_id(&self) -> ValidatorId { + self.local_id.clone() + } + + fn proposal(&self) -> Self::Proposal { + let proposal = { + let mut p = self.proposal.lock().unwrap(); + let x = *p; + *p = (*p * 2) + 1; + x + }; + + Ok(Candidate(proposal)).into_future() + } + + fn candidate_digest(&self, candidate: &Candidate) -> Digest { + Digest(candidate.0) + } + + fn sign_local(&self, message: Message) + -> LocalizedMessage + { + let signature = Signature(message.clone(), self.local_id.clone()); + LocalizedMessage { + message, + signature, + sender: self.local_id.clone() + } + } + + fn round_proposer(&self, round: usize) -> ValidatorId { + self.shared.lock().unwrap().round_proposer(round) + } + + fn candidate_valid(&self, candidate: &Candidate) -> bool { + candidate.0 % 3 != 0 + } + + fn begin_round_timeout(&self, round: usize) -> Self::RoundTimeout { + self.shared.lock().unwrap().round_timeout(round) + } +} + +type Comm = ContextCommunication; + +struct Network { + endpoints: Vec>, + input: mpsc::UnboundedReceiver<(usize, Comm)>, +} + +impl Network { + fn new(nodes: usize) + -> (Network, Vec>, Vec>) + { + let mut inputs = Vec::with_capacity(nodes); + let mut outputs = Vec::with_capacity(nodes); + let mut endpoints = Vec::with_capacity(nodes); + + let (in_tx, in_rx) = mpsc::unbounded(); + for _ in 0..nodes { + let (out_tx, out_rx) = mpsc::unbounded(); + inputs.push(in_tx.clone()); + outputs.push(out_rx); + endpoints.push(out_tx); + } + + let network = Network { + endpoints, + input: in_rx, + }; + + (network, inputs, outputs) + } + + fn route_on_thread(self) { + ::std::thread::spawn(move || { let _ = self.wait(); }); + } +} + +impl Future for Network { + type Item = (); + type Error = Error; + + fn poll(&mut self) -> Poll<(), Error> { + match self.input.poll() { + Err(_) => Err(Error), + Ok(Async::NotReady) => Ok(Async::NotReady), + Ok(Async::Ready(None)) => Ok(Async::Ready(())), + Ok(Async::Ready(Some((sender, item)))) => { + { + let receiving_endpoints = self.endpoints + .iter() + .enumerate() + .filter(|&(i, _)| i != sender) + .map(|(_, x)| x); + + for endpoint in receiving_endpoints { + let _ = endpoint.unbounded_send(item.clone()); + } + } + + self.poll() + } + } + } +} + +fn timeout_in(t: Duration) -> oneshot::Receiver<()> { + let (tx, rx) = oneshot::channel(); + ::std::thread::spawn(move || { + ::std::thread::sleep(t); + let _ = tx.send(()); + }); + + rx +} + +#[test] +fn consensus_completes_with_minimum_good() { + let node_count = 10; + let max_faulty = 3; + + let shared_context = Arc::new(Mutex::new(SharedContext::new(node_count))); + + let (network, net_send, net_recv) = Network::new(node_count); + network.route_on_thread(); + + let nodes = net_send + .into_iter() + .zip(net_recv) + .take(node_count - max_faulty) + .enumerate() + .map(|(i, (tx, rx))| { + let ctx = Ctx { + local_id: ValidatorId(i), + proposal: Mutex::new(i), + shared: shared_context.clone(), + }; + + agree( + ctx, + node_count, + max_faulty, + rx.map_err(|_| Error), + tx.sink_map_err(|_| Error).with(move |t| Ok((i, t))), + ) + }) + .collect::>(); + + ::std::thread::spawn(move || { + let mut timeout = ::std::time::Duration::from_millis(50); + loop { + ::std::thread::sleep(timeout.clone()); + shared_context.lock().unwrap().bump_round(); + timeout *= 2; + } + }); + + let timeout = timeout_in(Duration::from_millis(500)).map_err(|_| Error); + let results = ::futures::future::join_all(nodes) + .map(Some) + .select(timeout.map(|_| None)) + .wait() + .map(|(i, _)| i) + .map_err(|(e, _)| e) + .expect("to complete") + .expect("to not time out"); + + for result in &results { + assert_eq!(&result.justification.digest, &results[0].justification.digest); + } +} + +#[test] +fn consensus_does_not_complete_without_enough_nodes() { + let node_count = 10; + let max_faulty = 3; + + let shared_context = Arc::new(Mutex::new(SharedContext::new(node_count))); + + let (network, net_send, net_recv) = Network::new(node_count); + network.route_on_thread(); + + let nodes = net_send + .into_iter() + .zip(net_recv) + .take(node_count - max_faulty - 1) + .enumerate() + .map(|(i, (tx, rx))| { + let ctx = Ctx { + local_id: ValidatorId(i), + proposal: Mutex::new(i), + shared: shared_context.clone(), + }; + + agree( + ctx, + node_count, + max_faulty, + rx.map_err(|_| Error), + tx.sink_map_err(|_| Error).with(move |t| Ok((i, t))), + ) + }) + .collect::>(); + + let timeout = timeout_in(Duration::from_millis(500)).map_err(|_| Error); + let result = ::futures::future::join_all(nodes) + .map(Some) + .select(timeout.map(|_| None)) + .wait() + .map(|(i, _)| i) + .map_err(|(e, _)| e) + .expect("to complete"); + + assert!(result.is_none(), "not enough online nodes"); +} diff --git a/candidate-agreement/src/lib.rs b/candidate-agreement/src/lib.rs index b23b9c606cb07..09dd56f5f0874 100644 --- a/candidate-agreement/src/lib.rs +++ b/candidate-agreement/src/lib.rs @@ -29,7 +29,6 @@ //! //! Groups themselves may be compromised by malicious validators. -#[macro_use] extern crate futures; extern crate polkadot_primitives as primitives; From 409920adcad18c012ff0721ecf8783323fe93462 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 29 Dec 2017 02:56:04 +0100 Subject: [PATCH 31/40] remove unused context trait and fix warning --- Cargo.lock | 20 ++++++++++---------- candidate-agreement/src/bft/accumulator.rs | 12 ------------ candidate-agreement/src/table.rs | 2 +- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82f9c977063ee..a185bd50e069d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,13 +1,3 @@ -[root] -name = "polkadot-validator" -version = "0.1.0" -dependencies = [ - "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "polkadot-primitives 0.1.0", - "polkadot-serializer 0.1.0", - "serde 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "aho-corasick" version = "0.6.3" @@ -722,6 +712,16 @@ dependencies = [ "triehash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "polkadot-validator" +version = "0.1.0" +dependencies = [ + "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "polkadot-primitives 0.1.0", + "polkadot-serializer 0.1.0", + "serde 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "pretty_assertions" version = "0.4.0" diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs index 78f2958bc56c9..3bd5147e984e3 100644 --- a/candidate-agreement/src/bft/accumulator.rs +++ b/candidate-agreement/src/bft/accumulator.rs @@ -22,18 +22,6 @@ use std::hash::Hash; use super::{Message, LocalizedMessage}; -/// Context necessary to execute a round of BFT. -pub trait Context { - /// A full candidate. - type Candidate: Clone; - /// Unique digest of a proposed candidate (think hash). - type Digest: Hash + Eq + Clone; - /// Validator ID. - type ValidatorId: Hash + Eq; - /// A signature. - type Signature: Eq + Clone; -} - /// Justification at a given round. #[derive(PartialEq, Eq, Debug, Clone)] pub struct Justification { diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index cfcab5e1e4d95..f7dfb9766633c 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -284,7 +284,7 @@ impl Table { let candidate = &candidate_data.candidate; match best_candidates.entry(group_id.clone()) { BTreeEntry::Occupied(mut occ) => { - let mut candidate_ref = occ.get_mut(); + let candidate_ref = occ.get_mut(); if *candidate_ref < candidate { *candidate_ref = candidate; } From 922d5475d1be0ee270915e86dcc81c5656faae7a Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 29 Dec 2017 14:26:41 +0100 Subject: [PATCH 32/40] address some review grumbles --- candidate-agreement/src/bft/accumulator.rs | 118 +++++++++++---------- candidate-agreement/src/bft/mod.rs | 15 +-- candidate-agreement/src/bft/tests.rs | 10 +- 3 files changed, 75 insertions(+), 68 deletions(-) diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs index 3bd5147e984e3..42e5459392589 100644 --- a/candidate-agreement/src/bft/accumulator.rs +++ b/candidate-agreement/src/bft/accumulator.rs @@ -22,7 +22,7 @@ use std::hash::Hash; use super::{Message, LocalizedMessage}; -/// Justification at a given round. +/// Justification for some state at a given round. #[derive(PartialEq, Eq, Debug, Clone)] pub struct Justification { /// The round. @@ -40,8 +40,10 @@ impl Justification { /// digest. /// /// The closure should return true iff the round number, digest, and signature - /// represent a valid prepare message and the signer was authorized to issue + /// represent a valid message and the signer was authorized to issue /// it. + /// + /// The `check_message` closure may vary based on context. pub fn check(&self, threshold: usize, check_message: F) -> bool where F: Fn(usize, &D, &S) -> Option, @@ -49,22 +51,18 @@ impl Justification { { let mut voted = HashSet::new(); - let mut good = false; for signature in &self.signatures { match check_message(self.round_number, &self.digest, signature) { None => return false, Some(v) => { if !voted.insert(v) { return false; - } else if voted.len() >= threshold { - // don't return just yet since later signatures may be invalid. - good = true; } } } } - good + voted.len() >= threshold } } @@ -73,48 +71,54 @@ pub type PrepareJustification = Justification; /// The round's state, based on imported messages. #[derive(PartialEq, Eq, Debug)] -pub enum State { +pub enum State { /// No proposal yet. Begin, /// Proposal received. - Proposed(C), - /// Seen 2f + 1 prepares for this digest. - Prepared(PrepareJustification), - /// Seen 2f + 1 commits for a digest. - Committed(Justification), - /// Seen 2f + 1 round-advancement messages. - Advanced(Option>), + Proposed(Candidate), + /// Seen n - f prepares for this digest. + Prepared(PrepareJustification), + /// Seen n - f commits for a digest. + Committed(Justification), + /// Seen n - f round-advancement messages. + Advanced(Option>), +} + +#[derive(Debug, Default)] +struct VoteCounts { + prepared: usize, + committed: usize, } /// Accumulates messages for a given round of BFT consensus. #[derive(Debug)] -pub struct Accumulator +pub struct Accumulator where - C: Eq + Clone, - D: Hash + Eq + Clone, - V: Hash + Eq, - S: Eq + Clone, + Candidate: Eq + Clone, + Digest: Hash + Eq + Clone, + ValidatorId: Hash + Eq, + Signature: Eq + Clone, { round_number: usize, threshold: usize, - round_proposer: V, - proposal: Option, - prepares: HashMap, - commits: HashMap, - vote_counts: HashMap, - advance_round: HashSet, - state: State, + round_proposer: ValidatorId, + proposal: Option, + prepares: HashMap, + commits: HashMap, + vote_counts: HashMap, + advance_round: HashSet, + state: State, } -impl Accumulator +impl Accumulator where - C: Eq + Clone, - D: Hash + Eq + Clone, - V: Hash + Eq, - S: Eq + Clone, + Candidate: Eq + Clone, + Digest: Hash + Eq + Clone, + ValidatorId: Hash + Eq, + Signature: Eq + Clone, { /// Create a new state accumulator. - pub fn new(round_number: usize, threshold: usize, round_proposer: V) -> Self { + pub fn new(round_number: usize, threshold: usize, round_proposer: ValidatorId) -> Self { Accumulator { round_number, threshold, @@ -139,16 +143,16 @@ impl Accumulator } /// Get the round proposer. - pub fn round_proposer(&self) -> &V { + pub fn round_proposer(&self) -> &ValidatorId { &self.round_proposer } - pub fn proposal(&self) -> Option<&C> { + pub fn proposal(&self) -> Option<&Candidate> { self.proposal.as_ref() } /// Inspect the current consensus state. - pub fn state(&self) -> &State { + pub fn state(&self) -> &State { &self.state } @@ -156,10 +160,10 @@ impl Accumulator /// and authorization should have already been checked. pub fn import_message( &mut self, - message: LocalizedMessage, + message: LocalizedMessage, ) { - // old message. + // message from different round. if message.message.round_number() != self.round_number { return; } @@ -176,8 +180,8 @@ impl Accumulator fn import_proposal( &mut self, - proposal: C, - sender: V, + proposal: Candidate, + sender: ValidatorId, ) { if sender != self.round_proposer || self.proposal.is_some() { return } @@ -187,19 +191,19 @@ impl Accumulator fn import_prepare( &mut self, - candidate: D, - sender: V, - signature: S, + digest: Digest, + sender: ValidatorId, + signature: Signature, ) { // ignore any subsequent prepares by the same sender. // TODO: if digest is different, that's misbehavior. let prepared_for = if let Entry::Vacant(vacant) = self.prepares.entry(sender) { - vacant.insert((candidate.clone(), signature)); - let count = self.vote_counts.entry(candidate.clone()).or_insert((0, 0)); - count.0 += 1; + vacant.insert((digest.clone(), signature)); + let count = self.vote_counts.entry(digest.clone()).or_insert_with(Default::default); + count.prepared += 1; - if count.0 == self.threshold { - Some(candidate) + if count.prepared == self.threshold { + Some(digest) } else { None } @@ -230,19 +234,19 @@ impl Accumulator fn import_commit( &mut self, - candidate: D, - sender: V, - signature: S, + digest: Digest, + sender: ValidatorId, + signature: Signature, ) { // ignore any subsequent commits by the same sender. // TODO: if digest is different, that's misbehavior. let committed_for = if let Entry::Vacant(vacant) = self.commits.entry(sender) { - vacant.insert((candidate.clone(), signature)); - let count = self.vote_counts.entry(candidate.clone()).or_insert((0, 0)); - count.1 += 1; + vacant.insert((digest.clone(), signature)); + let count = self.vote_counts.entry(digest.clone()).or_insert_with(Default::default); + count.committed += 1; - if count.1 == self.threshold { - Some(candidate) + if count.committed == self.threshold { + Some(digest) } else { None } @@ -271,7 +275,7 @@ impl Accumulator fn import_advance_round( &mut self, - sender: V, + sender: ValidatorId, ) { self.advance_round.insert(sender); diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index bad684820eeab..5c6b126a1e483 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -34,9 +34,9 @@ pub use self::accumulator::{Accumulator, Justification, PrepareJustification}; /// Messages over the proposal. /// Each message carries an associated round number. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Message { +pub enum Message { /// Send a full proposal. - Propose(usize, P), + Propose(usize, C), /// Prepare to vote for proposal with digest D. Prepare(usize, D), /// Commit to proposal with digest D.. @@ -45,7 +45,7 @@ pub enum Message { AdvanceRound(usize), } -impl Message { +impl Message { fn round_number(&self) -> usize { match *self { Message::Propose(round, _) => round, @@ -58,9 +58,9 @@ impl Message { /// A localized message, including the sender. #[derive(Debug, Clone)] -pub struct LocalizedMessage { +pub struct LocalizedMessage { /// The message received. - pub message: Message, + pub message: Message, /// The sender of the message pub sender: V, /// The signature of the message. @@ -68,6 +68,9 @@ pub struct LocalizedMessage { } /// Context necessary for agreement. +/// +/// Provides necessary types for protocol messages, and functions necessary for a +/// participant to evaluate and create those messages. pub trait Context { /// Candidate proposed. type Candidate: Debug + Eq + Clone; @@ -162,7 +165,7 @@ impl Sending { } } - while self.flushing { + if self.flushing { match sink.poll_complete() { Err(e) => return Err(e), Ok(Async::NotReady) => return Ok(Async::NotReady), diff --git a/candidate-agreement/src/bft/tests.rs b/candidate-agreement/src/bft/tests.rs index 2228ab2604540..165166d76a0de 100644 --- a/candidate-agreement/src/bft/tests.rs +++ b/candidate-agreement/src/bft/tests.rs @@ -92,13 +92,13 @@ impl SharedContext { } } -struct Ctx { +struct TestContext { local_id: ValidatorId, proposal: Mutex, shared: Arc>, } -impl Context for Ctx { +impl Context for TestContext { type Candidate = Candidate; type Digest = Digest; type ValidatorId = ValidatorId; @@ -149,7 +149,7 @@ impl Context for Ctx { } } -type Comm = ContextCommunication; +type Comm = ContextCommunication; struct Network { endpoints: Vec>, @@ -239,7 +239,7 @@ fn consensus_completes_with_minimum_good() { .take(node_count - max_faulty) .enumerate() .map(|(i, (tx, rx))| { - let ctx = Ctx { + let ctx = TestContext { local_id: ValidatorId(i), proposal: Mutex::new(i), shared: shared_context.clone(), @@ -295,7 +295,7 @@ fn consensus_does_not_complete_without_enough_nodes() { .take(node_count - max_faulty - 1) .enumerate() .map(|(i, (tx, rx))| { - let ctx = Ctx { + let ctx = TestContext { local_id: ValidatorId(i), proposal: Mutex::new(i), shared: shared_context.clone(), From e594f61b0e60e99a0a0bab905ece14029751008f Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 29 Dec 2017 14:40:47 +0100 Subject: [PATCH 33/40] address some more review nits --- candidate-agreement/src/bft/accumulator.rs | 26 +++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs index 42e5459392589..e457bef859a70 100644 --- a/candidate-agreement/src/bft/accumulator.rs +++ b/candidate-agreement/src/bft/accumulator.rs @@ -91,6 +91,10 @@ struct VoteCounts { } /// Accumulates messages for a given round of BFT consensus. +/// +/// This isn't tied to the "view" of a single validator. It +/// keeps accurate track of the state of the BFT consensus based +/// on all messages imported. #[derive(Debug)] pub struct Accumulator where @@ -197,12 +201,12 @@ impl Accumulator= self.threshold { Some(digest) } else { None @@ -217,16 +221,16 @@ impl Accumulator false, }; - if let (true, Some(prepared_for)) = (valid_transition, prepared_for) { + if let (true, Some(threshold_prepared)) = (valid_transition, threshold_prepared) { let signatures = self.prepares .values() - .filter(|&&(ref d, _)| d == &prepared_for) + .filter(|&&(ref d, _)| d == &threshold_prepared) .map(|&(_, ref s)| s.clone()) .collect(); self.state = State::Prepared(PrepareJustification { round_number: self.round_number, - digest: prepared_for, + digest: threshold_prepared, signatures: signatures, }); } @@ -240,12 +244,12 @@ impl Accumulator= self.threshold { Some(digest) } else { None @@ -258,16 +262,16 @@ impl Accumulator Accumulator Date: Fri, 29 Dec 2017 15:07:14 +0100 Subject: [PATCH 34/40] fix lock import logic and add a test --- candidate-agreement/src/bft/mod.rs | 15 ++--- candidate-agreement/src/bft/tests.rs | 88 ++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 7 deletions(-) diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index 5c6b126a1e483..6beff362e4672 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -308,16 +308,17 @@ impl Strategy { ) { // TODO: find a way to avoid processing of the signatures if the sender is // not the primary or the round number is low. - let current_round_number = self.current_accumulator.round_number(); - if justification.round_number < current_round_number { - return - } else if justification.round_number == current_round_number { - self.locked = Some(Locked { justification }); - } else { + if justification.round_number > self.current_accumulator.round_number() { // jump ahead to the prior round as this is an indication of a supermajority // good nodes being at least on that round. self.advance_to_round(context, justification.round_number); - self.locked = Some(Locked { justification }); + } + + let lock_to_new = self.locked.as_ref() + .map_or(true, |l| l.justification.round_number < justification.round_number); + + if lock_to_new { + self.locked = Some(Locked { justification }) } } diff --git a/candidate-agreement/src/bft/tests.rs b/candidate-agreement/src/bft/tests.rs index 165166d76a0de..86857ba5dee9f 100644 --- a/candidate-agreement/src/bft/tests.rs +++ b/candidate-agreement/src/bft/tests.rs @@ -322,3 +322,91 @@ fn consensus_does_not_complete_without_enough_nodes() { assert!(result.is_none(), "not enough online nodes"); } + +#[test] +fn threshold_plus_one_locked_on_proposal_only_one_with_candidate() { + let node_count = 10; + let max_faulty = 3; + + let locked_proposal = Candidate(999_999_999); + let locked_digest = Digest(999_999_999); + let locked_round = 1; + let justification = PrepareJustification { + round_number: locked_round, + digest: locked_digest.clone(), + signatures: (0..7) + .map(|i| Signature(Message::Prepare(locked_round, locked_digest.clone()), ValidatorId(i))) + .collect() + }; + + let mut shared_context = SharedContext::new(node_count); + shared_context.current_round = locked_round + 1; + let shared_context = Arc::new(Mutex::new(shared_context)); + + let (network, net_send, net_recv) = Network::new(node_count); + network.route_on_thread(); + + let nodes = net_send + .into_iter() + .zip(net_recv) + .enumerate() + .map(|(i, (tx, rx))| { + let ctx = TestContext { + local_id: ValidatorId(i), + proposal: Mutex::new(i), + shared: shared_context.clone(), + }; + + let mut agreement = agree( + ctx, + node_count, + max_faulty, + rx.map_err(|_| Error), + tx.sink_map_err(|_| Error).with(move |t| Ok((i, t))), + ); + + agreement.strategy.advance_to_round( + &agreement.context, + locked_round + 1 + ); + + if i <= max_faulty { + agreement.strategy.locked = Some(Locked { + justification: justification.clone(), + }) + } + + if i == max_faulty { + agreement.strategy.notable_candidates.insert( + locked_digest.clone(), + locked_proposal.clone(), + ); + } + + agreement + }) + .collect::>(); + + ::std::thread::spawn(move || { + let mut timeout = ::std::time::Duration::from_millis(50); + loop { + ::std::thread::sleep(timeout.clone()); + shared_context.lock().unwrap().bump_round(); + timeout *= 2; + } + }); + + let timeout = timeout_in(Duration::from_millis(500)).map_err(|_| Error); + let results = ::futures::future::join_all(nodes) + .map(Some) + .select(timeout.map(|_| None)) + .wait() + .map(|(i, _)| i) + .map_err(|(e, _)| e) + .expect("to complete") + .expect("to not time out"); + + for result in &results { + assert_eq!(&result.justification.digest, &locked_digest); + } +} From 48712d6eb79b2bae878f409442a181b27491be3c Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 29 Dec 2017 15:41:39 +0100 Subject: [PATCH 35/40] fix spaces --- candidate-agreement/src/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index f7dfb9766633c..568a5bff50ff7 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -38,7 +38,7 @@ pub trait Context { type ValidatorId: Hash + Eq + Clone + Debug; /// The digest (hash or other unique attribute) of a candidate. type Digest: Hash + Eq + Clone + Debug; - /// Candidate type. + /// Candidate type. type Candidate: Ord + Eq + Clone + Debug; /// The group ID type type GroupId: Hash + Ord + Eq + Clone + Debug; From 91fb3896c8f0839cba332d5856f871fd20609d0e Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Sun, 31 Dec 2017 20:10:37 +0100 Subject: [PATCH 36/40] fix a couple more style grumbles --- candidate-agreement/src/bft/mod.rs | 4 ++-- candidate-agreement/src/bft/tests.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index 6beff362e4672..3979e0751fe92 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -492,11 +492,11 @@ impl Strategy { Some(_) => { // don't check validity if we are locked. // this is necessary to preserve the liveness property. - prepare_for = Some(digest) + prepare_for = Some(digest); } None => if context.candidate_valid(candidate) { prepare_for = Some(digest); - }, + } } } diff --git a/candidate-agreement/src/bft/tests.rs b/candidate-agreement/src/bft/tests.rs index 86857ba5dee9f..a5a5ead96e3ed 100644 --- a/candidate-agreement/src/bft/tests.rs +++ b/candidate-agreement/src/bft/tests.rs @@ -64,7 +64,7 @@ impl SharedContext { fn round_timeout(&mut self, round: usize) -> Box> { let (tx, rx) = oneshot::channel(); if round < self.current_round { - tx.send(()).unwrap() + tx.send(()).unwrap(); } else { self.awaiting_round_timeouts .entry(round) From 443d3ef052fcf9fac0d41af26de3e57dccf5b0ed Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 1 Jan 2018 20:00:44 +0100 Subject: [PATCH 37/40] more type-safe justifications --- candidate-agreement/src/bft/accumulator.rs | 75 ++++++++++++++-------- candidate-agreement/src/bft/mod.rs | 2 +- candidate-agreement/src/bft/tests.rs | 4 +- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/candidate-agreement/src/bft/accumulator.rs b/candidate-agreement/src/bft/accumulator.rs index e457bef859a70..8999a9f29bb00 100644 --- a/candidate-agreement/src/bft/accumulator.rs +++ b/candidate-agreement/src/bft/accumulator.rs @@ -23,8 +23,8 @@ use std::hash::Hash; use super::{Message, LocalizedMessage}; /// Justification for some state at a given round. -#[derive(PartialEq, Eq, Debug, Clone)] -pub struct Justification { +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UncheckedJustification { /// The round. pub round_number: usize, /// The digest prepared for. @@ -33,36 +33,61 @@ pub struct Justification { pub signatures: Vec, } -impl Justification { +impl UncheckedJustification { /// Fails if there are duplicate signatures or invalid. /// /// Provide a closure for checking whether the signature is valid on a /// digest. /// - /// The closure should return true iff the round number, digest, and signature + /// The closure should returns a checked justification iff the round number, digest, and signature /// represent a valid message and the signer was authorized to issue /// it. /// /// The `check_message` closure may vary based on context. - pub fn check(&self, threshold: usize, check_message: F) -> bool + pub fn check(self, threshold: usize, mut check_message: F) + -> Result, Self> where - F: Fn(usize, &D, &S) -> Option, + F: FnMut(usize, &D, &S) -> Option, V: Hash + Eq, { - let mut voted = HashSet::new(); - - for signature in &self.signatures { - match check_message(self.round_number, &self.digest, signature) { - None => return false, - Some(v) => { - if !voted.insert(v) { - return false; + let checks_out = { + let mut checks_out = || { + let mut voted = HashSet::new(); + + for signature in &self.signatures { + match check_message(self.round_number, &self.digest, signature) { + None => return false, + Some(v) => { + if !voted.insert(v) { + return false; + } + } } } - } + + voted.len() >= threshold + }; + + checks_out() + }; + + if checks_out { + Ok(Justification(self)) + } else { + Err(self) } + } +} + +/// A checked justification. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Justification(UncheckedJustification); - voted.len() >= threshold +impl ::std::ops::Deref for Justification { + type Target = UncheckedJustification; + + fn deref(&self) -> &Self::Target { + &self.0 } } @@ -228,11 +253,11 @@ impl Accumulator Accumulator Date: Mon, 1 Jan 2018 20:06:03 +0100 Subject: [PATCH 38/40] rename Communication enum variants --- candidate-agreement/src/bft/mod.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/candidate-agreement/src/bft/mod.rs b/candidate-agreement/src/bft/mod.rs index 054854be7cfc6..b17092c451412 100644 --- a/candidate-agreement/src/bft/mod.rs +++ b/candidate-agreement/src/bft/mod.rs @@ -114,9 +114,9 @@ pub trait Context { #[derive(Debug, Clone)] pub enum Communication { /// A consensus message (proposal or vote) - Message(LocalizedMessage), - /// A proof-of-lock. - Locked(PrepareJustification), + Consensus(LocalizedMessage), + /// Auxiliary communication (just proof-of-lock for now). + Auxiliary(PrepareJustification), } /// Type alias for a localized message using only type parameters from `Context`. @@ -461,7 +461,7 @@ impl Strategy { // broadcast the justification along with the proposal if we are locked. if let Some(ref locked) = self.locked { sending.push( - ContextCommunication(Communication::Locked(locked.justification.clone())) + ContextCommunication(Communication::Auxiliary(locked.justification.clone())) ); } @@ -610,7 +610,7 @@ impl Strategy { ) { let signed_message = context.sign_local(message); self.import_message(signed_message.clone()); - sending.push(ContextCommunication(Communication::Message(signed_message))); + sending.push(ContextCommunication(Communication::Consensus(signed_message))); } } @@ -657,8 +657,9 @@ impl Future for Agreement }; match message.0 { - Communication::Message(message) => self.strategy.import_message(message), - Communication::Locked(proof) => self.strategy.import_lock_proof(&self.context, proof), + Communication::Consensus(message) => self.strategy.import_message(message), + Communication::Auxiliary(lock_proof) + => self.strategy.import_lock_proof(&self.context, lock_proof), } } From 0951a3b1f1b797f1e838953c62ff1095be269bc4 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 1 Jan 2018 20:25:22 +0100 Subject: [PATCH 39/40] improve some panic guard proofs --- candidate-agreement/src/table.rs | 37 +++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 568a5bff50ff7..799d472c527ba 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -402,7 +402,7 @@ impl Table { // check that validator hasn't already specified another candidate. let digest = context.candidate_digest(&candidate); - match self.validator_data.entry(from.clone()) { + let new_proposal = match self.validator_data.entry(from.clone()) { Entry::Occupied(mut occ) => { // if digest is different, fetch candidate and // note misbehavior. @@ -410,8 +410,13 @@ impl Table { if let Some((ref old_digest, ref old_sig)) = existing.proposal { if old_digest != &digest { + const EXISTENCE_PROOF: &str = + "when proposal first received from validator, candidate \ + votes entry is created. proposal here is `Some`, therefore \ + candidate votes entry exists; qed"; + let old_candidate = self.candidate_votes.get(old_digest) - .expect("proposed digest implies existence of votes entry; qed") + .expect(EXISTENCE_PROOF) .candidate .clone(); @@ -423,8 +428,11 @@ impl Table { None, ); } + + false } else { existing.proposal = Some((digest.clone(), signature.clone())); + true } } Entry::Vacant(vacant) => { @@ -432,16 +440,20 @@ impl Table { proposal: Some((digest.clone(), signature.clone())), known_statements: HashSet::new(), }); - - // TODO: seed validity votes with issuer here? - self.candidate_votes.entry(digest.clone()).or_insert_with(move || CandidateData { - group_id: group, - candidate: candidate, - validity_votes: HashMap::new(), - availability_votes: HashMap::new(), - indicated_bad_by: Vec::new(), - }); + true } + }; + + // NOTE: altering this code may affect the existence proof above. ensure it remains + // valid. + if new_proposal { + self.candidate_votes.entry(digest.clone()).or_insert_with(move || CandidateData { + group_id: group, + candidate: candidate, + validity_votes: HashMap::new(), + availability_votes: HashMap::new(), + indicated_bad_by: Vec::new(), + }); } self.validity_vote( @@ -470,7 +482,8 @@ impl Table { ValidityVote::Valid(s) => (s, true), ValidityVote::Invalid(s) => (s, false), ValidityVote::Issued(_) => - panic!("implicit issuance vote only cast if the candidate entry already created successfully; qed"), + panic!("implicit issuance vote only cast from `import_candidate` after \ + checking group membership of issuer; qed"), }; return ( From 96cc399caa17cfbda7898ce7b9d8b50ac3d182ab Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 1 Jan 2018 20:36:54 +0100 Subject: [PATCH 40/40] add trailing comma --- candidate-agreement/src/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/candidate-agreement/src/table.rs b/candidate-agreement/src/table.rs index 799d472c527ba..381244e58b6d6 100644 --- a/candidate-agreement/src/table.rs +++ b/candidate-agreement/src/table.rs @@ -353,7 +353,7 @@ impl Table { signer.clone(), digest, signature, - ) + ), }; if let Some(misbehavior) = maybe_misbehavior {