diff --git a/.config/forest.dic b/.config/forest.dic index 37f2aa4d3422..df7e0c80d345 100644 --- a/.config/forest.dic +++ b/.config/forest.dic @@ -55,6 +55,7 @@ milliGas multihash multisig mutex +overallocation P2P parsable pointer/SM diff --git a/src/blocks/tipset.rs b/src/blocks/tipset.rs index dd3312d7ebc4..d8563acddb6c 100644 --- a/src/blocks/tipset.rs +++ b/src/blocks/tipset.rs @@ -3,6 +3,7 @@ use std::{fmt, sync::OnceLock}; +use crate::ipld::FrozenCids; use crate::networks::{calibnet, mainnet}; use crate::shim::{address::Address, clock::ChainEpoch}; use crate::utils::cid::CidCborExt; @@ -26,35 +27,36 @@ use super::{Block, BlockHeader, Error, Ticket}; #[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))] #[serde(transparent)] pub struct TipsetKeys { - pub cids: Vec, + pub cids: FrozenCids, } impl TipsetKeys { - pub fn new(cids: Vec) -> Self { + pub fn new(cids: FrozenCids) -> Self { Self { cids } } - /// Returns tipset header `cids` - pub fn cids(&self) -> &[Cid] { - &self.cids - } - // Special encoding to match Lotus. pub fn cid(&self) -> anyhow::Result { use fvm_ipld_encoding::RawBytes; let mut bytes = Vec::new(); - for cid in self.cids() { + for cid in &self.cids { bytes.append(&mut cid.to_bytes()) } Ok(Cid::from_cbor_blake2b256(&RawBytes::new(bytes))?) } } +impl From> for TipsetKeys { + fn from(cids: Vec) -> Self { + Self::new(FrozenCids::from(cids)) + } +} + impl fmt::Display for TipsetKeys { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let s = self - .cids() - .iter() + .cids + .into_iter() .map(|cid| cid.to_string()) .collect::>() .join(", "); @@ -141,9 +143,9 @@ impl Tipset { /// present but invalid. If the tipset is missing, None is returned. pub fn load(store: impl Blockstore, tsk: &TipsetKeys) -> anyhow::Result> { Ok(tsk - .cids() - .iter() - .map(|key| BlockHeader::load(&store, *key)) + .cids + .into_iter() + .map(|key| BlockHeader::load(&store, key)) .collect::>>()? .map(Tipset::new) .transpose()?) @@ -220,8 +222,8 @@ impl Tipset { }) } /// Returns slice of `CIDs` for the current tipset - pub fn cids(&self) -> &[Cid] { - self.key().cids() + pub fn cids(&self) -> Vec { + Vec::::from(&self.key().cids) } /// Returns the keys of the parents of the blocks in the tipset. pub fn parents(&self) -> &TipsetKeys { @@ -447,7 +449,7 @@ pub mod tipset_keys_json { where S: Serializer, { - crate::json::cid::vec::serialize(m.cids(), serializer) + crate::json::cid::vec::serialize(&Vec::::from(&m.cids), serializer) } pub fn deserialize<'de, D>(deserializer: D) -> Result @@ -455,7 +457,7 @@ pub mod tipset_keys_json { D: Deserializer<'de>, { Ok(TipsetKeys { - cids: crate::json::cid::vec::deserialize(deserializer)?, + cids: crate::json::cid::vec::deserialize(deserializer)?.into(), }) } } @@ -563,6 +565,7 @@ mod property_tests { #[cfg(test)] mod test { use crate::blocks::VRFProof; + use crate::ipld::FrozenCids; use crate::shim::address::Address; use cid::{ multihash::{Code::Identity, MultihashDigest}, @@ -712,10 +715,10 @@ mod test { .unwrap(); let h1 = BlockHeader::builder() .miner_address(Address::new_id(1)) - .parents(TipsetKeys::new(vec![Cid::new_v1( + .parents(TipsetKeys::new(FrozenCids::from_iter([Cid::new_v1( DAG_CBOR, Identity.digest(&[]), - )])) + )]))) .build() .unwrap(); assert_eq!( diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 605161b61bcb..f6251d2b3386 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -8,6 +8,7 @@ use crate::ipld::{stream_chain, CidHashSet}; use crate::utils::io::{AsyncWriterWithChecksum, Checksum}; use crate::utils::stream::par_buffer; use anyhow::{Context, Result}; +use cid::Cid; use digest::Digest; use fvm_ipld_blockstore::Blockstore; use std::sync::Arc; @@ -25,7 +26,7 @@ pub async fn export( ) -> Result>, Error> { let db = Arc::new(db); let stateroot_lookup_limit = tipset.epoch() - lookup_depth; - let roots = tipset.key().cids().to_vec(); + let roots = Vec::::from(&tipset.key().cids); // Wrap writer in optional checksum calculator let mut writer = AsyncWriterWithChecksum::::new(BufWriter::new(writer), !skip_checksum); diff --git a/src/chain/store/chain_store.rs b/src/chain/store/chain_store.rs index 45811b018dac..e9eecac525bc 100644 --- a/src/chain/store/chain_store.rs +++ b/src/chain/store/chain_store.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use crate::blocks::{BlockHeader, Tipset, TipsetKeys, TxMeta}; use crate::interpreter::BlockMessages; +use crate::ipld::FrozenCids; use crate::libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite}; use crate::message::{ChainMessage, Message as MessageTrait, SignedMessage}; use crate::networks::ChainConfig; @@ -114,7 +115,7 @@ where .read_obj::(HEAD_KEY)? .is_some_and(|tipset_keys| chain_index.load_tipset(&tipset_keys).is_ok()) { - let tipset_keys = TipsetKeys::new(vec![*genesis_block_header.cid()]); + let tipset_keys = TipsetKeys::new(FrozenCids::from_iter([*genesis_block_header.cid()])); settings.write_obj(HEAD_KEY, &tipset_keys)?; } @@ -206,7 +207,7 @@ where /// Returns Tipset from key-value store from provided CIDs #[tracing::instrument(skip_all)] pub fn tipset_from_keys(&self, tsk: &TipsetKeys) -> Result, Error> { - if tsk.cids().is_empty() { + if tsk.cids.is_empty() { return Ok(self.heaviest_tipset()); } self.chain_index.load_tipset(tsk) diff --git a/src/chain_sync/chain_muxer.rs b/src/chain_sync/chain_muxer.rs index 965c69fbb3fb..f5a4b3181667 100644 --- a/src/chain_sync/chain_muxer.rs +++ b/src/chain_sync/chain_muxer.rs @@ -259,7 +259,7 @@ where if network.peer_manager().is_peer_new(&peer_id).await { // Since the peer is new, send them a hello request let request = HelloRequest { - heaviest_tip_set: heaviest.cids().to_vec(), + heaviest_tip_set: heaviest.cids(), heaviest_tipset_height: heaviest.epoch(), heaviest_tipset_weight: heaviest.weight().clone().into(), genesis_cid: genesis_block_cid, @@ -372,7 +372,7 @@ where metrics::LIBP2P_MESSAGE_TOTAL .with_label_values(&[metrics::values::HELLO_RESPONSE_OUTBOUND]) .inc(); - let tipset_keys = TipsetKeys::new(request.heaviest_tip_set); + let tipset_keys = TipsetKeys::from(request.heaviest_tip_set); let tipset = match Self::get_full_tipset( network.clone(), chain_store.clone(), diff --git a/src/chain_sync/network_context.rs b/src/chain_sync/network_context.rs index a745b5b84f53..9c84b1f6d930 100644 --- a/src/chain_sync/network_context.rs +++ b/src/chain_sync/network_context.rs @@ -225,7 +225,7 @@ where T: TryFrom + Send + Sync + 'static, { let request = ChainExchangeRequest { - start: tsk.cids().to_vec(), + start: Vec::::from(&tsk.cids), request_len, options, }; diff --git a/src/chain_sync/tipset_syncer.rs b/src/chain_sync/tipset_syncer.rs index b7a1ab7cac47..43cece0ed336 100644 --- a/src/chain_sync/tipset_syncer.rs +++ b/src/chain_sync/tipset_syncer.rs @@ -882,7 +882,7 @@ async fn sync_headers_in_reverse( tipset: &TipsetKeys, descendant_blocks: &[Cid], ) -> Result<(), TipsetRangeSyncerError> { - for cid in tipset.cids() { - if let Some(reason) = bad_block_cache.get(cid) { + for cid in &tipset.cids { + if let Some(reason) = bad_block_cache.get(&cid) { for block_cid in descendant_blocks { bad_block_cache.put(*block_cid, format!("chain contained {cid}")); } - return Err(TipsetRangeSyncerError::TipsetRangeWithBadBlock( - *cid, reason, - )); + return Err(TipsetRangeSyncerError::TipsetRangeWithBadBlock(cid, reason)); } } Ok(()) diff --git a/src/cli/subcommands/chain_cmd.rs b/src/cli/subcommands/chain_cmd.rs index c8df506590c1..3f39457c3c91 100644 --- a/src/cli/subcommands/chain_cmd.rs +++ b/src/cli/subcommands/chain_cmd.rs @@ -91,12 +91,9 @@ impl ChainCommands { force: no_confirm, } => { maybe_confirm(*no_confirm, SET_HEAD_CONFIRMATION_MESSAGE)?; - chain_set_head( - (TipsetKeys { cids: cids.clone() },), - &config.client.rpc_token, - ) - .await - .map_err(handle_rpc_err) + chain_set_head((TipsetKeys::from(cids.clone()),), &config.client.rpc_token) + .await + .map_err(handle_rpc_err) } } } diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 553e512826df..8c523836c344 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -132,7 +132,7 @@ impl ForestCar { } pub fn heaviest_tipset(&self) -> anyhow::Result { - Tipset::load_required(self, &TipsetKeys::new(self.roots())) + Tipset::load_required(self, &TipsetKeys::from(self.roots())) } pub fn into_dyn(self) -> ForestCar> { diff --git a/src/db/car/plain.rs b/src/db/car/plain.rs index dfdee0124bf5..3f4ae36c0c0c 100644 --- a/src/db/car/plain.rs +++ b/src/db/car/plain.rs @@ -156,7 +156,7 @@ impl PlainCar { } pub fn heaviest_tipset(&self) -> anyhow::Result { - Tipset::load_required(self, &TipsetKeys::new(self.roots())) + Tipset::load_required(self, &TipsetKeys::from(self.roots())) } /// In an arbitrary order diff --git a/src/genesis/mod.rs b/src/genesis/mod.rs index 72fd348c4802..3836bc870cf5 100644 --- a/src/genesis/mod.rs +++ b/src/genesis/mod.rs @@ -117,7 +117,7 @@ where sm.chain_store().set_estimated_records(n_records as u64)?; } - let ts = sm.chain_store().tipset_from_keys(&TipsetKeys::new(cids))?; + let ts = sm.chain_store().tipset_from_keys(&TipsetKeys::from(cids))?; if !skip_load { let gb = sm.chain_store().chain_index.tipset_by_height( diff --git a/src/interpreter/fvm2.rs b/src/interpreter/fvm2.rs index 0283bed17b63..d81bfc53fb61 100644 --- a/src/interpreter/fvm2.rs +++ b/src/interpreter/fvm2.rs @@ -221,8 +221,8 @@ impl Consensus for ForestExternsV2 { let bh_3 = from_slice_with_fallback::(extra)?; if bh_1.parents() == bh_3.parents() && bh_1.epoch() == bh_3.epoch() - && bh_2.parents().cids().contains(bh_3.cid()) - && !bh_2.parents().cids().contains(bh_1.cid()) + && bh_2.parents().cids.contains(*bh_3.cid()) + && !bh_2.parents().cids.contains(*bh_1.cid()) { fault_type = Some(ConsensusFaultType::ParentGrinding); } diff --git a/src/interpreter/fvm3.rs b/src/interpreter/fvm3.rs index 43323b7174cf..9a98de6a2c38 100644 --- a/src/interpreter/fvm3.rs +++ b/src/interpreter/fvm3.rs @@ -242,8 +242,8 @@ impl Consensus for ForestExterns { let bh_3 = from_slice_with_fallback::(extra)?; if bh_1.parents() == bh_3.parents() && bh_1.epoch() == bh_3.epoch() - && bh_2.parents().cids().contains(bh_3.cid()) - && !bh_2.parents().cids().contains(bh_1.cid()) + && bh_2.parents().cids.contains(*bh_3.cid()) + && !bh_2.parents().cids.contains(*bh_1.cid()) { fault_type = Some(ConsensusFaultType::ParentGrinding); } diff --git a/src/ipld/cid_hashmap.rs b/src/ipld/cid_hashmap.rs index a3d8e83a84f2..9cb7314c2e1a 100644 --- a/src/ipld/cid_hashmap.rs +++ b/src/ipld/cid_hashmap.rs @@ -33,32 +33,30 @@ impl CidHashMap { /// Returns `true` if the map contains a value for the specified key. pub fn contains_key(&self, k: Cid) -> bool { - match k.try_into() { - Ok(CidVariant::V1DagCborBlake2b(bytes)) => { + match k.into() { + CidVariant::V1DagCborBlake2b(bytes) => { self.v1_dagcbor_blake2b_hash_map.contains_key(&bytes) } - Err(()) => self.fallback_hash_map.contains_key(&k), + CidVariant::Generic(_) => self.fallback_hash_map.contains_key(&k), } } /// Inserts a key-value pair into the map; if the map did not have this key present, [`None`] is returned. pub fn insert(&mut self, k: Cid, v: V) -> Option { - match k.try_into() { - Ok(CidVariant::V1DagCborBlake2b(bytes)) => { + match k.into() { + CidVariant::V1DagCborBlake2b(bytes) => { self.v1_dagcbor_blake2b_hash_map.insert(bytes, v) } - Err(()) => self.fallback_hash_map.insert(k, v), + CidVariant::Generic(_) => self.fallback_hash_map.insert(k, v), } } /// Removes a key from the map, returning the value at the key if the key /// was previously in the map. pub fn remove(&mut self, k: Cid) -> Option { - match k.try_into() { - Ok(CidVariant::V1DagCborBlake2b(bytes)) => { - self.v1_dagcbor_blake2b_hash_map.remove(&bytes) - } - Err(()) => self.fallback_hash_map.remove(&k), + match k.into() { + CidVariant::V1DagCborBlake2b(bytes) => self.v1_dagcbor_blake2b_hash_map.remove(&bytes), + CidVariant::Generic(_) => self.fallback_hash_map.remove(&k), } } @@ -69,9 +67,9 @@ impl CidHashMap { /// Returns a reference to the value corresponding to the key. pub fn get(&self, k: Cid) -> Option<&V> { - match k.try_into() { - Ok(CidVariant::V1DagCborBlake2b(bytes)) => self.v1_dagcbor_blake2b_hash_map.get(&bytes), - Err(()) => self.fallback_hash_map.get(&k), + match k.into() { + CidVariant::V1DagCborBlake2b(bytes) => self.v1_dagcbor_blake2b_hash_map.get(&bytes), + CidVariant::Generic(_) => self.fallback_hash_map.get(&k), } } diff --git a/src/ipld/frozen_cids.rs b/src/ipld/frozen_cids.rs new file mode 100644 index 000000000000..175a7151cd68 --- /dev/null +++ b/src/ipld/frozen_cids.rs @@ -0,0 +1,127 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::utils::cid::CidVariant; +use cid::Cid; +use serde::{Deserialize, Serialize}; + +/// Similar to the `CidHashMap` implementation, `FrozenCids` optimizes storage of +/// CIDs that would normally be stored as a vector of CIDs. The `V1 DAG-CBOR Blake2b-256` +/// variant (which can be stored in 32 bytes vs 96 bytes for a `Cid` +/// type) is `+99.99%` of all CIDs, so very few CIDs need to be stored in the +/// `Generic(Box)` variant of `CidVariant`. +/// +/// We use `Box<[...]>` to save memory, avoiding vector overallocation. +#[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))] +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct FrozenCids( + #[cfg_attr(test, arbitrary(gen( + |g| Box::new([CidVariant::arbitrary(g)])) +))] + Box<[CidVariant]>, +); + +impl Default for FrozenCids { + fn default() -> Self { + FrozenCids(Box::new([])) + } +} + +pub struct Iter<'a> { + cids: std::slice::Iter<'a, CidVariant>, +} + +impl<'a> IntoIterator for &'a FrozenCids { + type Item = Cid; + type IntoIter = Iter<'a>; + fn into_iter(self) -> Self::IntoIter { + Iter { + cids: self.0.iter(), + } + } +} + +impl<'a> Iterator for Iter<'a> { + type Item = Cid; + fn next(&mut self) -> Option { + self.cids.next().map(Cid::from) + } +} + +impl FromIterator for FrozenCids { + fn from_iter>(iter: T) -> Self { + FrozenCids::from(iter.into_iter().collect::>()) + } +} + +impl From> for FrozenCids { + fn from(cids: Vec) -> Self { + let mut small_cids = Vec::with_capacity(cids.len()); + for cid in cids { + match cid.into() { + CidVariant::V1DagCborBlake2b(bytes) => { + small_cids.push(CidVariant::V1DagCborBlake2b(bytes)) + } + _ => small_cids.push(CidVariant::Generic(Box::new(cid))), + } + } + FrozenCids(small_cids.into_boxed_slice()) + } +} + +impl From for Vec { + fn from(frozen_cids: FrozenCids) -> Self { + Vec::::from(&frozen_cids) + } +} + +impl From<&FrozenCids> for Vec { + fn from(frozen_cids: &FrozenCids) -> Self { + let mut cids = Vec::with_capacity(frozen_cids.0.len()); + for cid in frozen_cids.into_iter() { + match cid.into() { + CidVariant::V1DagCborBlake2b(bytes) => { + cids.push(Cid::from(CidVariant::V1DagCborBlake2b(bytes))) + } + _ => cids.push(cid), + } + } + cids + } +} + +impl FrozenCids { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn contains(&self, cid: Cid) -> bool { + let cid = CidVariant::from(cid); + self.0.contains(&cid) + } +} + +#[cfg(test)] +mod test { + use super::*; + use quickcheck_macros::quickcheck; + + #[quickcheck] + fn cidvec_to_vec_of_cids_to_cidvec(cidvec: FrozenCids) { + assert_eq!(cidvec, FrozenCids::from(Vec::::from(cidvec.clone()))); + } + + #[quickcheck] + fn serialize_vec_of_cids_deserialize_cidvec(vec_of_cids: Vec) { + let serialized = serde_json::to_string(&vec_of_cids).unwrap(); + let parsed: FrozenCids = serde_json::from_str(&serialized).unwrap(); + assert_eq!(vec_of_cids, Vec::::from(parsed)); + } + + #[quickcheck] + fn serialize_cidvec_deserialize_vec_of_cids(cidvec: FrozenCids) { + let serialized = serde_json::to_string(&cidvec).unwrap(); + let parsed: Vec = serde_json::from_str(&serialized).unwrap(); + assert_eq!(Vec::::from(cidvec), parsed); + } +} diff --git a/src/ipld/mod.rs b/src/ipld/mod.rs index 49f5e4b901e3..a587751e76f4 100644 --- a/src/ipld/mod.rs +++ b/src/ipld/mod.rs @@ -3,6 +3,7 @@ mod cid_hashmap; mod cid_hashset; +mod frozen_cids; pub mod json; pub mod selector; pub mod util; @@ -13,6 +14,7 @@ pub use util::*; pub use self::cid_hashmap::CidHashMap; pub use self::cid_hashset::CidHashSet; +pub use self::frozen_cids::FrozenCids; pub use libipld_core::serde::{from_ipld, to_ipld}; #[cfg(test)] diff --git a/src/ipld/util.rs b/src/ipld/util.rs index 92f3caf4afb6..289bc290e756 100644 --- a/src/ipld/util.rs +++ b/src/ipld/util.rs @@ -128,7 +128,7 @@ where let wp = WithProgressRaw::new(message, estimated_total_records); let mut seen = CidHashSet::default(); - let mut blocks_to_walk: VecDeque = tipset.cids().to_vec().into(); + let mut blocks_to_walk: VecDeque = tipset.cids().into(); let mut current_min_height = tipset.epoch(); let incl_roots_epoch = tipset.epoch() - recent_roots; @@ -171,12 +171,12 @@ where } if h.epoch() > 0 { - for p in h.parents().cids() { - blocks_to_walk.push_back(*p); + for p in &h.parents().cids { + blocks_to_walk.push_back(p); } } else { - for p in h.parents().cids() { - load_block(*p).await?; + for p in &h.parents().cids { + load_block(p).await?; } } @@ -393,8 +393,8 @@ impl + Unpin> Stream for ChainStream< if block.epoch() == 0 { // The genesis block has some kind of dummy parent that needs to be emitted. - for p in block.parents().cids() { - this.dfs.push_back(Emit(*p)); + for p in &block.parents().cids { + this.dfs.push_back(Emit(p)); } } diff --git a/src/libp2p/chain_exchange/provider.rs b/src/libp2p/chain_exchange/provider.rs index 3126af4e0f7f..bbff39343b4d 100644 --- a/src/libp2p/chain_exchange/provider.rs +++ b/src/libp2p/chain_exchange/provider.rs @@ -27,7 +27,7 @@ where loop { let mut tipset_bundle: TipsetBundle = TipsetBundle::default(); - let tipset = match cs.tipset_from_keys(&TipsetKeys::new(curr_tipset_cids)) { + let tipset = match cs.tipset_from_keys(&TipsetKeys::from(curr_tipset_cids)) { Ok(tipset) => tipset, Err(err) => { debug!("Cannot get tipset from keys: {}", err); @@ -55,7 +55,7 @@ where } } - curr_tipset_cids = tipset.parents().cids().to_vec(); + curr_tipset_cids = Vec::::from(&tipset.parents().cids); let tipset_epoch = tipset.epoch(); if request.include_blocks() { diff --git a/src/lotus_json/tipset_keys.rs b/src/lotus_json/tipset_keys.rs index 245b3f15bfbf..c454bf5dc6d5 100644 --- a/src/lotus_json/tipset_keys.rs +++ b/src/lotus_json/tipset_keys.rs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::blocks::TipsetKeys; +use crate::ipld::FrozenCids; +use ::cid::Cid; use super::*; @@ -15,7 +17,7 @@ impl HasLotusJson for TipsetKeys { vec![( json!([{"/": "baeaaaaa"}]), TipsetKeys { - cids: vec![::cid::Cid::default()], + cids: FrozenCids::from(vec![::cid::Cid::default()]), }, )] } @@ -24,13 +26,13 @@ impl HasLotusJson for TipsetKeys { impl From for TipsetKeysLotusJson { fn from(value: TipsetKeys) -> Self { let TipsetKeys { cids } = value; - Self(cids.into()) + Self(VecLotusJson::::from(Vec::::from(cids))) } } impl From for TipsetKeys { fn from(value: TipsetKeysLotusJson) -> Self { let TipsetKeysLotusJson(cids) = value; - Self { cids: cids.into() } + Self { cids: FrozenCids::from(Vec::::from(cids)) } } } diff --git a/src/message_pool/msgpool/test_provider.rs b/src/message_pool/msgpool/test_provider.rs index 1f2b011b17d3..3bfae732a77a 100644 --- a/src/message_pool/msgpool/test_provider.rs +++ b/src/message_pool/msgpool/test_provider.rs @@ -203,7 +203,7 @@ impl Provider for TestApi { fn load_tipset(&self, tsk: &TipsetKeys) -> Result, Error> { let inner = self.inner.lock(); for ts in &inner.tipsets { - if tsk.cids == ts.cids() { + if tsk == ts.key() { return Ok(ts.clone().into()); } } diff --git a/src/rpc/chain_api.rs b/src/rpc/chain_api.rs index 32754da81eb4..e13661571c75 100644 --- a/src/rpc/chain_api.rs +++ b/src/rpc/chain_api.rs @@ -256,10 +256,10 @@ where let new_head = data.state_manager.chain_store().tipset_from_keys(¶ms)?; let mut current = data.state_manager.chain_store().heaviest_tipset(); while current.epoch() >= new_head.epoch() { - for cid in current.key().cids() { + for cid in ¤t.key().cids { data.state_manager .chain_store() - .unmark_block_as_validated(cid); + .unmark_block_as_validated(&cid); } let parents = current.blocks()[0].parents(); current = data.state_manager.chain_store().tipset_from_keys(parents)?; diff --git a/src/rpc/mpool_api.rs b/src/rpc/mpool_api.rs index f299227df3e8..e058feafdf02 100644 --- a/src/rpc/mpool_api.rs +++ b/src/rpc/mpool_api.rs @@ -28,7 +28,7 @@ where DB: Blockstore + Send + Sync + 'static, { let (CidJsonVec(cid_vec),) = params; - let tsk = TipsetKeys::new(cid_vec); + let tsk = TipsetKeys::new(cid_vec.into()); let mut ts = data.state_manager.chain_store().tipset_from_keys(&tsk)?; let (mut pending, mpts) = data.mpool.pending()?; diff --git a/src/rpc/sync_api.rs b/src/rpc/sync_api.rs index 5551320b9ac6..b64331e173c1 100644 --- a/src/rpc/sync_api.rs +++ b/src/rpc/sync_api.rs @@ -97,7 +97,7 @@ mod tests { .unwrap(); let cs_arc = Arc::new( - ChainStore::new(db.clone(), db, chain_config.clone(), genesis_header.clone()).unwrap(), + ChainStore::new(db.clone(), db, chain_config.clone(), genesis_header).unwrap(), ); let state_manager = Arc::new(StateManager::new(cs_arc.clone(), chain_config).unwrap()); @@ -110,10 +110,12 @@ mod tests { let header = from_slice_with_fallback::(&bz).unwrap(); let ts = Tipset::from(header); let db = cs_for_test.blockstore(); - let tsk = ts.key().cids.clone(); - cs_for_test.set_heaviest_tipset(Arc::new(ts)).unwrap(); + let tsk = ts.key(); + cs_for_test + .set_heaviest_tipset(Arc::new(ts.clone())) + .unwrap(); - for i in tsk { + for i in tsk.cids.into_iter() { let bz2 = bz.clone(); db.put_keyed(&i, &bz2).unwrap(); } diff --git a/src/tool/subcommands/benchmark_cmd.rs b/src/tool/subcommands/benchmark_cmd.rs index 6b73ee83d58b..122bf5ce5771 100644 --- a/src/tool/subcommands/benchmark_cmd.rs +++ b/src/tool/subcommands/benchmark_cmd.rs @@ -11,6 +11,7 @@ use crate::shim::clock::ChainEpoch; use crate::utils::db::car_stream::CarStream; use crate::utils::stream::par_buffer; use anyhow::{Context as _, Result}; +use cid::Cid; use clap::Subcommand; use futures::{StreamExt, TryStreamExt}; use indicatif::{ProgressBar, ProgressStyle}; @@ -183,7 +184,8 @@ async fn benchmark_exporting( compression_level, par_buffer(1024, blocks.map_err(anyhow::Error::from)), ); - crate::db::car::forest::Encoder::write(&mut dest, ts.key().cids.clone(), frames).await?; + crate::db::car::forest::Encoder::write(&mut dest, Vec::::from(&ts.key().cids), frames) + .await?; dest.flush().await?; Ok(()) } diff --git a/src/utils/cid/mod.rs b/src/utils/cid/mod.rs index 439d14431a71..bc466749443e 100644 --- a/src/utils/cid/mod.rs +++ b/src/utils/cid/mod.rs @@ -2,10 +2,13 @@ // SPDX-License-Identifier: Apache-2.0, MIT use cid::{ - multihash::{Code, MultihashDigest}, + multihash::{self, Code, Code::Blake2b256, MultihashDigest}, Cid, Version, }; use fvm_ipld_encoding::{Error, DAG_CBOR}; +#[cfg(test)] +use quickcheck::Arbitrary; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; /// Extension methods for constructing `dag-cbor` [Cid] pub trait CidCborExt { @@ -28,30 +31,75 @@ impl CidCborExt for Cid {} pub const BLAKE2B256_SIZE: usize = 32; -// `CidVariant` is an enumeration of known CID types that are used in the Filecoin blockchain. CIDs -// contain a significant amount of static data (such as version, codec, hash identifier, hash -// length). This static data represented by a single tag in the enum. -// -// Nearly all Filecoin CIDs are V1, DagCbor encoded, and hashed with Blake2b256 (which has a hash -// length of 256bits). Naively representing such a CID requires 96 bytes but `CidVariant` does it in -// only 40 bytes. If other types of CID become popular, they can be added to the CidVariant -// structure. +/// `CidVariant` is an enumeration of known CID types that are used in the Filecoin blockchain. CIDs +/// contain a significant amount of static data (such as version, codec, hash identifier, hash +/// length). This static data represented by a single tag in the `enum`. +/// +/// Nearly all Filecoin CIDs are `V1`,`DagCbor` encoded, and hashed with `Blake2b256` (which has a hash +/// length of 256 bits). Naively representing such a CID requires 96 bytes but `CidVariant` does it in +/// only 40 bytes. If other types of CID become popular, they can be added to the `CidVariant` +/// structure. +/// +/// The `Generic` variant is used for CIDs that do not fit into the other variants. +/// These variants are used for optimizing storage of CIDs in the `FrozenCids` structure. +#[cfg_attr(test, derive(derive_quickcheck_arbitrary::Arbitrary))] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum CidVariant { - V1DagCborBlake2b([u8; BLAKE2B256_SIZE]), + Generic(Box), + V1DagCborBlake2b( + #[cfg_attr(test, arbitrary(gen(|g: &mut quickcheck::Gen| std::array::from_fn(|_ix| Arbitrary::arbitrary(g)))))] + [u8; BLAKE2B256_SIZE], + ), +} + +impl Serialize for CidVariant { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + Cid::from(self).serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for CidVariant { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(Self::from(Cid::deserialize(deserializer)?)) + } } -impl TryFrom for CidVariant { - type Error = (); - fn try_from(cid: Cid) -> Result { +impl From for CidVariant { + fn from(cid: Cid) -> Self { if cid.version() == Version::V1 && cid.codec() == DAG_CBOR { if let Ok(small_hash) = cid.hash().resize() { let (code, bytes, size) = small_hash.into_inner(); if code == u64::from(Code::Blake2b256) && size as usize == BLAKE2B256_SIZE { - return Ok(CidVariant::V1DagCborBlake2b(bytes)); + return CidVariant::V1DagCborBlake2b(bytes); } } } - Err(()) + CidVariant::Generic(Box::new(cid)) + } +} + +impl From for Cid { + fn from(variant: CidVariant) -> Self { + Cid::from(&variant) + } +} + +impl From<&CidVariant> for Cid { + fn from(variant: &CidVariant) -> Self { + match variant { + CidVariant::Generic(cid) => **cid, + CidVariant::V1DagCborBlake2b(digest) => Cid::new_v1( + DAG_CBOR, + multihash::Multihash::wrap(Blake2b256.into(), digest) + .expect("failed to convert Blake2b digest to V1 DAG-CBOR Blake2b CID"), + ), + } } }