From 73fbc98d70eb779d02cd259b9f3a0209c0ff8960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 17 Sep 2022 14:37:27 +0200 Subject: [PATCH 01/15] Save --- frame/support/src/lib.rs | 107 +++++++++++++++++++++++ frame/support/src/storage/types/value.rs | 4 + frame/system/src/lib.rs | 6 +- 3 files changed, 114 insertions(+), 3 deletions(-) diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index e9578cccc23bc..0188a14ea725b 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2580,3 +2580,110 @@ pub mod pallet_prelude { /// ``` /// * use the newest nightly possible. pub use frame_support_procedural::pallet; + +use sp_std::vec::Vec; + +pub trait StreamRead { + type Iterator: sp_std::iter::Iterator; + + fn stream(key: Vec) -> Self::Iterator; +} + +pub struct StreamReadVec { + marker: sp_std::marker::PhantomData, + input: StorageInput, + length: u32, + read: u32, +} + +impl StreamReadVec { + pub fn new(key: Vec) -> Result { + let mut input = StorageInput::new(key)?; + let length = if input.exists { codec::Compact::::decode(&mut input)?.0 } else { 0 }; + + Ok(Self { marker: sp_std::marker::PhantomData, input, length, read: 0 }) + } +} + +impl sp_std::iter::Iterator for StreamReadVec { + type Item = T; + + fn next(&mut self) -> Option { + if self.read >= self.length { + None + } else { + self.read += 1; + + codec::Decode::decode(&mut self.input).ok() + } + } +} + +impl StreamRead for Vec { + type Iterator = StreamReadVec; + + fn stream(key: Vec) -> Self::Iterator { + StreamReadVec::new(key).unwrap() + } +} + +pub struct StorageInput { + key: Vec, + offset: u32, + total_length: u32, + exists: bool, + buffer: Vec, + buffer_pos: usize, +} + +impl StorageInput { + pub fn new(key: Vec) -> Result { + let (total_length, exists) = if let Some(len) = sp_io::storage::read(&key, &mut [], 0) { + (len, true) + } else { + (0, false) + }; + + Ok(Self { total_length, offset: 0, key, exists, buffer: Vec::with_capacity(2048), buffer_pos: 0 }) + } + + fn fill_buffer(&mut self) { + self.buffer.copy_within(self.buffer_pos.., 0); + unsafe { + self.buffer.set_len(self.buffer.capacity()); + } + + if let Some(read) = + sp_io::storage::read(&self.key, &mut self.buffer[self.buffer_pos..], self.offset) + { + unsafe { + self.buffer.set_len(sp_std::cmp::min( + self.buffer_pos + read as usize, + self.buffer.len() - self.buffer_pos, + )); + } + + self.offset += read; + self.buffer_pos = 0; + } + } +} + +impl codec::Input for StorageInput { + fn remaining_len(&mut self) -> Result, codec::Error> { + Ok(Some(self.total_length.saturating_sub(self.offset) as usize)) + } + + fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { + if into.len() > self.buffer.capacity() { + unimplemented!() + } else if self.buffer_pos + into.len() > self.buffer.len() { + self.fill_buffer(); + } + + let end = self.buffer_pos + into.len(); + into[..].copy_from_slice(&self.buffer[self.buffer_pos..end]); + + Ok(()) + } +} diff --git a/frame/support/src/storage/types/value.rs b/frame/support/src/storage/types/value.rs index f145e9fb30414..d2411dced957d 100644 --- a/frame/support/src/storage/types/value.rs +++ b/frame/support/src/storage/types/value.rs @@ -199,6 +199,10 @@ where { >::try_append(item) } + + pub fn stream() -> ::Iterator where Value: crate::StreamRead { + ::stream(Self::hashed_key().into()) + } } impl StorageEntryMetadataBuilder diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index f60e3e7349d6b..52d5cb5eab7ad 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1423,15 +1423,15 @@ impl Pallet { pub fn events() -> Vec> { // Dereferencing the events here is fine since we are not in the // memory-restricted runtime. - Self::read_events_no_consensus().into_iter().map(|e| *e).collect() + Self::read_events_no_consensus().map(|e| *e).collect() } /// Get the current events deposited by the runtime. /// /// Should only be called if you know what you are doing and outside of the runtime block /// execution else it can have a large impact on the PoV size of a block. - pub fn read_events_no_consensus() -> Vec>> { - Events::::get() + pub fn read_events_no_consensus() -> impl sp_std::iter::Iterator>> { + Events::::stream() } /// Set the block number to something in particular. Can be used as an alternative to From f5be68abdd6753c834d1f22ed93137450fabdce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 19 Oct 2022 16:46:54 +0200 Subject: [PATCH 02/15] Add some test --- frame/support/src/lib.rs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 300dc3755ea62..b077de705ae1f 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2764,12 +2764,19 @@ pub struct StorageInput { impl StorageInput { pub fn new(key: Vec) -> Result { let (total_length, exists) = if let Some(len) = sp_io::storage::read(&key, &mut [], 0) { - (len, true) + dbg!((len, true)) } else { (0, false) }; - Ok(Self { total_length, offset: 0, key, exists, buffer: Vec::with_capacity(2048), buffer_pos: 0 }) + Ok(Self { + total_length, + offset: 0, + key, + exists, + buffer: Vec::with_capacity(2048), + buffer_pos: 0, + }) } fn fill_buffer(&mut self) { @@ -2802,13 +2809,27 @@ impl codec::Input for StorageInput { fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { if into.len() > self.buffer.capacity() { unimplemented!() - } else if self.buffer_pos + into.len() > self.buffer.len() { + } else if dbg!(self.buffer_pos) + dbg!(into.len()) > dbg!(self.buffer.len()) { self.fill_buffer(); } let end = self.buffer_pos + into.len(); into[..].copy_from_slice(&self.buffer[self.buffer_pos..end]); + self.buffer_pos = end; Ok(()) } } + +#[test] +fn stream_read_test() { + #[crate::storage_alias] + pub type StreamReadTest = StorageValue>; + + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec = vec![1, 2, 3, 4, 5]; + StreamReadTest::put(&data); + + assert_eq!(data, StreamReadTest::stream().collect::>()); + }) +} From d13e7ab6d76e28b9412d7cb73f5cf3cfb52e0575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 20 Oct 2022 23:29:11 +0200 Subject: [PATCH 03/15] Yep --- frame/support/src/lib.rs | 89 +++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index b077de705ae1f..aa6548cb959ae 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2730,7 +2730,7 @@ impl StreamReadVec { } } -impl sp_std::iter::Iterator for StreamReadVec { +impl sp_std::iter::Iterator for StreamReadVec { type Item = T; fn next(&mut self) -> Option { @@ -2744,7 +2744,7 @@ impl sp_std::iter::Iterator for StreamReadVec { } } -impl StreamRead for Vec { +impl StreamRead for Vec { type Iterator = StreamReadVec; fn stream(key: Vec) -> Self::Iterator { @@ -2763,11 +2763,17 @@ pub struct StorageInput { impl StorageInput { pub fn new(key: Vec) -> Result { - let (total_length, exists) = if let Some(len) = sp_io::storage::read(&key, &mut [], 0) { - dbg!((len, true)) - } else { - (0, false) - }; + let mut buffer = Vec::with_capacity(2048); + unsafe { + buffer.set_len(2048); + } + + let (total_length, exists) = + if let Some(total_length) = sp_io::storage::read(&key, &mut buffer, 0) { + (total_length, true) + } else { + (0, false) + }; Ok(Self { total_length, @@ -2779,42 +2785,79 @@ impl StorageInput { }) } - fn fill_buffer(&mut self) { + fn fill_buffer(&mut self) -> Result<(), codec::Error> { self.buffer.copy_within(self.buffer_pos.., 0); + self.buffer_pos = self.buffer.len() - self.buffer_pos; unsafe { self.buffer.set_len(self.buffer.capacity()); } - if let Some(read) = + if let Some(after_offset) = sp_io::storage::read(&self.key, &mut self.buffer[self.buffer_pos..], self.offset) { + let buffer_len = if (after_offset as usize) < self.buffer.len() - self.buffer_pos { + after_offset as usize + } else { + self.buffer.len() + }; + unsafe { - self.buffer.set_len(sp_std::cmp::min( - self.buffer_pos + read as usize, - self.buffer.len() - self.buffer_pos, - )); + self.buffer.set_len(dbg!(buffer_len)); } - self.offset += read; + self.offset += buffer_len as u32; self.buffer_pos = 0; + + Ok(()) + } else { + Err("Value doesn't exist in the state?".into()) } } + + pub fn exists(&self) -> bool { + self.exists + } } impl codec::Input for StorageInput { fn remaining_len(&mut self) -> Result, codec::Error> { - Ok(Some(self.total_length.saturating_sub(self.offset) as usize)) + Ok(Some(self.total_length.saturating_sub(self.buffer_pos as u32) as usize)) } fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { if into.len() > self.buffer.capacity() { - unimplemented!() - } else if dbg!(self.buffer_pos) + dbg!(into.len()) > dbg!(self.buffer.len()) { - self.fill_buffer(); + let remaining_to_read = self.buffer.len() - self.buffer_pos; + + into[..remaining_to_read].copy_from_slice(&self.buffer[self.buffer_pos..]); + unsafe { + self.buffer.set_len(0); + } + self.buffer_pos = 0; + + if let Some(after_offset) = + sp_io::storage::read(&self.key, &mut into[remaining_to_read..], self.offset) + { + let required_to_read = (into.len() - remaining_to_read) as u32; + + if after_offset < required_to_read { + return Err("not enough data to fill the buffer".into()) + } + + self.offset += required_to_read; + return Ok(()) + } else { + return Err("Value doesn't exist in the state?".into()) + } + } else if self.buffer_pos + into.len() >= self.buffer.len() && + self.offset < self.total_length + { + self.fill_buffer()?; + } else if into.len() + self.buffer_pos > self.buffer.len() { + return Err("not enough data to fill the buffer".into()) } let end = self.buffer_pos + into.len(); - into[..].copy_from_slice(&self.buffer[self.buffer_pos..end]); + into.copy_from_slice(&self.buffer[self.buffer_pos..end]); self.buffer_pos = end; Ok(()) @@ -2826,10 +2869,18 @@ fn stream_read_test() { #[crate::storage_alias] pub type StreamReadTest = StorageValue>; + #[crate::storage_alias] + pub type StreamReadTest2 = StorageValue>>; + sp_io::TestExternalities::default().execute_with(|| { let data: Vec = vec![1, 2, 3, 4, 5]; StreamReadTest::put(&data); assert_eq!(data, StreamReadTest::stream().collect::>()); + + let data: Vec> = vec![vec![0; 3000], vec![1; 2500]]; + StreamReadTest2::put(&data); + + assert_eq!(data, StreamReadTest2::stream().collect::>()); }) } From 38a256c3a065f26b9bcdd80318e3828a07944da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 9 Nov 2022 16:12:16 +0100 Subject: [PATCH 04/15] Move to its own file --- frame/support/src/lib.rs | 178 --------------------- frame/support/src/storage/mod.rs | 1 + frame/support/src/storage/stream_iter.rs | 195 +++++++++++++++++++++++ 3 files changed, 196 insertions(+), 178 deletions(-) create mode 100644 frame/support/src/storage/stream_iter.rs diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 7140eab0094ca..ee7be211ffb61 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2736,181 +2736,3 @@ pub mod pallet_macros { }; } -use sp_std::vec::Vec; - -pub trait StreamRead { - type Iterator: sp_std::iter::Iterator; - - fn stream(key: Vec) -> Self::Iterator; -} - -pub struct StreamReadVec { - marker: sp_std::marker::PhantomData, - input: StorageInput, - length: u32, - read: u32, -} - -impl StreamReadVec { - pub fn new(key: Vec) -> Result { - let mut input = StorageInput::new(key)?; - let length = if input.exists { codec::Compact::::decode(&mut input)?.0 } else { 0 }; - - Ok(Self { marker: sp_std::marker::PhantomData, input, length, read: 0 }) - } -} - -impl sp_std::iter::Iterator for StreamReadVec { - type Item = T; - - fn next(&mut self) -> Option { - if self.read >= self.length { - None - } else { - self.read += 1; - - codec::Decode::decode(&mut self.input).ok() - } - } -} - -impl StreamRead for Vec { - type Iterator = StreamReadVec; - - fn stream(key: Vec) -> Self::Iterator { - StreamReadVec::new(key).unwrap() - } -} - -pub struct StorageInput { - key: Vec, - offset: u32, - total_length: u32, - exists: bool, - buffer: Vec, - buffer_pos: usize, -} - -impl StorageInput { - pub fn new(key: Vec) -> Result { - let mut buffer = Vec::with_capacity(2048); - unsafe { - buffer.set_len(2048); - } - - let (total_length, exists) = - if let Some(total_length) = sp_io::storage::read(&key, &mut buffer, 0) { - (total_length, true) - } else { - (0, false) - }; - - Ok(Self { - total_length, - offset: 0, - key, - exists, - buffer: Vec::with_capacity(2048), - buffer_pos: 0, - }) - } - - fn fill_buffer(&mut self) -> Result<(), codec::Error> { - self.buffer.copy_within(self.buffer_pos.., 0); - self.buffer_pos = self.buffer.len() - self.buffer_pos; - unsafe { - self.buffer.set_len(self.buffer.capacity()); - } - - if let Some(after_offset) = - sp_io::storage::read(&self.key, &mut self.buffer[self.buffer_pos..], self.offset) - { - let buffer_len = if (after_offset as usize) < self.buffer.len() - self.buffer_pos { - after_offset as usize - } else { - self.buffer.len() - }; - - unsafe { - self.buffer.set_len(dbg!(buffer_len)); - } - - self.offset += buffer_len as u32; - self.buffer_pos = 0; - - Ok(()) - } else { - Err("Value doesn't exist in the state?".into()) - } - } - - pub fn exists(&self) -> bool { - self.exists - } -} - -impl codec::Input for StorageInput { - fn remaining_len(&mut self) -> Result, codec::Error> { - Ok(Some(self.total_length.saturating_sub(self.buffer_pos as u32) as usize)) - } - - fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { - if into.len() > self.buffer.capacity() { - let remaining_to_read = self.buffer.len() - self.buffer_pos; - - into[..remaining_to_read].copy_from_slice(&self.buffer[self.buffer_pos..]); - unsafe { - self.buffer.set_len(0); - } - self.buffer_pos = 0; - - if let Some(after_offset) = - sp_io::storage::read(&self.key, &mut into[remaining_to_read..], self.offset) - { - let required_to_read = (into.len() - remaining_to_read) as u32; - - if after_offset < required_to_read { - return Err("not enough data to fill the buffer".into()) - } - - self.offset += required_to_read; - return Ok(()) - } else { - return Err("Value doesn't exist in the state?".into()) - } - } else if self.buffer_pos + into.len() >= self.buffer.len() && - self.offset < self.total_length - { - self.fill_buffer()?; - } else if into.len() + self.buffer_pos > self.buffer.len() { - return Err("not enough data to fill the buffer".into()) - } - - let end = self.buffer_pos + into.len(); - into.copy_from_slice(&self.buffer[self.buffer_pos..end]); - self.buffer_pos = end; - - Ok(()) - } -} - -#[test] -fn stream_read_test() { - #[crate::storage_alias] - pub type StreamReadTest = StorageValue>; - - #[crate::storage_alias] - pub type StreamReadTest2 = StorageValue>>; - - sp_io::TestExternalities::default().execute_with(|| { - let data: Vec = vec![1, 2, 3, 4, 5]; - StreamReadTest::put(&data); - - assert_eq!(data, StreamReadTest::stream().collect::>()); - - let data: Vec> = vec![vec![0; 3000], vec![1; 2500]]; - StreamReadTest2::put(&data); - - assert_eq!(data, StreamReadTest2::stream().collect::>()); - }) -} diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 333f4382557b1..118ff45c527ab 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -47,6 +47,7 @@ pub mod generator; pub mod hashed; pub mod migration; pub mod storage_noop_guard; +pub mod stream_iter; pub mod transactional; pub mod types; pub mod unhashed; diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs new file mode 100644 index 0000000000000..235d11ec0f682 --- /dev/null +++ b/frame/support/src/storage/stream_iter.rs @@ -0,0 +1,195 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sp_std::vec::Vec; + +pub trait StreamIter { + type Iterator: sp_std::iter::Iterator; + + fn stream_iter(key: Vec) -> Self::Iterator; +} + +pub struct StreamReadVec { + marker: sp_std::marker::PhantomData, + input: StorageInput, + length: u32, + read: u32, +} + +impl StreamReadVec { + pub fn new(key: Vec) -> Result { + let mut input = StorageInput::new(key)?; + let length = if input.exists { codec::Compact::::decode(&mut input)?.0 } else { 0 }; + + Ok(Self { marker: sp_std::marker::PhantomData, input, length, read: 0 }) + } +} + +impl sp_std::iter::Iterator for StreamReadVec { + type Item = T; + + fn next(&mut self) -> Option { + if self.read >= self.length { + None + } else { + self.read += 1; + + codec::Decode::decode(&mut self.input).ok() + } + } +} + +impl StreamIter for Vec { + type Iterator = StreamReadVec; + + fn stream_iter(key: Vec) -> Self::Iterator { + StreamReadVec::new(key).unwrap() + } +} + +pub struct StorageInput { + key: Vec, + offset: u32, + total_length: u32, + exists: bool, + buffer: Vec, + buffer_pos: usize, +} + +impl StorageInput { + pub fn new(key: Vec) -> Result { + let mut buffer = Vec::with_capacity(2048); + unsafe { + buffer.set_len(2048); + } + + let (total_length, exists) = + if let Some(total_length) = sp_io::storage::read(&key, &mut buffer, 0) { + (total_length, true) + } else { + (0, false) + }; + + Ok(Self { + total_length, + offset: 0, + key, + exists, + buffer: Vec::with_capacity(2048), + buffer_pos: 0, + }) + } + + fn fill_buffer(&mut self) -> Result<(), codec::Error> { + self.buffer.copy_within(self.buffer_pos.., 0); + self.buffer_pos = self.buffer.len() - self.buffer_pos; + unsafe { + self.buffer.set_len(self.buffer.capacity()); + } + + if let Some(after_offset) = + sp_io::storage::read(&self.key, &mut self.buffer[self.buffer_pos..], self.offset) + { + let buffer_len = if (after_offset as usize) < self.buffer.len() - self.buffer_pos { + after_offset as usize + } else { + self.buffer.len() + }; + + unsafe { + self.buffer.set_len(buffer_len); + } + + self.offset += buffer_len as u32; + self.buffer_pos = 0; + + Ok(()) + } else { + Err("Value doesn't exist in the state?".into()) + } + } + + pub fn exists(&self) -> bool { + self.exists + } +} + +impl codec::Input for StorageInput { + fn remaining_len(&mut self) -> Result, codec::Error> { + Ok(Some(self.total_length.saturating_sub(self.buffer_pos as u32) as usize)) + } + + fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { + if into.len() > self.buffer.capacity() { + let remaining_to_read = self.buffer.len() - self.buffer_pos; + + into[..remaining_to_read].copy_from_slice(&self.buffer[self.buffer_pos..]); + unsafe { + self.buffer.set_len(0); + } + self.buffer_pos = 0; + + if let Some(after_offset) = + sp_io::storage::read(&self.key, &mut into[remaining_to_read..], self.offset) + { + let required_to_read = (into.len() - remaining_to_read) as u32; + + if after_offset < required_to_read { + return Err("not enough data to fill the buffer".into()) + } + + self.offset += required_to_read; + return Ok(()) + } else { + return Err("Value doesn't exist in the state?".into()) + } + } else if self.buffer_pos + into.len() >= self.buffer.len() && + self.offset < self.total_length + { + self.fill_buffer()?; + } else if into.len() + self.buffer_pos > self.buffer.len() { + return Err("not enough data to fill the buffer".into()) + } + + let end = self.buffer_pos + into.len(); + into.copy_from_slice(&self.buffer[self.buffer_pos..end]); + self.buffer_pos = end; + + Ok(()) + } +} + +#[test] +fn stream_read_test() { + #[crate::storage_alias] + pub type StreamReadTest = StorageValue>; + + #[crate::storage_alias] + pub type StreamReadTest2 = StorageValue>>; + + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec = vec![1, 2, 3, 4, 5]; + StreamReadTest::put(&data); + + assert_eq!(data, StreamReadTest::stream().collect::>()); + + let data: Vec> = vec![vec![0; 3000], vec![1; 2500]]; + StreamReadTest2::put(&data); + + assert_eq!(data, StreamReadTest2::stream().collect::>()); + }) +} From 5fec484b58188d83bc209dbe6a203b6525a84584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 10 Nov 2022 22:50:17 +0100 Subject: [PATCH 05/15] More work --- frame/support/src/storage/stream_iter.rs | 93 ++++++++++++++++++++---- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index 235d11ec0f682..ee38748c918a5 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -15,22 +15,82 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::{BoundedBTreeMap, BoundedBTreeSet, BoundedVec, WeakBoundedVec}; use sp_std::vec::Vec; +pub trait ScaleContainerType: private::Sealed { + type Stored: codec::Decode; +} + +impl ScaleContainerType for Vec { + type Stored = T; +} + +impl ScaleContainerType for sp_std::collections::btree_set::BTreeSet { + type Stored = T; +} + +impl ScaleContainerType + for sp_std::collections::btree_map::BTreeMap +{ + type Stored = (K, V); +} + +impl ScaleContainerType for BoundedVec { + type Stored = T; +} + +impl ScaleContainerType for WeakBoundedVec { + type Stored = T; +} + +impl ScaleContainerType for BoundedBTreeMap { + type Stored = (K, V); +} + +impl ScaleContainerType for BoundedBTreeSet { + type Stored = T; +} + +mod private { + use super::*; + + pub trait Sealed {} + + impl Sealed for Vec {} + impl Sealed for sp_std::collections::btree_set::BTreeSet {} + impl Sealed for sp_std::collections::btree_map::BTreeMap {} + + impl Sealed for BoundedVec {} + impl Sealed for WeakBoundedVec {} + impl Sealed for BoundedBTreeMap {} + impl Sealed for BoundedBTreeSet {} +} + pub trait StreamIter { type Iterator: sp_std::iter::Iterator; - fn stream_iter(key: Vec) -> Self::Iterator; + fn stream_iter() -> Self::Iterator; } -pub struct StreamReadVec { +impl> StreamIter + for StorageValue +{ + type Iterator = ScaleContainerStreamIter; + + fn stream_iter() -> Self::Iterator { + ScaleContainerStreamIter::new(Self::storage_value_final_key().into()).unwrap() + } +} + +pub struct ScaleContainerStreamIter { marker: sp_std::marker::PhantomData, input: StorageInput, length: u32, read: u32, } -impl StreamReadVec { +impl ScaleContainerStreamIter { pub fn new(key: Vec) -> Result { let mut input = StorageInput::new(key)?; let length = if input.exists { codec::Compact::::decode(&mut input)?.0 } else { 0 }; @@ -39,7 +99,7 @@ impl StreamReadVec { } } -impl sp_std::iter::Iterator for StreamReadVec { +impl sp_std::iter::Iterator for ScaleContainerStreamIter { type Item = T; fn next(&mut self) -> Option { @@ -54,13 +114,15 @@ impl sp_std::iter::Iterator for StreamReadVec { } impl StreamIter for Vec { - type Iterator = StreamReadVec; + type Iterator = ScaleContainerStreamIter; fn stream_iter(key: Vec) -> Self::Iterator { - StreamReadVec::new(key).unwrap() + ScaleContainerStreamIter::new(key).unwrap() } } +const STORAGE_INPUT_BUFFER_CAPACITY: usize = 16 * 1024; + pub struct StorageInput { key: Vec, offset: u32, @@ -72,9 +134,9 @@ pub struct StorageInput { impl StorageInput { pub fn new(key: Vec) -> Result { - let mut buffer = Vec::with_capacity(2048); + let mut buffer = Vec::with_capacity(STORAGE_INPUT_BUFFER_CAPACITY); unsafe { - buffer.set_len(2048); + buffer.set_len(buffer.capacity()); } let (total_length, exists) = @@ -84,14 +146,13 @@ impl StorageInput { (0, false) }; - Ok(Self { - total_length, - offset: 0, - key, - exists, - buffer: Vec::with_capacity(2048), - buffer_pos: 0, - }) + if total_length < buffer.len() { + unsafe { + buffer.set_len(total_length as usize); + } + } + + Ok(Self { total_length, offset: 0, key, exists, buffer, buffer_pos: 0 }) } fn fill_buffer(&mut self) -> Result<(), codec::Error> { From 59a74b18df148b9b4b460ba1d0322948f96d04d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 11 Nov 2022 12:55:41 +0100 Subject: [PATCH 06/15] More --- frame/support/src/storage/stream_iter.rs | 94 ++++++++++++++---------- frame/support/src/storage/types/value.rs | 4 - frame/system/src/lib.rs | 4 +- 3 files changed, 59 insertions(+), 43 deletions(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index ee38748c918a5..08742393c9558 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -16,40 +16,71 @@ // limitations under the License. use crate::{BoundedBTreeMap, BoundedBTreeSet, BoundedVec, WeakBoundedVec}; +use codec::Decode; use sp_std::vec::Vec; -pub trait ScaleContainerType: private::Sealed { - type Stored: codec::Decode; +pub trait StreamIter: private::Sealed { + type Iterator: sp_std::iter::Iterator; + + fn stream_iter(key: Vec) -> Self::Iterator; } -impl ScaleContainerType for Vec { - type Stored = T; +impl StreamIter for Vec { + type Iterator = ScaleContainerStreamIter; + + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key).unwrap() + } } -impl ScaleContainerType for sp_std::collections::btree_set::BTreeSet { - type Stored = T; +impl StreamIter for sp_std::collections::btree_set::BTreeSet { + type Iterator = ScaleContainerStreamIter; + + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key).unwrap() + } } -impl ScaleContainerType +impl StreamIter for sp_std::collections::btree_map::BTreeMap { - type Stored = (K, V); + type Iterator = ScaleContainerStreamIter<(K, V)>; + + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key).unwrap() + } } -impl ScaleContainerType for BoundedVec { - type Stored = T; +impl StreamIter for BoundedVec { + type Iterator = ScaleContainerStreamIter; + + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key).unwrap() + } } -impl ScaleContainerType for WeakBoundedVec { - type Stored = T; +impl StreamIter for WeakBoundedVec { + type Iterator = ScaleContainerStreamIter; + + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key).unwrap() + } } -impl ScaleContainerType for BoundedBTreeMap { - type Stored = (K, V); +impl StreamIter for BoundedBTreeMap { + type Iterator = ScaleContainerStreamIter<(K, V)>; + + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key).unwrap() + } } -impl ScaleContainerType for BoundedBTreeSet { - type Stored = T; +impl StreamIter for BoundedBTreeSet { + type Iterator = ScaleContainerStreamIter; + + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key).unwrap() + } } mod private { @@ -67,19 +98,15 @@ mod private { impl Sealed for BoundedBTreeSet {} } -pub trait StreamIter { - type Iterator: sp_std::iter::Iterator; - - fn stream_iter() -> Self::Iterator; +pub trait StreamIterExt { + fn stream_iter() -> T::Iterator; } -impl> StreamIter +impl> StreamIterExt for StorageValue { - type Iterator = ScaleContainerStreamIter; - - fn stream_iter() -> Self::Iterator { - ScaleContainerStreamIter::new(Self::storage_value_final_key().into()).unwrap() + fn stream_iter() -> T::Iterator { + T::stream_iter(Self::hashed_key().into()) } } @@ -113,17 +140,10 @@ impl sp_std::iter::Iterator for ScaleContainerStreamIter { } } -impl StreamIter for Vec { - type Iterator = ScaleContainerStreamIter; - - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() - } -} - const STORAGE_INPUT_BUFFER_CAPACITY: usize = 16 * 1024; -pub struct StorageInput { +/// Implementation of [`codec::Input`] +struct StorageInput { key: Vec, offset: u32, total_length: u32, @@ -146,7 +166,7 @@ impl StorageInput { (0, false) }; - if total_length < buffer.len() { + if (total_length as usize) < buffer.len() { unsafe { buffer.set_len(total_length as usize); } @@ -246,11 +266,11 @@ fn stream_read_test() { let data: Vec = vec![1, 2, 3, 4, 5]; StreamReadTest::put(&data); - assert_eq!(data, StreamReadTest::stream().collect::>()); + assert_eq!(data, StreamReadTest::stream_iter().collect::>()); let data: Vec> = vec![vec![0; 3000], vec![1; 2500]]; StreamReadTest2::put(&data); - assert_eq!(data, StreamReadTest2::stream().collect::>()); + assert_eq!(data, StreamReadTest2::stream_iter().collect::>()); }) } diff --git a/frame/support/src/storage/types/value.rs b/frame/support/src/storage/types/value.rs index d2411dced957d..f145e9fb30414 100644 --- a/frame/support/src/storage/types/value.rs +++ b/frame/support/src/storage/types/value.rs @@ -199,10 +199,6 @@ where { >::try_append(item) } - - pub fn stream() -> ::Iterator where Value: crate::StreamRead { - ::stream(Self::hashed_key().into()) - } } impl StorageEntryMetadataBuilder diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index ffc4b77110dc0..248f46d1170d7 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -88,7 +88,7 @@ use frame_support::{ extract_actual_pays_fee, extract_actual_weight, DispatchClass, DispatchInfo, DispatchResult, DispatchResultWithPostInfo, PerDispatchClass, }, - storage, + storage::{self, stream_iter::StreamIterExt}, traits::{ ConstU32, Contains, EnsureOrigin, Get, HandleLifetime, OnKilledAccount, OnNewAccount, OriginTrait, PalletInfo, SortedMembers, StoredMap, TypedGet, @@ -1434,7 +1434,7 @@ impl Pallet { /// Should only be called if you know what you are doing and outside of the runtime block /// execution else it can have a large impact on the PoV size of a block. pub fn read_events_no_consensus() -> impl sp_std::iter::Iterator>> { - Events::::stream() + Events::::stream_iter() } /// Set the block number to something in particular. Can be used as an alternative to From c2d86704d9dd636931ecdc14474d29be0ca4d6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 16 Nov 2022 00:16:47 +0100 Subject: [PATCH 07/15] Finish implementation --- frame/support/src/storage/mod.rs | 3 +- frame/support/src/storage/stream_iter.rs | 313 ++++++++++++++++------- frame/support/src/storage/unhashed.rs | 5 +- frame/system/src/lib.rs | 5 +- 4 files changed, 229 insertions(+), 97 deletions(-) diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 118ff45c527ab..8c0d6207c3f4d 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -30,6 +30,7 @@ use sp_runtime::generic::{Digest, DigestItem}; use sp_std::{collections::btree_set::BTreeSet, marker::PhantomData, prelude::*}; pub use self::{ + stream_iter::StorageStreamIter, transactional::{ in_storage_layer, with_storage_layer, with_transaction, with_transaction_unchecked, }, @@ -47,7 +48,7 @@ pub mod generator; pub mod hashed; pub mod migration; pub mod storage_noop_guard; -pub mod stream_iter; +mod stream_iter; pub mod transactional; pub mod types; pub mod unhashed; diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index 08742393c9558..3ec68eae460b6 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -19,97 +19,109 @@ use crate::{BoundedBTreeMap, BoundedBTreeSet, BoundedVec, WeakBoundedVec}; use codec::Decode; use sp_std::vec::Vec; -pub trait StreamIter: private::Sealed { - type Iterator: sp_std::iter::Iterator; - - fn stream_iter(key: Vec) -> Self::Iterator; -} +/// Provides the sealed trait `StreamIter`. +mod private { + use super::*; -impl StreamIter for Vec { - type Iterator = ScaleContainerStreamIter; + /// Used as marker trait for types that support stream iteration. + pub trait StreamIter { + /// The actual iterator implementation. + type Iterator: sp_std::iter::Iterator; - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() + /// Create the stream iterator for the value found at `key`. + fn stream_iter(key: Vec) -> Self::Iterator; } -} -impl StreamIter for sp_std::collections::btree_set::BTreeSet { - type Iterator = ScaleContainerStreamIter; + impl StreamIter for Vec { + type Iterator = ScaleContainerStreamIter; - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key) + } } -} -impl StreamIter - for sp_std::collections::btree_map::BTreeMap -{ - type Iterator = ScaleContainerStreamIter<(K, V)>; + impl StreamIter for sp_std::collections::btree_set::BTreeSet { + type Iterator = ScaleContainerStreamIter; - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key) + } } -} -impl StreamIter for BoundedVec { - type Iterator = ScaleContainerStreamIter; + impl StreamIter + for sp_std::collections::btree_map::BTreeMap + { + type Iterator = ScaleContainerStreamIter<(K, V)>; - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key) + } } -} -impl StreamIter for WeakBoundedVec { - type Iterator = ScaleContainerStreamIter; + impl StreamIter for BoundedVec { + type Iterator = ScaleContainerStreamIter; - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key) + } } -} -impl StreamIter for BoundedBTreeMap { - type Iterator = ScaleContainerStreamIter<(K, V)>; + impl StreamIter for WeakBoundedVec { + type Iterator = ScaleContainerStreamIter; - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key) + } } -} -impl StreamIter for BoundedBTreeSet { - type Iterator = ScaleContainerStreamIter; + impl StreamIter for BoundedBTreeMap { + type Iterator = ScaleContainerStreamIter<(K, V)>; - fn stream_iter(key: Vec) -> Self::Iterator { - ScaleContainerStreamIter::new(key).unwrap() + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key) + } } -} -mod private { - use super::*; - - pub trait Sealed {} + impl StreamIter for BoundedBTreeSet { + type Iterator = ScaleContainerStreamIter; - impl Sealed for Vec {} - impl Sealed for sp_std::collections::btree_set::BTreeSet {} - impl Sealed for sp_std::collections::btree_map::BTreeMap {} - - impl Sealed for BoundedVec {} - impl Sealed for WeakBoundedVec {} - impl Sealed for BoundedBTreeMap {} - impl Sealed for BoundedBTreeSet {} + fn stream_iter(key: Vec) -> Self::Iterator { + ScaleContainerStreamIter::new(key) + } + } } -pub trait StreamIterExt { +/// An iterator that streams values directly from storage. +/// +/// Requires that `T` implements the sealed trait `StreamIter`. +/// +/// Instead of loading the entire `T` into memory, the iterator only loads a certain number of bytes +/// into memory to decode the next `T::Item`. The iterator implementation is allowed to have some +/// internal buffer to reduce the number of storage reads. The iterator should have an almost +/// constant memory usage over its lifetime. If at some point there is a decoding error, the +/// iterator will return `None` to signal that the iterator is finished. +pub trait StorageStreamIter { + /// Create the streaming iterator. fn stream_iter() -> T::Iterator; } -impl> StreamIterExt - for StorageValue +impl> + StorageStreamIter for StorageValue { fn stream_iter() -> T::Iterator { T::stream_iter(Self::hashed_key().into()) } } +/// A streaming iterator implementation for SCALE container types. +/// +/// SCALE container types follow the same type of encoding `Compact(len) ++ data`. +/// This type provides an [`Iterator`](sp_std::iter::Iterator) implementation that decodes +/// one item after another with each call to [`next`](Self::next). The bytes representing +/// the container are also not read at once into memory and instead being read in chunks. As long +/// as individual items are smaller than these chunks the memory usage of this iterator should +/// be constant. On decoding errors [`next`](Self::next) will return `None` to signal that the +/// iterator is finished. pub struct ScaleContainerStreamIter { marker: sp_std::marker::PhantomData, input: StorageInput, @@ -118,11 +130,31 @@ pub struct ScaleContainerStreamIter { } impl ScaleContainerStreamIter { - pub fn new(key: Vec) -> Result { - let mut input = StorageInput::new(key)?; - let length = if input.exists { codec::Compact::::decode(&mut input)?.0 } else { 0 }; + /// Creates a new instance of the stream iterator. + /// + /// - `key`: Storage key of the container in the state. + pub fn new(key: Vec) -> Self { + let mut input = StorageInput::new(key); + let length = if input.exists() { + match codec::Compact::::decode(&mut input) { + Ok(length) => length.0, + Err(e) => { + // TODO #3700: error should be handleable. + log::error!( + target: "runtime::storage", + "Corrupted state at `{:?}`: {:?}", + input.key, + e, + ); + + 0 + }, + } + } else { + 0 + }; - Ok(Self { marker: sp_std::marker::PhantomData, input, length, read: 0 }) + Self { marker: sp_std::marker::PhantomData, input, length, read: 0 } } } @@ -140,9 +172,19 @@ impl sp_std::iter::Iterator for ScaleContainerStreamIter { } } -const STORAGE_INPUT_BUFFER_CAPACITY: usize = 16 * 1024; - -/// Implementation of [`codec::Input`] +/// The size of the internal buffer used by [`StorageInput`]. +/// +/// This internal buffer is used to speed up implementation as reading from the +/// state for every access is too slow. +const STORAGE_INPUT_BUFFER_CAPACITY: usize = 2 * 1024; + +/// Implementation of [`codec::Input`] using [`sp_io::storage::read`]. +/// +/// Keeps an internal buffer with a size of [`STORAGE_INPUT_BUFFER_CAPACITY`]. All read accesses +/// are tried to be served by this buffer. If the buffer doesn't hold enough bytes to fullfill the +/// current read access, the buffer is re-filled from the state. A read request that is bigger than +/// the internal buffer is directly forwarded to the state to reduce the number of reads from the +/// state. struct StorageInput { key: Vec, offset: u32, @@ -153,7 +195,10 @@ struct StorageInput { } impl StorageInput { - pub fn new(key: Vec) -> Result { + /// Create a new instance of the input. + /// + /// - `key`: The storage key of the storage item that this input will read. + fn new(key: Vec) -> Self { let mut buffer = Vec::with_capacity(STORAGE_INPUT_BUFFER_CAPACITY); unsafe { buffer.set_len(buffer.capacity()); @@ -172,21 +217,24 @@ impl StorageInput { } } - Ok(Self { total_length, offset: 0, key, exists, buffer, buffer_pos: 0 }) + Self { total_length, offset: buffer.len() as u32, key, exists, buffer, buffer_pos: 0 } } + /// Fill the internal buffer from the state. fn fill_buffer(&mut self) -> Result<(), codec::Error> { self.buffer.copy_within(self.buffer_pos.., 0); - self.buffer_pos = self.buffer.len() - self.buffer_pos; + let present_bytes = self.buffer.len() - self.buffer_pos; + self.buffer_pos = 0; + unsafe { self.buffer.set_len(self.buffer.capacity()); } if let Some(after_offset) = - sp_io::storage::read(&self.key, &mut self.buffer[self.buffer_pos..], self.offset) + sp_io::storage::read(&self.key, &mut self.buffer[present_bytes..], self.offset) { - let buffer_len = if (after_offset as usize) < self.buffer.len() - self.buffer_pos { - after_offset as usize + let buffer_len = if (after_offset as usize) < (self.buffer.len() - present_bytes) { + present_bytes + after_offset as usize } else { self.buffer.len() }; @@ -195,16 +243,23 @@ impl StorageInput { self.buffer.set_len(buffer_len); } - self.offset += buffer_len as u32; + self.offset += (buffer_len - present_bytes) as u32; self.buffer_pos = 0; Ok(()) } else { + // The value was deleted, let's ensure we don't read anymore. + self.offset = self.total_length; + unsafe { + self.buffer.set_len(0); + } + Err("Value doesn't exist in the state?".into()) } } - pub fn exists(&self) -> bool { + /// Returns if the value to read exists in the state. + fn exists(&self) -> bool { self.exists } } @@ -216,18 +271,15 @@ impl codec::Input for StorageInput { fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { if into.len() > self.buffer.capacity() { - let remaining_to_read = self.buffer.len() - self.buffer_pos; + let num_cached = self.buffer.len() - self.buffer_pos; - into[..remaining_to_read].copy_from_slice(&self.buffer[self.buffer_pos..]); - unsafe { - self.buffer.set_len(0); - } - self.buffer_pos = 0; + into[..num_cached].copy_from_slice(&self.buffer[self.buffer_pos..]); + self.buffer_pos = self.buffer.len(); if let Some(after_offset) = - sp_io::storage::read(&self.key, &mut into[remaining_to_read..], self.offset) + sp_io::storage::read(&self.key, &mut into[num_cached..], self.offset) { - let required_to_read = (into.len() - remaining_to_read) as u32; + let required_to_read = (into.len() - num_cached) as u32; if after_offset < required_to_read { return Err("not enough data to fill the buffer".into()) @@ -254,23 +306,100 @@ impl codec::Input for StorageInput { } } -#[test] -fn stream_read_test() { +#[cfg(test)] +mod tests { + use super::*; + use codec::Encode; + #[crate::storage_alias] - pub type StreamReadTest = StorageValue>; + pub type TestVecU32 = StorageValue>; #[crate::storage_alias] - pub type StreamReadTest2 = StorageValue>>; + pub type TestVecVecU8 = StorageValue>>; + + #[test] + fn stream_read_test() { + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec = vec![1, 2, 3, 4, 5]; + TestVecU32::put(&data); - sp_io::TestExternalities::default().execute_with(|| { - let data: Vec = vec![1, 2, 3, 4, 5]; - StreamReadTest::put(&data); + assert_eq!(data, TestVecU32::stream_iter().collect::>()); - assert_eq!(data, StreamReadTest::stream_iter().collect::>()); + let data: Vec> = vec![vec![0; 3000], vec![1; 2500]]; + TestVecVecU8::put(&data); - let data: Vec> = vec![vec![0; 3000], vec![1; 2500]]; - StreamReadTest2::put(&data); + assert_eq!(data, TestVecVecU8::stream_iter().collect::>()); + }) + } - assert_eq!(data, StreamReadTest2::stream_iter().collect::>()); - }) + #[test] + fn reading_big_intermediate_value() { + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec> = + vec![vec![0; 20], vec![1; STORAGE_INPUT_BUFFER_CAPACITY * 2], vec![2; 30]]; + TestVecVecU8::put(&data); + + assert_eq!(data, TestVecVecU8::stream_iter().collect::>()); + + let data: Vec> = vec![ + vec![0; 20], + vec![1; STORAGE_INPUT_BUFFER_CAPACITY * 2], + vec![2; STORAGE_INPUT_BUFFER_CAPACITY * 2], + vec![3; 30], + vec![4; 30], + vec![5; STORAGE_INPUT_BUFFER_CAPACITY * 2], + vec![6; 30], + ]; + TestVecVecU8::put(&data); + + assert_eq!(data, TestVecVecU8::stream_iter().collect::>()); + }) + } + + #[test] + fn reading_invalid_data_from_state() { + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec = vec![1, 2, 3, 4, 5]; + + let mut data_encoded = data.encode(); + data_encoded.truncate(data_encoded.len() - 2); + sp_io::storage::set(&TestVecU32::hashed_key(), &data_encoded); + assert_eq!( + data.iter().copied().take(data.len() - 1).collect::>(), + TestVecU32::stream_iter().collect::>() + ); + + let data_encoded = data.encode()[2..].to_vec(); + sp_io::storage::set(&TestVecU32::hashed_key(), &data_encoded); + assert!(TestVecU32::stream_iter().collect::>().is_empty()); + + let data: Vec> = vec![vec![0; 20], vec![1; STORAGE_INPUT_BUFFER_CAPACITY * 2]]; + let mut data_encoded = data.encode(); + data_encoded.truncate(data_encoded.len() - 100); + sp_io::storage::set(&TestVecVecU8::hashed_key(), &data_encoded); + + assert_eq!( + data.iter().cloned().take(1).collect::>(), + TestVecVecU8::stream_iter().collect::>() + ); + }) + } + + #[test] + fn reading_with_fill_buffer() { + sp_io::TestExternalities::default().execute_with(|| { + const BUFFER_SIZE: usize = 300; + // Ensure that the capacity isn't dividable by `300`. + assert!(STORAGE_INPUT_BUFFER_CAPACITY % BUFFER_SIZE != 0, "Please update buffer size"); + // Create some items where the last item is partially in the inner buffer so that + // we need to fill the buffer to read the entire item. + let data: Vec> = (0..=(STORAGE_INPUT_BUFFER_CAPACITY / BUFFER_SIZE)) + .into_iter() + .map(|i| vec![i as u8; BUFFER_SIZE]) + .collect::>>(); + TestVecVecU8::put(&data); + + assert_eq!(data, TestVecVecU8::stream_iter().collect::>()); + }) + } } diff --git a/frame/support/src/storage/unhashed.rs b/frame/support/src/storage/unhashed.rs index 089230bc9bfdf..850e93e7d7fe4 100644 --- a/frame/support/src/storage/unhashed.rs +++ b/frame/support/src/storage/unhashed.rs @@ -23,12 +23,13 @@ use sp_std::prelude::*; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get(key: &[u8]) -> Option { sp_io::storage::get(key).and_then(|val| { - Decode::decode(&mut &val[..]).map(Some).unwrap_or_else(|_| { + Decode::decode(&mut &val[..]).map(Some).unwrap_or_else(|e| { // TODO #3700: error should be handleable. log::error!( target: "runtime::storage", - "Corrupted state at {:?}", + "Corrupted state at `{:?}: {:?}`", key, + e, ); None }) diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 248f46d1170d7..a12a90a70f87c 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -88,7 +88,7 @@ use frame_support::{ extract_actual_pays_fee, extract_actual_weight, DispatchClass, DispatchInfo, DispatchResult, DispatchResultWithPostInfo, PerDispatchClass, }, - storage::{self, stream_iter::StreamIterExt}, + storage::{self, StorageStreamIter}, traits::{ ConstU32, Contains, EnsureOrigin, Get, HandleLifetime, OnKilledAccount, OnNewAccount, OriginTrait, PalletInfo, SortedMembers, StoredMap, TypedGet, @@ -1433,7 +1433,8 @@ impl Pallet { /// /// Should only be called if you know what you are doing and outside of the runtime block /// execution else it can have a large impact on the PoV size of a block. - pub fn read_events_no_consensus() -> impl sp_std::iter::Iterator>> { + pub fn read_events_no_consensus( + ) -> impl sp_std::iter::Iterator>> { Events::::stream_iter() } From 32339f4c5c1540e5ab3d78d9ac964ef160bdf307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Nov 2022 17:01:09 +0100 Subject: [PATCH 08/15] Start resolving comments and fixing bugs --- frame/support/src/storage/stream_iter.rs | 87 +++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index 3ec68eae460b6..0bba34979bfd3 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -133,6 +133,8 @@ impl ScaleContainerStreamIter { /// Creates a new instance of the stream iterator. /// /// - `key`: Storage key of the container in the state. + /// + /// Same as [`Self::try_new`], but logs a potential error and sets the length to `0`. pub fn new(key: Vec) -> Self { let mut input = StorageInput::new(key); let length = if input.exists() { @@ -156,6 +158,18 @@ impl ScaleContainerStreamIter { Self { marker: sp_std::marker::PhantomData, input, length, read: 0 } } + + /// Creates a new instance of the stream iterator. + /// + /// - `key`: Storage key of the container in the state. + /// + /// Returns an error if the length of the container fails to decode. + pub fn new_try(key: Vec) -> Result { + let mut input = StorageInput::new(key); + let length = if input.exists() { codec::Compact::::decode(&mut input)?.0 } else { 0 }; + + Ok(Self { marker: sp_std::marker::PhantomData, input, length, read: 0 }) + } } impl sp_std::iter::Iterator for ScaleContainerStreamIter { @@ -170,6 +184,12 @@ impl sp_std::iter::Iterator for ScaleContainerStreamIter { codec::Decode::decode(&mut self.input).ok() } } + + fn size_hint(&self) -> (usize, Option) { + let left = self.length.saturating_sub(self.read) as usize; + + (left, Some(left)) + } } /// The size of the internal buffer used by [`StorageInput`]. @@ -266,7 +286,9 @@ impl StorageInput { impl codec::Input for StorageInput { fn remaining_len(&mut self) -> Result, codec::Error> { - Ok(Some(self.total_length.saturating_sub(self.buffer_pos as u32) as usize)) + Ok(Some(self.total_length.saturating_sub( + self.offset.saturating_sub((self.buffer.len() - self.buffer_pos) as u32), + ) as usize)) } fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { @@ -309,7 +331,7 @@ impl codec::Input for StorageInput { #[cfg(test)] mod tests { use super::*; - use codec::Encode; + use codec::{Compact, CompactLen, Encode, Input}; #[crate::storage_alias] pub type TestVecU32 = StorageValue>; @@ -317,6 +339,67 @@ mod tests { #[crate::storage_alias] pub type TestVecVecU8 = StorageValue>>; + #[test] + fn remaining_len_works() { + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec = vec![1, 2, 3, 4, 5]; + TestVecU32::put(&data); + + let mut input = StorageInput::new(TestVecU32::hashed_key().into()); + assert_eq!( + 5 * std::mem::size_of::() + Compact::::compact_len(&5) as usize, + input.remaining_len().ok().flatten().unwrap() + ); + + assert_eq!(5, Compact::::decode(&mut input).unwrap().0); + assert_eq!( + 5 * std::mem::size_of::(), + input.remaining_len().ok().flatten().unwrap() + ); + + for i in &data { + assert_eq!(*i, u32::decode(&mut input).unwrap()); + assert_eq!( + (5 - *i as usize) * std::mem::size_of::(), + input.remaining_len().ok().flatten().unwrap() + ); + } + + let data: Vec> = vec![ + vec![0; 20], + vec![1; STORAGE_INPUT_BUFFER_CAPACITY * 2], + vec![2; STORAGE_INPUT_BUFFER_CAPACITY * 2], + vec![3; 30], + vec![4; 30], + vec![5; STORAGE_INPUT_BUFFER_CAPACITY * 2], + vec![6; 30], + ]; + TestVecVecU8::put(&data); + + let mut input = StorageInput::new(TestVecVecU8::hashed_key().into()); + let total_data_len = data + .iter() + .map(|v| v.len() + Compact::::compact_len(&(v.len() as u32)) as usize) + .sum::(); + assert_eq!( + total_data_len + Compact::::compact_len(&(data.len() as u32)) as usize, + input.remaining_len().ok().flatten().unwrap() + ); + + assert_eq!(data.len(), Compact::::decode(&mut input).unwrap().0 as usize); + assert_eq!(total_data_len, input.remaining_len().ok().flatten().unwrap()); + + let mut remaining_len = total_data_len; + for i in data { + assert_eq!(i, Vec::::decode(&mut input).unwrap()); + + remaining_len -= i.len() + Compact::::compact_len(&(i.len() as u32)) as usize; + + assert_eq!(remaining_len, input.remaining_len().ok().flatten().unwrap()); + } + }) + } + #[test] fn stream_read_test() { sp_io::TestExternalities::default().execute_with(|| { From 4fda5bbff5199a30e1911c8f6a22298afa5fcff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 27 Nov 2022 19:29:11 +0100 Subject: [PATCH 09/15] Fix all review comments --- frame/support/src/storage/stream_iter.rs | 255 +++++++++++++++++++---- 1 file changed, 216 insertions(+), 39 deletions(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index 0bba34979bfd3..68b60bd455ec1 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -181,7 +181,21 @@ impl sp_std::iter::Iterator for ScaleContainerStreamIter { } else { self.read += 1; - codec::Decode::decode(&mut self.input).ok() + match codec::Decode::decode(&mut self.input) { + Ok(r) => Some(r), + Err(e) => { + self.read = self.length; + + log::error!( + target: "runtime::storage", + "Corrupted state at `{:?}`: {:?}", + self.input.key, + e, + ); + + None + }, + } } } @@ -250,29 +264,24 @@ impl StorageInput { self.buffer.set_len(self.buffer.capacity()); } - if let Some(after_offset) = + if let Some(length_minus_offset) = sp_io::storage::read(&self.key, &mut self.buffer[present_bytes..], self.offset) { - let buffer_len = if (after_offset as usize) < (self.buffer.len() - present_bytes) { - present_bytes + after_offset as usize - } else { - self.buffer.len() - }; - + let bytes_read = + sp_std::cmp::min(length_minus_offset as usize, self.buffer.len() - present_bytes); + let buffer_len = present_bytes + bytes_read; unsafe { self.buffer.set_len(buffer_len); } - self.offset += (buffer_len - present_bytes) as u32; - self.buffer_pos = 0; + self.ensure_total_length_did_not_change(length_minus_offset)?; + + self.offset += bytes_read as u32; Ok(()) } else { // The value was deleted, let's ensure we don't read anymore. - self.offset = self.total_length; - unsafe { - self.buffer.set_len(0); - } + self.stop_reading(); Err("Value doesn't exist in the state?".into()) } @@ -282,6 +291,72 @@ impl StorageInput { fn exists(&self) -> bool { self.exists } + + /// Reads directly into the given slice `into`. + /// + /// Should be used when `into.len() > self.buffer.capacity()` to reduce the number of reads from + /// the state. + #[inline(never)] + fn read_big_item(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { + let num_cached = self.buffer.len() - self.buffer_pos; + + into[..num_cached].copy_from_slice(&self.buffer[self.buffer_pos..]); + + self.buffer_pos = 0; + unsafe { + self.buffer.set_len(0); + } + + if let Some(length_minus_offset) = + sp_io::storage::read(&self.key, &mut into[num_cached..], self.offset) + { + let required_to_read = (into.len() - num_cached) as u32; + + if length_minus_offset < required_to_read { + return Err("Not enough data to fill the buffer".into()) + } + + self.ensure_total_length_did_not_change(length_minus_offset)?; + + self.offset += required_to_read; + + Ok(()) + } else { + // The value was deleted, let's ensure we don't read anymore. + self.stop_reading(); + + Err("Value doesn't exist in the state?".into()) + } + } + + /// Ensures that the expected total length of the value did not change. + /// + /// On error ensures that further reading is prohibited. + fn ensure_total_length_did_not_change( + &mut self, + length_minus_offset: u32, + ) -> Result<(), codec::Error> { + if self.total_length == self.offset + length_minus_offset { + Ok(()) + } else { + // The value total length changed, let's ensure we don't read anymore. + self.stop_reading(); + + Err("Storage value changed while it is being read!".into()) + } + } + + /// Ensure that we are stop reading from this value in the state. + /// + /// Should be used when there happened an unrecoverable error while reading. + fn stop_reading(&mut self) { + self.offset = self.total_length; + + self.buffer_pos = 0; + unsafe { + self.buffer.set_len(0); + } + } } impl codec::Input for StorageInput { @@ -292,32 +367,19 @@ impl codec::Input for StorageInput { } fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { - if into.len() > self.buffer.capacity() { - let num_cached = self.buffer.len() - self.buffer_pos; - - into[..num_cached].copy_from_slice(&self.buffer[self.buffer_pos..]); - self.buffer_pos = self.buffer.len(); - - if let Some(after_offset) = - sp_io::storage::read(&self.key, &mut into[num_cached..], self.offset) - { - let required_to_read = (into.len() - num_cached) as u32; - - if after_offset < required_to_read { - return Err("not enough data to fill the buffer".into()) - } - - self.offset += required_to_read; - return Ok(()) - } else { - return Err("Value doesn't exist in the state?".into()) + // If there is still data left to be read from the state. + if self.offset < self.total_length { + if into.len() > self.buffer.capacity() { + return self.read_big_item(into) + } else if self.buffer_pos + into.len() >= self.buffer.len() { + self.fill_buffer()?; } - } else if self.buffer_pos + into.len() >= self.buffer.len() && - self.offset < self.total_length - { - self.fill_buffer()?; - } else if into.len() + self.buffer_pos > self.buffer.len() { - return Err("not enough data to fill the buffer".into()) + } + + // Guard against `fill_buffer` not reading enough data or just not having enough data + // anymore. + if into.len() + self.buffer_pos > self.buffer.len() { + return Err("Not enough data to fill the buffer".into()) } let end = self.buffer_pos + into.len(); @@ -400,6 +462,41 @@ mod tests { }) } + #[test] + fn detects_value_total_length_change() { + sp_io::TestExternalities::default().execute_with(|| { + let test_data: Vec>> = vec![ + vec![vec![0; 20], vec![1; STORAGE_INPUT_BUFFER_CAPACITY * 2]], + vec![ + vec![0; STORAGE_INPUT_BUFFER_CAPACITY - 1], + vec![1; STORAGE_INPUT_BUFFER_CAPACITY - 1], + ], + ]; + + for data in test_data { + TestVecVecU8::put(&data); + + let mut input = StorageInput::new(TestVecVecU8::hashed_key().into()); + + Compact::::decode(&mut input).unwrap(); + Vec::::decode(&mut input).unwrap(); + + TestVecVecU8::append(vec![1, 2, 3]); + + assert!(Vec::::decode(&mut input) + .unwrap_err() + .to_string() + .contains("Storage value changed while it is being read")); + + // Reading a second time should now prevent reading at all. + assert!(Vec::::decode(&mut input) + .unwrap_err() + .to_string() + .contains("Not enough data to fill the buffer")); + } + }) + } + #[test] fn stream_read_test() { sp_io::TestExternalities::default().execute_with(|| { @@ -439,6 +536,27 @@ mod tests { }) } + #[test] + fn reading_more_data_as_in_the_state_is_detected() { + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec> = vec![vec![0; 20], vec![1; STORAGE_INPUT_BUFFER_CAPACITY * 2]]; + TestVecVecU8::put(&data); + + let mut input = StorageInput::new(TestVecVecU8::hashed_key().into()); + + Compact::::decode(&mut input).unwrap(); + + Vec::::decode(&mut input).unwrap(); + + let mut buffer = vec![0; STORAGE_INPUT_BUFFER_CAPACITY * 4]; + assert!(input + .read(&mut buffer) + .unwrap_err() + .to_string() + .contains("Not enough data to fill the buffer")); + }) + } + #[test] fn reading_invalid_data_from_state() { sp_io::TestExternalities::default().execute_with(|| { @@ -483,6 +601,65 @@ mod tests { TestVecVecU8::put(&data); assert_eq!(data, TestVecVecU8::stream_iter().collect::>()); + + let mut input = StorageInput::new(TestVecVecU8::hashed_key().into()); + + Compact::::decode(&mut input).unwrap(); + + (0..data.len() - 1).into_iter().for_each(|_| { + Vec::::decode(&mut input).unwrap(); + }); + + // Try reading a more data than there should be left. + let mut result_buffer = vec![0; BUFFER_SIZE * 2]; + assert!(input + .read(&mut result_buffer) + .unwrap_err() + .to_string() + .contains("Not enough data to fill the buffer")); + }) + } + + #[test] + fn detect_value_deleted_in_state() { + sp_io::TestExternalities::default().execute_with(|| { + let data: Vec> = vec![vec![0; 20], vec![1; STORAGE_INPUT_BUFFER_CAPACITY * 2]]; + TestVecVecU8::put(&data); + + let mut input = StorageInput::new(TestVecVecU8::hashed_key().into()); + TestVecVecU8::kill(); + + Compact::::decode(&mut input).unwrap(); + Vec::::decode(&mut input).unwrap(); + + assert!(Vec::::decode(&mut input) + .unwrap_err() + .to_string() + .contains("Value doesn't exist in the state?")); + + const BUFFER_SIZE: usize = 300; + // Ensure that the capacity isn't dividable by `300`. + assert!(STORAGE_INPUT_BUFFER_CAPACITY % BUFFER_SIZE != 0, "Please update buffer size"); + // Create some items where the last item is partially in the inner buffer so that + // we need to fill the buffer to read the entire item. + let data: Vec> = (0..=(STORAGE_INPUT_BUFFER_CAPACITY / BUFFER_SIZE)) + .into_iter() + .map(|i| vec![i as u8; BUFFER_SIZE]) + .collect::>>(); + TestVecVecU8::put(&data); + + let mut input = StorageInput::new(TestVecVecU8::hashed_key().into()); + TestVecVecU8::kill(); + + Compact::::decode(&mut input).unwrap(); + (0..data.len() - 1).into_iter().for_each(|_| { + Vec::::decode(&mut input).unwrap(); + }); + + assert!(Vec::::decode(&mut input) + .unwrap_err() + .to_string() + .contains("Value doesn't exist in the state?")); }) } } From 74ae71558801fa3fe65aea61d510dafc2d104347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 17 Dec 2022 00:22:30 +0100 Subject: [PATCH 10/15] Update frame/support/src/storage/stream_iter.rs Co-authored-by: Koute --- frame/support/src/storage/stream_iter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index 68b60bd455ec1..aa1856fdcc60a 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -144,7 +144,7 @@ impl ScaleContainerStreamIter { // TODO #3700: error should be handleable. log::error!( target: "runtime::storage", - "Corrupted state at `{:?}`: {:?}", + "Corrupted state at `{:?}`: failed to decode element count: {:?}", input.key, e, ); From 42314d876aa0649353580d8317d082d48facdac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 17 Dec 2022 00:23:14 +0100 Subject: [PATCH 11/15] Update frame/support/src/storage/stream_iter.rs Co-authored-by: Koute --- frame/support/src/storage/stream_iter.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index aa1856fdcc60a..df241da4bfc93 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -179,20 +179,22 @@ impl sp_std::iter::Iterator for ScaleContainerStreamIter { if self.read >= self.length { None } else { - self.read += 1; - match codec::Decode::decode(&mut self.input) { - Ok(r) => Some(r), + Ok(r) => { + self.read += 1; + Some(r) + }, Err(e) => { - self.read = self.length; - log::error!( target: "runtime::storage", - "Corrupted state at `{:?}`: {:?}", + "Corrupted state at `{:?}`: failed to decode element {} (out of {} in total): {:?}", self.input.key, + self.read, + self.length, e, ); + self.read = self.length; None }, } From ac25cb1e674ca44f92e04081410490f8e6e1aee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 17 Dec 2022 00:43:35 +0100 Subject: [PATCH 12/15] Review feedback --- frame/support/src/storage/stream_iter.rs | 55 ++++++++++++------------ 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index df241da4bfc93..ccca9581665ad 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -177,32 +177,32 @@ impl sp_std::iter::Iterator for ScaleContainerStreamIter { fn next(&mut self) -> Option { if self.read >= self.length { - None - } else { - match codec::Decode::decode(&mut self.input) { - Ok(r) => { - self.read += 1; - Some(r) - }, - Err(e) => { - log::error!( - target: "runtime::storage", - "Corrupted state at `{:?}`: failed to decode element {} (out of {} in total): {:?}", - self.input.key, - self.read, - self.length, - e, - ); + return None + } - self.read = self.length; - None - }, - } + match codec::Decode::decode(&mut self.input) { + Ok(r) => { + self.read += 1; + Some(r) + }, + Err(e) => { + log::error!( + target: "runtime::storage", + "Corrupted state at `{:?}`: failed to decode element {} (out of {} in total): {:?}", + self.input.key, + self.read, + self.length, + e, + ); + + self.read = self.length; + None + }, } } fn size_hint(&self) -> (usize, Option) { - let left = self.length.saturating_sub(self.read) as usize; + let left = (self.length - self.read) as usize; (left, Some(left)) } @@ -302,7 +302,8 @@ impl StorageInput { fn read_big_item(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { let num_cached = self.buffer.len() - self.buffer_pos; - into[..num_cached].copy_from_slice(&self.buffer[self.buffer_pos..]); + let (out_already_read, mut out_remaining) = into.split_at_mut(num_cached); + out_already_read.copy_from_slice(&self.buffer[self.buffer_pos..]); self.buffer_pos = 0; unsafe { @@ -310,17 +311,15 @@ impl StorageInput { } if let Some(length_minus_offset) = - sp_io::storage::read(&self.key, &mut into[num_cached..], self.offset) + sp_io::storage::read(&self.key, &mut out_remaining, self.offset) { - let required_to_read = (into.len() - num_cached) as u32; - - if length_minus_offset < required_to_read { + if (length_minus_offset as usize) < out_remaining.len() { return Err("Not enough data to fill the buffer".into()) } self.ensure_total_length_did_not_change(length_minus_offset)?; - self.offset += required_to_read; + self.offset += out_remaining.len() as u32; Ok(()) } else { @@ -373,7 +372,7 @@ impl codec::Input for StorageInput { if self.offset < self.total_length { if into.len() > self.buffer.capacity() { return self.read_big_item(into) - } else if self.buffer_pos + into.len() >= self.buffer.len() { + } else if self.buffer_pos + into.len() > self.buffer.len() { self.fill_buffer()?; } } From a0302c5e36e86328a371f04156e1bf725bdfc605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 17 Dec 2022 00:49:29 +0100 Subject: [PATCH 13/15] FMT --- frame/support/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 166f5b92f2d1c..efecbb75f9c62 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2744,4 +2744,3 @@ pub mod pallet_macros { type_value, unbounded, validate_unsigned, weight, whitelist_storage, }; } - From 005101d6b78acb587f96ef09b3cf3de1d423cbbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 17 Dec 2022 08:23:35 +0100 Subject: [PATCH 14/15] Okay, let's initialize the values... --- frame/support/src/storage/stream_iter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index 68b60bd455ec1..9604db163801b 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -233,7 +233,7 @@ impl StorageInput { /// /// - `key`: The storage key of the storage item that this input will read. fn new(key: Vec) -> Self { - let mut buffer = Vec::with_capacity(STORAGE_INPUT_BUFFER_CAPACITY); + let mut buffer = vec![0; STORAGE_INPUT_BUFFER_CAPACITY]; unsafe { buffer.set_len(buffer.capacity()); } From a4dae597e9e6107c05148eb235606b882c6c7209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 17 Dec 2022 09:46:46 +0100 Subject: [PATCH 15/15] Fix... --- frame/support/src/storage/stream_iter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/src/storage/stream_iter.rs b/frame/support/src/storage/stream_iter.rs index ff48fd6874843..4e25ca6ca1f6c 100644 --- a/frame/support/src/storage/stream_iter.rs +++ b/frame/support/src/storage/stream_iter.rs @@ -235,7 +235,7 @@ impl StorageInput { /// /// - `key`: The storage key of the storage item that this input will read. fn new(key: Vec) -> Self { - let mut buffer = vec![0; STORAGE_INPUT_BUFFER_CAPACITY]; + let mut buffer = sp_std::vec![0; STORAGE_INPUT_BUFFER_CAPACITY]; unsafe { buffer.set_len(buffer.capacity()); }