move storage maps to blake2_128

core/client/src/client.rs

@@ -1534,7 +1534,7 @@ impl<B, E, Block, RA> backend::AuxStore for Client<B, E, Block, RA>
 pub(crate) mod tests {
 	use std::collections::HashMap;
 	use super::*;
-	use primitives::twox_128;
+	use primitives::blake2_256;
 	use runtime_primitives::traits::DigestItem as DigestItemT;
 	use runtime_primitives::generic::DigestItem;
 	use test_client::{self, TestClient, AccountKeyring};
@@ -1584,12 +1584,12 @@ pub(crate) mod tests {
 		}
 
 		// prepare test cases
-		let alice = twox_128(&runtime::system::balance_of_key(AccountKeyring::Alice.into())).to_vec();
-		let bob = twox_128(&runtime::system::balance_of_key(AccountKeyring::Bob.into())).to_vec();
-		let charlie = twox_128(&runtime::system::balance_of_key(AccountKeyring::Charlie.into())).to_vec();
-		let dave = twox_128(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
-		let eve = twox_128(&runtime::system::balance_of_key(AccountKeyring::Eve.into())).to_vec();
-		let ferdie = twox_128(&runtime::system::balance_of_key(AccountKeyring::Ferdie.into())).to_vec();
+		let alice = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Alice.into())).to_vec();
+		let bob = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Bob.into())).to_vec();
+		let charlie = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Charlie.into())).to_vec();
+		let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
+		let eve = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Eve.into())).to_vec();
+		let ferdie = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Ferdie.into())).to_vec();
 		let test_cases = vec![
 			(1, 4, alice.clone(), vec![(4, 0), (1, 0)]),
 			(1, 3, alice.clone(), vec![(1, 0)]),

core/client/src/light/fetcher.rs

@@ -404,7 +404,7 @@ pub mod tests {
 	use crate::light::fetcher::{Fetcher, FetchChecker, LightDataChecker,
 		RemoteCallRequest, RemoteHeaderRequest};
 	use crate::light::blockchain::tests::{DummyStorage, DummyBlockchain};
-	use primitives::{twox_128, Blake2Hasher};
+	use primitives::{blake2_256, Blake2Hasher};
 	use primitives::storage::{StorageKey, well_known_keys};
 	use runtime_primitives::generic::BlockId;
 	use state_machine::Backend;
@@ -587,7 +587,7 @@ pub mod tests {
 		// we're testing this test case here:
 		// (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]),
 		let (remote_client, remote_roots, _) = prepare_client_with_key_changes();
-		let dave = twox_128(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
+		let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
 		let dave = StorageKey(dave);
 
 		// 'fetch' changes proof from remote node:
@@ -699,7 +699,7 @@ pub mod tests {
 		let (remote_client, remote_roots, _) = prepare_client_with_key_changes();
 		let local_cht_root = cht::compute_root::<Header, Blake2Hasher, _>(
 			4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap();
-		let dave = twox_128(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
+		let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
 		let dave = StorageKey(dave);
 
 		// 'fetch' changes proof from remote node:

core/executor/src/wasm_executor.rs

@@ -28,7 +28,7 @@ use wasmi::memory_units::{Pages};
 use state_machine::Externalities;
 use crate::error::{Error, ErrorKind, Result};
 use crate::wasm_utils::UserError;
-use primitives::{blake2_256, twox_128, twox_256, ed25519, sr25519, Pair};
+use primitives::{blake2_128, blake2_256, twox_64, twox_128, twox_256, ed25519, sr25519, Pair};
 use primitives::hexdisplay::HexDisplay;
 use primitives::sandbox as sandbox_primitives;
 use primitives::{H256, Blake2Hasher};
@@ -418,6 +418,30 @@ impl_function_executor!(this: FunctionExecutor<'e, E>,
 	ext_chain_id() -> u64 => {
 		Ok(this.ext.chain_id())
 	},
+	ext_twox_64(data: *const u8, len: u32, out: *mut u8) => {
+		let result: [u8; 8] = if len == 0 {
+			let hashed = twox_64(&[0u8; 0]);
+			debug_trace!(target: "xxhash", "XXhash: '' -> {}", HexDisplay::from(&hashed));
+			this.hash_lookup.insert(hashed.to_vec(), vec![]);
+			hashed
+		} else {
+			let key = this.memory.get(data, len as usize).map_err(|_| UserError("Invalid attempt to get key in ext_twox_64"))?;
+			let hashed_key = twox_64(&key);
+			debug_trace!(target: "xxhash", "XXhash: {} -> {}",
+				if let Ok(_skey) = ::std::str::from_utf8(&key) {
+					_skey
+				} else {
+					&format!("{}", HexDisplay::from(&key))
+				},
+				HexDisplay::from(&hashed_key)
+			);
+			this.hash_lookup.insert(hashed_key.to_vec(), key);
+			hashed_key
+		};
+
+		this.memory.set(out, &result).map_err(|_| UserError("Invalid attempt to set result in ext_twox_64"))?;
+		Ok(())
+	},
 	ext_twox_128(data: *const u8, len: u32, out: *mut u8) => {
 		let result: [u8; 16] = if len == 0 {
 			let hashed = twox_128(&[0u8; 0]);
@@ -451,6 +475,21 @@ impl_function_executor!(this: FunctionExecutor<'e, E>,
 		this.memory.set(out, &result).map_err(|_| UserError("Invalid attempt to set result in ext_twox_256"))?;
 		Ok(())
 	},
+	ext_blake2_128(data: *const u8, len: u32, out: *mut u8) => {
+		let result: [u8; 16] = if len == 0 {
+			let hashed = blake2_128(&[0u8; 0]);
+			this.hash_lookup.insert(hashed.to_vec(), vec![]);
+			hashed
+		} else {
+			let key = this.memory.get(data, len as usize).map_err(|_| UserError("Invalid attempt to get key in ext_blake2_128"))?;
+			let hashed_key = blake2_128(&key);
+			this.hash_lookup.insert(hashed_key.to_vec(), key);
+			hashed_key
+		};
+
+		this.memory.set(out, &result).map_err(|_| UserError("Invalid attempt to set result in ext_blake2_128"))?;
+		Ok(())
+	},
 	ext_blake2_256(data: *const u8, len: u32, out: *mut u8) => {
 		let result: [u8; 32] = if len == 0 {
 			blake2_256(&[0u8; 0])
@@ -910,6 +949,21 @@ mod tests {
 		);
 	}
 
+	#[test]
+	fn blake2_128_should_work() {
+		let mut ext = TestExternalities::default();
+		let test_code = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm");
+		assert_eq!(
+			WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_128", &[]).unwrap(),
+			blake2_128(&b""[..]).encode()
+		);
+		assert_eq!(
+			WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_128", b"Hello world!").unwrap(),
+			blake2_128(&b"Hello world!"[..]).encode()
+		);
+	}
+
+
 	#[test]
 	fn twox_256_should_work() {
 		let mut ext = TestExternalities::default();

core/executor/wasm/src/lib.rs

@@ -7,7 +7,7 @@ use alloc::vec::Vec;
 use alloc::slice;
 
 use runtime_io::{
-	set_storage, storage, clear_prefix, print, blake2_256,
+	set_storage, storage, clear_prefix, print, blake2_128, blake2_256,
 	twox_128, twox_256, ed25519_verify, sr25519_verify, enumerated_trie_root
 };
 
@@ -68,6 +68,7 @@ impl_stubs!(
 		input.to_vec()
 	},
 	test_blake2_256 => |input| blake2_256(input).to_vec(),
+	test_blake2_128 => |input| blake2_128(input).to_vec(),
 	test_twox_256 => |input| twox_256(input).to_vec(),
 	test_twox_128 => |input| twox_128(input).to_vec(),
 	test_ed25519_verify => |input: &[u8]| {

core/primitives/Cargo.toml

@@ -34,6 +34,8 @@ regex = {version = "1.1", optional = true }
 substrate-serializer = { path = "../serializer" }
 pretty_assertions = "0.5"
 heapsize = "0.4"
+hex-literal = "0.1"
+rand = "0.6"
 
 [features]
 default = ["std"]

core/primitives/benches/benches.rs

@@ -0,0 +1,60 @@
+// Copyright 2019 Parity Technologies
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// TODO: Move benchmark to criterion #2354
+#![feature(test)]
+
+extern crate test;
+use hex_literal::{hex, hex_impl};
+use substrate_primitives::hashing::{twox_128, blake2_128};
+
+
+const MAX_KEY_SIZE: u32 = 32;
+
+fn data_set() -> Vec<Vec<u8>> {
+	use rand::SeedableRng;
+	use rand::Rng;
+
+	let rnd: [u8; 32] = rand::rngs::StdRng::seed_from_u64(12).gen();
+	let mut rnd = rnd.iter().cycle();
+	let mut res = Vec::new();
+	for size in 1..=MAX_KEY_SIZE {
+		for _ in 0..1_000 {
+			let value = (0..size)
+				.map(|_| rnd.next().unwrap().clone())
+				.collect();
+			res.push(value);
+		}
+	}
+	res
+}
+
+fn bench_hash_128(b: &mut test::Bencher, f: &Fn(&[u8]) -> [u8; 16]) {
+	let data_set = data_set();
+	b.iter(|| {
+		for data in &data_set {
+			let _a = f(data);
+		}
+	});
+}
+
+#[bench]
+fn bench_blake2_128(b: &mut test::Bencher) {
+	bench_hash_128(b, &blake2_128);
+}
+
+#[bench]
+fn bench_twox_128(b: &mut test::Bencher) {
+	bench_hash_128(b, &twox_128);
+}

core/primitives/src/hashing.rs

@@ -55,6 +55,23 @@ pub fn blake2_128(data: &[u8]) -> [u8; 16] {
 	r
 }
 
+/// Do a XX 64-bit hash and place result in `dest`.
+pub fn twox_64_into(data: &[u8], dest: &mut [u8; 8]) {
+	use ::core::hash::Hasher;
+	let mut h0 = twox_hash::XxHash::with_seed(0);
+	h0.write(data);
+	let r0 = h0.finish();
+	use byteorder::{ByteOrder, LittleEndian};
+	LittleEndian::write_u64(&mut dest[0..8], r0);
+}
+
+/// Do a XX 64-bit hash and return result.
+pub fn twox_64(data: &[u8]) -> [u8; 8] {
+	let mut r: [u8; 8] = [0; 8];
+	twox_64_into(data, &mut r);
+	r
+}
+
 /// Do a XX 128-bit hash and place result in `dest`.
 pub fn twox_128_into(data: &[u8], dest: &mut [u8; 16]) {
 	use ::core::hash::Hasher;

core/primitives/src/lib.rs

@@ -46,7 +46,7 @@ pub use impl_serde::serialize as bytes;
 #[cfg(feature = "std")]
 pub mod hashing;
 #[cfg(feature = "std")]
-pub use hashing::{blake2_256, twox_128, twox_256};
+pub use hashing::{blake2_128, blake2_256, twox_64, twox_128, twox_256};
 #[cfg(feature = "std")]
 pub mod hexdisplay;
 pub mod crypto;

core/rpc/src/state/tests.rs

@@ -17,7 +17,7 @@
 use super::*;
 use self::error::{Error, ErrorKind};
 
-use sr_io::twox_128;
+use sr_io::blake2_256;
 use assert_matches::assert_matches;
 use consensus::BlockOrigin;
 use test_client::{self, runtime, AccountKeyring, TestClient, BlockBuilderExt};
@@ -88,7 +88,7 @@ fn should_send_initial_storage_changes_and_notifications() {
 	{
 		let api = State::new(Arc::new(test_client::new()), Subscriptions::new(remote));
 
-		let alice_balance_key = twox_128(&test_runtime::system::balance_of_key(AccountKeyring::Alice.into()));
+		let alice_balance_key = blake2_256(&test_runtime::system::balance_of_key(AccountKeyring::Alice.into()));
 
 		api.subscribe_storage(Default::default(), subscriber, Some(vec![
 			StorageKey(alice_balance_key.to_vec()),
@@ -147,7 +147,7 @@ fn should_query_storage() {
 		let block2_hash = add_block(1);
 		let genesis_hash = client.genesis_hash();
 
-		let alice_balance_key = twox_128(&test_runtime::system::balance_of_key(AccountKeyring::Alice.into()));
+		let alice_balance_key = blake2_256(&test_runtime::system::balance_of_key(AccountKeyring::Alice.into()));
 
 		let mut expected = vec![
 			StorageChangeSet {

core/sr-io/with_std.rs

@@ -18,8 +18,8 @@
 pub use parity_codec as codec;
 // re-export hashing functions.
 pub use primitives::{
-	blake2_256, twox_128, twox_256, ed25519, Blake2Hasher, sr25519,
-	Pair
+	blake2_128, blake2_256, twox_128, twox_256, twox_64, ed25519, Blake2Hasher,
+	sr25519, Pair
 };
 pub use tiny_keccak::keccak256 as keccak_256;
 // Switch to this after PoC-3

core/sr-io/without_std.rs

@@ -273,7 +273,9 @@ extern_functions! {
 
 	/// Hash calculation and verification
 	fn ext_blake2_256_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8);
+	fn ext_blake2_128(data: *const u8, len: u32, out: *mut u8);
 	fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8);
+	fn ext_twox_64(data: *const u8, len: u32, out: *mut u8);
 	fn ext_twox_128(data: *const u8, len: u32, out: *mut u8);
 	fn ext_twox_256(data: *const u8, len: u32, out: *mut u8);
 	fn ext_keccak_256(data: *const u8, len: u32, out: *mut u8);
@@ -544,6 +546,15 @@ pub fn blake2_256(data: &[u8]) -> [u8; 32] {
 	result
 }
 
+/// Conduct a 128-bit Blake2 hash.
+pub fn blake2_128(data: &[u8]) -> [u8; 16] {
+	let mut result: [u8; 16] = Default::default();
+	unsafe {
+		ext_blake2_128.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr());
+	}
+	result
+}
+
 /// Conduct a 256-bit Keccak hash.
 pub fn keccak_256(data: &[u8]) -> [u8; 32] {
 	let mut result: [u8; 32] = Default::default();
@@ -571,6 +582,15 @@ pub fn twox_128(data: &[u8]) -> [u8; 16] {
 	result
 }
 
+/// Conduct two XX hashes to give a 64-bit result.
+pub fn twox_64(data: &[u8]) -> [u8; 8] {
+	let mut result: [u8; 8] = Default::default();
+	unsafe {
+		ext_twox_64.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr());
+	}
+	result
+}
+
 /// Verify a ed25519 signature.
 pub fn ed25519_verify<P: AsRef<[u8]>>(sig: &[u8; 64], msg: &[u8], pubkey: P) -> bool {
 	unsafe {

core/sr-primitives/src/lib.rs

@@ -90,14 +90,6 @@ pub use serde_derive::{Serialize, Deserialize};
 /// Complex storage builder stuff.
 #[cfg(feature = "std")]
 pub trait BuildStorage: Sized {
-	/// Hash given slice.
-	///
-	/// Default to xx128 hashing.
-	fn hash(data: &[u8]) -> [u8; 16] {
-		let r = runtime_io::twox_128(data);
-		log::trace!(target: "build_storage", "{} <= {}", substrate_primitives::hexdisplay::HexDisplay::from(&r), ascii_format(data));
-		r
-	}
 	/// Build the storage out of this builder.
 	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
 		let mut storage = Default::default();

core/test-runtime/src/genesismap.rs

@@ -17,7 +17,7 @@
 //! Tool for creating the genesis block.
 
 use std::collections::HashMap;
-use runtime_io::twox_128;
+use runtime_io::{blake2_256, twox_128};
 use super::AccountId;
 use parity_codec::{Encode, KeyedVec, Joiner};
 use primitives::{ChangesTrieConfiguration, map, storage::well_known_keys};
@@ -47,7 +47,7 @@ impl GenesisConfig {
 		let wasm_runtime = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm").to_vec();
 		let mut map: HashMap<Vec<u8>, Vec<u8>> = self.balances.iter()
 			.map(|&(ref account, balance)| (account.to_keyed_vec(b"balance:"), vec![].and(&balance)))
-			.map(|(k, v)| (twox_128(&k[..])[..].to_vec(), v.to_vec()))
+			.map(|(k, v)| (blake2_256(&k[..])[..].to_vec(), v.to_vec()))
 			.chain(vec![
 				(well_known_keys::CODE.into(), wasm_runtime),
 				(well_known_keys::HEAP_PAGES.into(), vec![].and(&(16 as u64))),