feat: abstract Merk storage

Cargo.toml

@@ -2,5 +2,6 @@
 members = [
     "grovedb",
     "merk",
-    "node-grove"
+    "node-grove",
+    "storage",
 ]

grovedb/Cargo.toml

@@ -10,5 +10,6 @@ thiserror = "1.0.30"
 tempdir = "0.3.7"
 bincode = "1.3.3"
 serde = { version = "1.0.130", features = ["derive"] }
+storage = { path = "../storage" }
 
 [features]

grovedb/src/lib.rs

@@ -1,4 +1,3 @@
-#![feature(trivial_bounds)]
 mod subtree;
 #[cfg(test)]
 mod tests;
@@ -9,9 +8,13 @@ use std::{
     rc::Rc,
 };
 
-use merk::{self, rocksdb, Merk};
+use merk::{self, Merk};
 use rs_merkle::{algorithms::Sha256, MerkleTree};
 pub use subtree::Element;
+use storage::{
+    rocksdb_storage::{PrefixedRocksDbStorage, PrefixedRocksDbStorageError},
+    Storage,
+};
 
 /// Limit of possible indirections
 const MAX_REFERENCE_HOPS: usize = 10;
@@ -24,7 +27,7 @@ const ROOT_LEAFS_SERIALIZED_KEY: &[u8] = b"rootLeafsSerialized";
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("rocksdb error")]
-    RocksDBError(#[from] merk::rocksdb::Error),
+    RocksDBError(#[from] PrefixedRocksDbStorageError),
     #[error("unable to open Merk db")]
     MerkError(merk::Error),
     #[error("invalid path: {0}")]
@@ -46,57 +49,65 @@ impl From<merk::Error> for Error {
 pub struct GroveDb {
     root_tree: MerkleTree<Sha256>,
     root_leaf_keys: HashMap<Vec<u8>, usize>,
-    subtrees: HashMap<Vec<u8>, Merk>,
-    db: Rc<rocksdb::DB>,
+    subtrees: HashMap<Vec<u8>, Merk<PrefixedRocksDbStorage>>,
+    meta_storage: PrefixedRocksDbStorage,
+    db: Rc<storage::rocksdb_storage::DB>,
 }
 
 impl GroveDb {
     pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
-        let db = Rc::new(rocksdb::DB::open_cf_descriptors(
-            &Merk::default_db_opts(),
-            path,
-            merk::column_families(),
-        )?);
+        let db = Rc::new(
+            storage::rocksdb_storage::DB::open_cf_descriptors(
+                &storage::rocksdb_storage::default_db_opts(),
+                path,
+                storage::rocksdb_storage::column_families(),
+            )
+            .map_err(Into::<PrefixedRocksDbStorageError>::into)?,
+        );
+        let meta_storage = PrefixedRocksDbStorage::new(db.clone(), Vec::new())?;
 
         let mut subtrees = HashMap::new();
         // TODO: owned `get` is not required for deserialization
-        if let Some(prefixes_serialized) = db.get(SUBTRESS_SERIALIZED_KEY)? {
+        if let Some(prefixes_serialized) = meta_storage.get_meta(SUBTRESS_SERIALIZED_KEY)? {
             let subtrees_prefixes: Vec<Vec<u8>> = bincode::deserialize(&prefixes_serialized)?;
             for prefix in subtrees_prefixes {
-                let subtree_merk = Merk::open(db.clone(), prefix.to_vec())?;
+                let subtree_merk =
+                    Merk::open(PrefixedRocksDbStorage::new(db.clone(), prefix.to_vec())?)?;
                 subtrees.insert(prefix.to_vec(), subtree_merk);
             }
         }
 
         // TODO: owned `get` is not required for deserialization
-        let root_leaf_keys: HashMap<Vec<u8>, usize> =
-            if let Some(root_leaf_keys_serialized) = db.get(ROOT_LEAFS_SERIALIZED_KEY)? {
-                bincode::deserialize(&root_leaf_keys_serialized)?
-            } else {
-                HashMap::new()
-            };
+        let root_leaf_keys: HashMap<Vec<u8>, usize> = if let Some(root_leaf_keys_serialized) =
+            meta_storage.get_meta(ROOT_LEAFS_SERIALIZED_KEY)?
+        {
+            bincode::deserialize(&root_leaf_keys_serialized)?
+        } else {
+            HashMap::new()
+        };
 
         Ok(GroveDb {
             root_tree: Self::build_root_tree(&subtrees, &root_leaf_keys),
-            db: db.clone(),
+            db,
             subtrees,
             root_leaf_keys,
+            meta_storage,
         })
     }
 
     fn store_subtrees_keys_data(&self) -> Result<(), Error> {
         let prefixes: Vec<Vec<u8>> = self.subtrees.keys().map(|x| x.clone()).collect();
-        self.db
-            .put(SUBTRESS_SERIALIZED_KEY, bincode::serialize(&prefixes)?)?;
-        self.db.put(
+        self.meta_storage
+            .put_meta(SUBTRESS_SERIALIZED_KEY, &bincode::serialize(&prefixes)?)?;
+        self.meta_storage.put_meta(
             ROOT_LEAFS_SERIALIZED_KEY,
-            bincode::serialize(&self.root_leaf_keys)?,
+            &bincode::serialize(&self.root_leaf_keys)?,
         )?;
         Ok(())
     }
 
     fn build_root_tree(
-        subtrees: &HashMap<Vec<u8>, Merk>,
+        subtrees: &HashMap<Vec<u8>, Merk<PrefixedRocksDbStorage>>,
         root_leaf_keys: &HashMap<Vec<u8>, usize>,
     ) -> MerkleTree<Sha256> {
         let mut leaf_hashes: Vec<[u8; 32]> = vec![[0; 32]; root_leaf_keys.len()];
@@ -121,13 +132,17 @@ impl GroveDb {
         match &mut element {
             Element::Tree(subtree_root_hash) => {
                 // Helper closure to create a new subtree under path + key
-                let create_subtree_merk = || -> Result<(Vec<u8>, Merk), Error> {
-                    let compressed_path_subtree = Self::compress_path(path, Some(&key));
-                    Ok((
-                        compressed_path_subtree.clone(),
-                        Merk::open(self.db.clone(), compressed_path_subtree)?,
-                    ))
-                };
+                let create_subtree_merk =
+                    || -> Result<(Vec<u8>, Merk<PrefixedRocksDbStorage>), Error> {
+                        let compressed_path_subtree = Self::compress_path(path, Some(&key));
+                        Ok((
+                            compressed_path_subtree.clone(),
+                            Merk::open(PrefixedRocksDbStorage::new(
+                                self.db.clone(),
+                                compressed_path_subtree,
+                            )?)?,
+                        ))
+                    };
                 if path.is_empty() {
                     // Add subtree to the root tree
 
@@ -200,7 +215,7 @@ impl GroveDb {
         Element::get(&merk, key)
     }
 
-    fn follow_reference<'a>(&self, mut path: Vec<Vec<u8>>) -> Result<subtree::Element, Error> {
+    fn follow_reference(&self, mut path: Vec<Vec<u8>>) -> Result<subtree::Element, Error> {
         let mut hops_left = MAX_REFERENCE_HOPS;
         let mut current_element;
         let mut visited = HashSet::new();

grovedb/src/subtree.rs

@@ -3,6 +3,7 @@
 //! Merk API to GroveDB needs.
 use merk::Op;
 use serde::{Deserialize, Serialize};
+use storage::rocksdb_storage::PrefixedRocksDbStorage;
 
 use crate::{Error, Merk};
 
@@ -27,7 +28,7 @@ impl Element {
 
     /// Get an element from Merk under a key; path should be resolved and proper
     /// Merk should be loaded by this moment
-    pub fn get(merk: &Merk, key: &[u8]) -> Result<Element, Error> {
+    pub fn get(merk: &Merk<PrefixedRocksDbStorage>, key: &[u8]) -> Result<Element, Error> {
         let element = bincode::deserialize(
             merk.get(&key)?
                 .ok_or(Error::InvalidPath("key not found in Merk"))?
@@ -38,7 +39,11 @@ impl Element {
 
     /// Insert an element in Merk under a key; path should be resolved and
     /// proper Merk should be loaded by this moment
-    pub fn insert(&self, merk: &mut Merk, key: Vec<u8>) -> Result<(), Error> {
+    pub fn insert(
+        &self,
+        merk: &mut Merk<PrefixedRocksDbStorage>,
+        key: Vec<u8>,
+    ) -> Result<(), Error> {
         let batch = [(key, Op::Put(bincode::serialize(self)?))];
         merk.apply(&batch, &[]).map_err(|e| e.into())
     }

merk/Cargo.toml

@@ -8,6 +8,10 @@ license = "MIT"
 
 [dependencies]
 tempdir = "0.3.7"
+storage = { path = "../storage" }
+thiserror = "1.0.30"
+failure = "0.1.8"
+rocksdb = "0.17.0"
 
 [dependencies.time]
 version = "0.1.42"
@@ -29,10 +33,6 @@ optional = true
 version = "1.3.2" 
 optional = true
 
-[dependencies.failure]
-version = "0.1.6"
-optional = true
-
 [dependencies.ed]
 version = "0.1.6"
 optional = true
@@ -46,13 +46,6 @@ version = "0.8.3"
 features = ["small_rng"]
 optional = true
 
-[dependencies.rocksdb]
-git = "https://github.com/rust-rocksdb/rust-rocksdb"
-version = "0.16.0"
-rev = "v0.16.0"
-default-features = false
-optional = true
-
 [dependencies.jemallocator]
 version = "0.3.2"
 features = ["disable_initial_exec_tls"]
@@ -61,18 +54,15 @@ optional = true
 [features]
 default = ["full", "verify"]
 full = ["rand", 
-        "rocksdb", 
-        "time", 
+        "time",
         "hex", 
         "colored",
         "num_cpus",
         "byteorder",
-        "failure",
         "ed",
         "blake3",
         "jemallocator"
 ]
 verify = ["ed",
-          "failure",
           "blake3"
 ]

merk/src/lib.rs

@@ -1,18 +1,11 @@
 #![feature(map_first_last)]
 
-#[global_allocator]
-#[cfg(feature = "full")]
-static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
-
-#[cfg(feature = "full")]
-pub use rocksdb;
-
 /// Error and Result types.
 mod error;
 /// The top-level store API.
 #[cfg(feature = "full")]
 mod merk;
-pub use crate::merk::column_families;
+
 /// Provides a container type that allows temporarily taking ownership of a
 /// value.
 // TODO: move this into its own crate
@@ -29,8 +22,8 @@ pub use error::{Error, Result};
 #[allow(deprecated)]
 pub use proofs::query::verify_query;
 pub use proofs::query::{execute_proof, verify};
-pub use tree::{Batch, BatchEntry, Hash, Op, PanicSource, HASH_LENGTH};
+pub use tree::{BatchEntry, Hash, MerkBatch, Op, PanicSource, HASH_LENGTH};
 
-#[cfg(feature = "full")]
-// pub use crate::merk::{chunks, restore, Merk};
-pub use crate::merk::{chunks, Merk};
+// #[cfg(feature = "full")]
+// // pub use crate::merk::{chunks, restore, Merk};
+pub use crate::merk::Merk;