From 5aea1ce776d862a556c026990b90d3dbd02d5a9e Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 15 Nov 2024 17:53:00 +0100 Subject: [PATCH 1/2] feat: log proof log errors to db --- Cargo.toml | 2 +- .../query_dpns_contested_resources.rs | 38 +++++- src/database/initialization.rs | 69 ++++++++-- src/database/mod.rs | 1 + src/database/proof_log.rs | 125 ++++++++++++++++++ src/database/settings.rs | 14 ++ src/model/mod.rs | 1 + src/model/proof_log_item.rs | 94 +++++++++++++ 8 files changed, 331 insertions(+), 13 deletions(-) create mode 100644 src/database/proof_log.rs create mode 100644 src/model/proof_log_item.rs diff --git a/Cargo.toml b/Cargo.toml index 8d6fae730..5152a5e0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ strum = { version = "0.26.1", features = ["derive"] } bs58 = "0.5.0" base64 = "0.22.1" copypasta = "0.10.1" -dash-sdk = { git = "https://github.com/dashpay/platform", rev = "994f7627080e4ab490f1dbf0f09c0fb37d0fbbb0" } +dash-sdk = { git = "https://github.com/dashpay/platform", branch = "test/testWithoutSpan2" } thiserror = "1" serde = "1.0.197" serde_json = "1.0.120" diff --git a/src/backend_task/contested_names/query_dpns_contested_resources.rs b/src/backend_task/contested_names/query_dpns_contested_resources.rs index e300d7d70..9d7b9a892 100644 --- a/src/backend_task/contested_names/query_dpns_contested_resources.rs +++ b/src/backend_task/contested_names/query_dpns_contested_resources.rs @@ -1,5 +1,6 @@ use crate::app::TaskResult; use crate::context::AppContext; +use crate::model::proof_log_item::{ProofLogItem, RequestType}; use dash_sdk::dpp::data_contract::accessors::v0::DataContractV0Getters; use dash_sdk::dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; use dash_sdk::drive::query::vote_polls_by_document_type_query::VotePollsByDocumentTypeQuery; @@ -33,11 +34,44 @@ impl AppContext { order_ascending: true, }; - let contested_resources = - ContestedResource::fetch_many(&sdk, query) + let (contested_resources, metadata, proof) = + ContestedResource::fetch_many_with_metadata_and_proof(&sdk, query.clone(), None) .await .map_err(|e| { tracing::error!("error fetching contested resources: {}", e); + if let dash_sdk::Error::Proof(dash_sdk::ProofVerifierError::GroveDBError { + proof_bytes, + height, + time_ms, + error, + }) = &e + { + // Encode the query using bincode + let encoded_query = + match bincode::encode_to_vec(&query, bincode::config::standard()) + .map_err(|encode_err| { + tracing::error!("error encoding query: {}", encode_err); + format!("error encoding query: {}", encode_err) + }) { + Ok(encoded_query) => encoded_query, + Err(e) => return e, + }; + + if let Err(e) = self + .db + .insert_proof_log_item(ProofLogItem { + request_type: RequestType::GetContestedResources, + request_bytes: encoded_query, + height: *height, + time_ms: *time_ms, + proof_bytes: proof_bytes.clone(), + error: Some(error.clone()), + }) + .map_err(|e| e.to_string()) + { + return e; + } + } format!("error fetching contested resources: {}", e) })?; diff --git a/src/database/initialization.rs b/src/database/initialization.rs index 01b157c34..8c9ea3fe2 100644 --- a/src/database/initialization.rs +++ b/src/database/initialization.rs @@ -5,7 +5,7 @@ use std::fs; use std::path::Path; pub const DEFAULT_DB_VERSION: u16 = 1; -pub const MIN_SUPPORTED_DB_VERSION: u16 = 1; +pub const CURRENT_DB_VERSION: u16 = 2; pub const DEFAULT_NETWORK: &str = "dash"; impl Database { @@ -16,17 +16,46 @@ impl Database { self.set_default_version()?; } else { // Perform version check and back up and recreate the database if outdated. - if self.is_outdated()? { - self.backup_and_recreate_db(db_file_path)?; - self.create_tables()?; - self.set_default_version()?; - println!("Database reinitialized with default settings."); + if let Some(current_version) = self.is_outdated()? { + self.backup_db(db_file_path)?; + if !self + .try_perform_migration(current_version, CURRENT_DB_VERSION) + .is_ok() + { + // The migration failed + self.recreate_db(db_file_path)?; + self.create_tables()?; + self.set_default_version()?; + println!("Database reinitialized with default settings."); + } } } Ok(()) } + fn apply_version_changes(&self, version: u16) -> rusqlite::Result<()> { + match version { + 2 => { + self.initialize_proof_log_table()?; + } + _ => {} + } + + Ok(()) + } + fn try_perform_migration( + &self, + original_version: u16, + to_version: u16, + ) -> rusqlite::Result<()> { + for version in (original_version + 1)..=to_version { + self.apply_version_changes(version)?; + self.update_database_version(version)?; + } + Ok(()) + } + /// Checks if the `settings` table is empty or missing, indicating a first-time setup. fn is_first_time_setup(&self) -> rusqlite::Result { let conn = self.conn.lock().unwrap(); @@ -50,7 +79,7 @@ impl Database { } /// Checks if the current database version is below the minimum supported version. - fn is_outdated(&self) -> rusqlite::Result { + fn is_outdated(&self) -> rusqlite::Result> { let conn = self.conn.lock().unwrap(); let version: u16 = conn .query_row( @@ -59,11 +88,15 @@ impl Database { |row| row.get(0), ) .unwrap_or(0); // Default to version 0 if there's no version set - Ok(version < MIN_SUPPORTED_DB_VERSION) + if version < CURRENT_DB_VERSION { + Ok(Some(version)) + } else { + Ok(None) + } } /// Backs up the existing database with a unique timestamped filename, recreates `data.db`, and refreshes the connection. - fn backup_and_recreate_db(&self, db_file_path: &Path) -> rusqlite::Result<()> { + fn backup_db(&self, db_file_path: &Path) -> rusqlite::Result<()> { if db_file_path.exists() { // Create a "backups" folder in the same directory as `data.db` if not exists let backups_dir = db_file_path @@ -82,11 +115,25 @@ impl Database { let backup_path = backups_dir.join(backup_filename); // Rename `data.db` to the unique backup file - fs::rename(db_file_path, &backup_path) + fs::copy(db_file_path, &backup_path) .map_err(|e| rusqlite::Error::ToSqlConversionFailure(e.into()))?; println!("Old database backed up to {:?}", backup_path); } + Ok(()) + } + /// Backs up the existing database with a unique timestamped filename, recreates `data.db`, and refreshes the connection. + fn recreate_db(&self, db_file_path: &Path) -> rusqlite::Result<()> { + // Remove the existing database file if it exists + if db_file_path.exists() { + fs::remove_file(db_file_path).map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + Box::new(e), + ) + })?; + } // Create a new empty `data.db` file and set up the initial schema let new_conn = Connection::open(db_file_path)?; @@ -291,6 +338,8 @@ impl Database { [], )?; + self.initialize_proof_log_table()?; + Ok(()) } diff --git a/src/database/mod.rs b/src/database/mod.rs index d3a068dbb..83d06cf13 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -3,6 +3,7 @@ mod contested_names; mod contracts; mod identities; mod initialization; +mod proof_log; mod settings; mod utxo; mod wallet; diff --git a/src/database/proof_log.rs b/src/database/proof_log.rs new file mode 100644 index 000000000..42d3e718d --- /dev/null +++ b/src/database/proof_log.rs @@ -0,0 +1,125 @@ +use crate::database::Database; +use crate::model::proof_log_item::{ProofLogItem, RequestType}; +use rusqlite::params; +use std::ops::Range; + +impl Database { + pub fn initialize_proof_log_table(&self) -> rusqlite::Result<()> { + // Create the proof log tree + self.execute( + "CREATE TABLE IF NOT EXISTS proof_log ( + proof_id INTEGER PRIMARY KEY AUTOINCREMENT, + request_type INTEGER NOT NULL, + request_bytes BLOB NOT NULL, + height INTEGER NOT NULL, + time_ms INTEGER NOT NULL, + proof_bytes BLOB NOT NULL, + error TEXT + )", + [], + )?; + + // Create an index on request_type and time for combined queries + self.execute( + "CREATE INDEX IF NOT EXISTS idx_proof_log_request_type_time ON proof_log (request_type, time_ms)", + [], + )?; + + // Create an index on time for queries ordered by time + self.execute( + "CREATE INDEX IF NOT EXISTS idx_proof_log_time ON proof_log (time_ms)", + [], + )?; + + // Index for error, request_type, and time + self.execute( + "CREATE INDEX IF NOT EXISTS idx_proof_log_error_request_type_time ON proof_log (error, request_type, time_ms)", + [], + )?; + + // Index for error and time + self.execute( + "CREATE INDEX IF NOT EXISTS idx_proof_log_error_time ON proof_log (error, time_ms)", + [], + )?; + Ok(()) + } + + /// Inserts a new ProofLogItem into the proof_log table + pub fn insert_proof_log_item(&self, item: ProofLogItem) -> rusqlite::Result<()> { + let conn = self.conn.lock().unwrap(); + + // Convert RequestType to u8 + let request_type_int: u8 = item.request_type.into(); + + conn.execute( + "INSERT INTO proof_log (request_type, request_bytes, height, time_ms, proof_bytes, error) + VALUES (?, ?, ?, ?, ?, ?)", + params![ + request_type_int, + item.request_bytes, + item.height, + item.time_ms, + item.proof_bytes, + item.error, + ], + )?; + + Ok(()) + } + + /// Retrieves ProofLogItems with options for filtering and pagination + pub fn get_proof_log_items( + &self, + only_get_errored: bool, + range: Range, + ) -> rusqlite::Result> { + let conn = self.conn.lock().unwrap(); + + // Build the query based on the only_get_errored flag + let mut query = String::from( + "SELECT request_type, request_bytes, height, time_ms, proof_bytes, error FROM proof_log", + ); + + if only_get_errored { + query.push_str(" WHERE error IS NOT NULL"); + } + + query.push_str(" ORDER BY time_ms DESC LIMIT ? OFFSET ?"); + + let mut stmt = conn.prepare(&query)?; + + let proof_log_iter = + stmt.query_map(params![range.end - range.start, range.start], |row| { + let request_type_int: u8 = row.get(0)?; + let request_bytes: Vec = row.get(1)?; + let height: u64 = row.get(2)?; + let time_ms: u64 = row.get(3)?; + let proof_bytes: Vec = row.get(4)?; + let error: Option = row.get(5)?; + + // Convert u8 to RequestType + let request_type = RequestType::try_from(request_type_int).map_err(|_| { + rusqlite::Error::FromSqlConversionFailure( + request_type_int as usize, + rusqlite::types::Type::Integer, + Box::new(std::fmt::Error), + ) + })?; + + Ok(ProofLogItem { + request_type, + request_bytes, + height, + time_ms, + proof_bytes, + error, + }) + })?; + + // Collect the results into a vector + let items: rusqlite::Result> = proof_log_iter.collect(); + + items + } +} diff --git a/src/database/settings.rs b/src/database/settings.rs index 6dc252dd6..edec45cf6 100644 --- a/src/database/settings.rs +++ b/src/database/settings.rs @@ -43,6 +43,20 @@ impl Database { Ok(()) } + + /// Updates the database version in the settings table. + pub fn update_database_version(&self, new_version: u16) -> Result<()> { + // Ensure the database version is updated + self.execute( + "UPDATE settings + SET database_version = ? + WHERE id = 1", + params![new_version], + )?; + + Ok(()) + } + /// Retrieves the settings from the database. pub fn get_settings(&self) -> Result)>> { // Query the settings row diff --git a/src/model/mod.rs b/src/model/mod.rs index 1e93d702b..105cce062 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,5 +1,6 @@ pub mod contested_name; pub mod password_info; +pub mod proof_log_item; pub mod qualified_contract; pub mod qualified_identity; pub mod wallet; diff --git a/src/model/proof_log_item.rs b/src/model/proof_log_item.rs new file mode 100644 index 000000000..caea63608 --- /dev/null +++ b/src/model/proof_log_item.rs @@ -0,0 +1,94 @@ +#[derive(Debug, Clone, Copy)] +pub enum RequestType { + BroadcastStateTransition = 1, + GetIdentity = 2, + GetIdentityKeys = 3, + GetIdentitiesContractKeys = 4, + GetIdentityNonce = 5, + GetIdentityContractNonce = 6, + GetIdentityBalance = 7, + GetIdentitiesBalances = 8, + GetIdentityBalanceAndRevision = 9, + GetEvonodesProposedEpochBlocksByIds = 10, + GetEvonodesProposedEpochBlocksByRange = 11, + GetProofs = 12, + GetDataContract = 13, + GetDataContractHistory = 14, + GetDataContracts = 15, + GetDocuments = 16, + GetIdentityByPublicKeyHash = 17, + WaitForStateTransitionResult = 18, + GetConsensusParams = 19, + GetProtocolVersionUpgradeState = 20, + GetProtocolVersionUpgradeVoteStatus = 21, + GetEpochsInfo = 22, + GetContestedResources = 23, + GetContestedResourceVoteState = 24, + GetContestedResourceVotersForIdentity = 25, + GetContestedResourceIdentityVotes = 26, + GetVotePollsByEndDate = 27, + GetPrefundedSpecializedBalance = 28, + GetTotalCreditsInPlatform = 29, + GetPathElements = 30, + GetStatus = 31, + GetCurrentQuorumsInfo = 32, +} + +use std::convert::TryFrom; + +impl From for u8 { + fn from(request_type: RequestType) -> Self { + request_type as u8 + } +} + +impl TryFrom for RequestType { + type Error = (); + + fn try_from(value: u8) -> Result { + match value { + 1 => Ok(RequestType::BroadcastStateTransition), + 2 => Ok(RequestType::GetIdentity), + 3 => Ok(RequestType::GetIdentityKeys), + 4 => Ok(RequestType::GetIdentitiesContractKeys), + 5 => Ok(RequestType::GetIdentityNonce), + 6 => Ok(RequestType::GetIdentityContractNonce), + 7 => Ok(RequestType::GetIdentityBalance), + 8 => Ok(RequestType::GetIdentitiesBalances), + 9 => Ok(RequestType::GetIdentityBalanceAndRevision), + 10 => Ok(RequestType::GetEvonodesProposedEpochBlocksByIds), + 11 => Ok(RequestType::GetEvonodesProposedEpochBlocksByRange), + 12 => Ok(RequestType::GetProofs), + 13 => Ok(RequestType::GetDataContract), + 14 => Ok(RequestType::GetDataContractHistory), + 15 => Ok(RequestType::GetDataContracts), + 16 => Ok(RequestType::GetDocuments), + 17 => Ok(RequestType::GetIdentityByPublicKeyHash), + 18 => Ok(RequestType::WaitForStateTransitionResult), + 19 => Ok(RequestType::GetConsensusParams), + 20 => Ok(RequestType::GetProtocolVersionUpgradeState), + 21 => Ok(RequestType::GetProtocolVersionUpgradeVoteStatus), + 22 => Ok(RequestType::GetEpochsInfo), + 23 => Ok(RequestType::GetContestedResources), + 24 => Ok(RequestType::GetContestedResourceVoteState), + 25 => Ok(RequestType::GetContestedResourceVotersForIdentity), + 26 => Ok(RequestType::GetContestedResourceIdentityVotes), + 27 => Ok(RequestType::GetVotePollsByEndDate), + 28 => Ok(RequestType::GetPrefundedSpecializedBalance), + 29 => Ok(RequestType::GetTotalCreditsInPlatform), + 30 => Ok(RequestType::GetPathElements), + 31 => Ok(RequestType::GetStatus), + 32 => Ok(RequestType::GetCurrentQuorumsInfo), + _ => Err(()), + } + } +} + +pub struct ProofLogItem { + pub request_type: RequestType, + pub request_bytes: Vec, + pub height: u64, + pub time_ms: u64, + pub proof_bytes: Vec, + pub error: Option, +} From d16c8f48ecf44b5144d0700a7c26fdbc140c3ae6 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Fri, 15 Nov 2024 18:30:49 +0100 Subject: [PATCH 2/2] Update src/database/initialization.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/database/initialization.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/database/initialization.rs b/src/database/initialization.rs index 8c9ea3fe2..15d047f0e 100644 --- a/src/database/initialization.rs +++ b/src/database/initialization.rs @@ -18,11 +18,9 @@ impl Database { // Perform version check and back up and recreate the database if outdated. if let Some(current_version) = self.is_outdated()? { self.backup_db(db_file_path)?; - if !self - .try_perform_migration(current_version, CURRENT_DB_VERSION) - .is_ok() - { + if let Err(e) = self.try_perform_migration(current_version, CURRENT_DB_VERSION) { // The migration failed + println!("Migration failed: {:?}", e); self.recreate_db(db_file_path)?; self.create_tables()?; self.set_default_version()?;