From 2aae0c42a9daa191cd2c5cfc0d42f84394e66239 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 27 Apr 2026 15:53:37 +0200 Subject: [PATCH 1/4] metrics for real minter balance --- minter/src/balance_check.rs | 47 +++++++++++++++++++++++++++++++++++++ minter/src/lib.rs | 1 + minter/src/main.rs | 4 ++++ minter/src/metrics.rs | 17 ++++++++++++++ minter/src/rpc/mod.rs | 35 +++++++++++++++++++++++++-- minter/src/state/mod.rs | 32 +++++++++++++++++++++++++ minter/src/state/tests.rs | 1 + 7 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 minter/src/balance_check.rs diff --git a/minter/src/balance_check.rs b/minter/src/balance_check.rs new file mode 100644 index 00000000..7046724d --- /dev/null +++ b/minter/src/balance_check.rs @@ -0,0 +1,47 @@ +use crate::{ + address::{lazy_get_schnorr_master_key, minter_address}, + guard::TimerGuard, + rpc::get_balance, + runtime::CanisterRuntime, + state::{TaskType, mutate_state}, +}; +use canlog::log; +use cksol_types_internal::log::Priority; +use std::time::Duration; + +pub const REFRESH_REAL_BALANCE_DELAY: Duration = Duration::from_secs(24 * 60 * 60); + +/// Refresh the cached on-chain balance of the minter's main account. +/// Each call makes an RPC request to the Solana network, so this runs at most +/// once per day (see [`REFRESH_REAL_BALANCE_DELAY`]). +pub async fn refresh_real_balance(runtime: R) { + let _guard = match TimerGuard::new(TaskType::RefreshRealBalance) { + Ok(guard) => guard, + Err(_) => return, + }; + + let master_key = lazy_get_schnorr_master_key(&runtime).await; + let address = minter_address(&master_key, &runtime); + + match get_balance(&runtime, address).await { + Ok(balance) => { + let observed_at = runtime.time(); + let diff = mutate_state(|s| { + s.record_balance_discrepancy(balance, observed_at); + s.last_balance_discrepancy() + .expect("BUG: just-recorded discrepancy must be present") + .diff_lamports + }); + log!( + Priority::Info, + "Refreshed real balance for minter {address}: {balance} lamports (discrepancy vs. tracked: {diff} lamports)" + ); + } + Err(e) => { + log!( + Priority::Info, + "Failed to refresh real balance for minter {address}: {e}" + ); + } + } +} diff --git a/minter/src/lib.rs b/minter/src/lib.rs index 08a6f526..9a5b5c3c 100644 --- a/minter/src/lib.rs +++ b/minter/src/lib.rs @@ -1,4 +1,5 @@ pub mod address; +pub mod balance_check; pub mod consolidate; mod constants; mod cycles; diff --git a/minter/src/main.rs b/minter/src/main.rs index 4d144410..8c8b6cf2 100644 --- a/minter/src/main.rs +++ b/minter/src/main.rs @@ -2,6 +2,7 @@ use candid::Principal; use canlog::{Log, Sort}; use cksol_minter::{ address::lazy_get_schnorr_master_key, + balance_check::{REFRESH_REAL_BALANCE_DELAY, refresh_real_balance}, consolidate::{DEPOSIT_CONSOLIDATION_DELAY, consolidate_deposits}, deposit::automatic::{POLL_MONITORED_ADDRESSES_DELAY, poll_monitored_addresses}, monitor::{ @@ -370,6 +371,9 @@ fn setup_timers() { ic_cdk_timers::set_timer_interval(POLL_MONITORED_ADDRESSES_DELAY, async || { poll_monitored_addresses(IcCanisterRuntime::new()).await; }); + ic_cdk_timers::set_timer_interval(REFRESH_REAL_BALANCE_DELAY, async || { + refresh_real_balance(IcCanisterRuntime::new()).await; + }); } fn main() {} diff --git a/minter/src/metrics.rs b/minter/src/metrics.rs index 7be18766..d2324939 100644 --- a/minter/src/metrics.rs +++ b/minter/src/metrics.rs @@ -92,6 +92,23 @@ pub fn encode_metrics(w: &mut MetricsEncoder>, s: &State) -> std::io::Re s.balance().metric_value(), "Minter balance in Lamports.", )?; + let (discrepancy_lamports, discrepancy_refresh_seconds) = match s.last_balance_discrepancy() { + Some(observation) => ( + observation.diff_lamports as f64, + (observation.observed_at / 1_000_000_000) as f64, + ), + None => (0.0, 0.0), + }; + w.encode_gauge( + "minter_balance_discrepancy_lamports", + discrepancy_lamports, + "Difference (real on-chain balance minus tracked balance) of the minter's main account at the time of the last successful refresh, in lamports. Both sides are sampled together; refreshed once per day. Reports 0 until the first refresh succeeds.", + )?; + w.encode_gauge( + "minter_balance_discrepancy_last_refresh_timestamp_seconds", + discrepancy_refresh_seconds, + "Unix timestamp (seconds) of the last successful computation of `minter_balance_discrepancy_lamports`. Reports 0 until the first refresh succeeds.", + )?; w.encode_gauge( "post_upgrade_instructions_consumed", storage::with_unstable_metrics(|m| m.post_upgrade_instructions_consumed).metric_value(), diff --git a/minter/src/rpc/mod.rs b/minter/src/rpc/mod.rs index cbbfc860..20f6fc21 100644 --- a/minter/src/rpc/mod.rs +++ b/minter/src/rpc/mod.rs @@ -7,9 +7,10 @@ use cksol_types::ProcessDepositError; use derive_more::From; use ic_canister_runtime::IcError; use sol_rpc_types::{ - CommitmentLevel, ConfirmedTransactionStatusWithSignature, GetSignaturesForAddressParams, - GetTransactionEncoding, MultiRpcResult, RpcError, Slot, + CommitmentLevel, ConfirmedTransactionStatusWithSignature, GetBalanceParams, + GetSignaturesForAddressParams, GetTransactionEncoding, Lamport, MultiRpcResult, RpcError, Slot, }; +use solana_address::Address; use solana_hash::Hash; use solana_signature::Signature; use solana_transaction::Transaction; @@ -162,3 +163,33 @@ pub enum GetSignaturesForAddressError { #[error("Inconsistent RPC results for getSignaturesForAddress")] InconsistentRpcResults, } + +pub async fn get_balance( + runtime: &R, + address: Address, +) -> Result { + let client = read_state(|state| state.sol_rpc_client(runtime.inter_canister_call_runtime())); + let result = client + .get_balance(GetBalanceParams { + pubkey: address.into(), + commitment: Some(CommitmentLevel::Finalized), + min_context_slot: None, + }) + .try_send() + .await; + match result? { + MultiRpcResult::Consistent(Ok(balance)) => Ok(balance), + MultiRpcResult::Consistent(Err(e)) => Err(GetBalanceError::RpcError(e)), + MultiRpcResult::Inconsistent(_) => Err(GetBalanceError::InconsistentRpcResults), + } +} + +#[derive(Debug, PartialEq, Error)] +pub enum GetBalanceError { + #[error("Error while calling SOL RPC canister: {0}")] + IcError(#[from] IcError), + #[error("RPC error while fetching balance: {0}")] + RpcError(RpcError), + #[error("Inconsistent RPC results for getBalance")] + InconsistentRpcResults, +} diff --git a/minter/src/state/mod.rs b/minter/src/state/mod.rs index c265d456..3683e3c7 100644 --- a/minter/src/state/mod.rs +++ b/minter/src/state/mod.rs @@ -109,6 +109,22 @@ pub struct State { consolidation_transactions: InsertionOrderedMap, active_tasks: BTreeSet, balance: Lamport, + /// Last observed difference (`real_balance - tracked_balance`) between the + /// real on-chain balance of the minter's main account and the value + /// tracked by the minter's state, together with the timestamp + /// (nanoseconds since the Unix epoch) at which it was computed. + /// + /// Both sides of the difference are sampled at the same time so that the + /// stored value remains meaningful even though `tracked_balance` keeps + /// changing afterwards. Refreshed once per day by a timer; `None` until + /// the first refresh succeeds. + last_balance_discrepancy: Option, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct BalanceDiscrepancyObservation { + pub diff_lamports: i128, + pub observed_at: u64, } impl State { @@ -246,6 +262,20 @@ impl State { self.balance } + pub fn last_balance_discrepancy(&self) -> Option { + self.last_balance_discrepancy + } + + /// Records the difference between `real_balance` (just observed on-chain) + /// and the tracked balance, sampled at `observed_at`. + pub fn record_balance_discrepancy(&mut self, real_balance: Lamport, observed_at: u64) { + let diff_lamports = real_balance as i128 - self.balance as i128; + self.last_balance_discrepancy = Some(BalanceDiscrepancyObservation { + diff_lamports, + observed_at, + }); + } + pub fn monitored_accounts(&self) -> &InsertionOrderedSet { &self.monitored_accounts } @@ -796,6 +826,7 @@ impl TryFrom for State { consolidation_transactions: InsertionOrderedMap::new(), active_tasks: BTreeSet::new(), balance: 0, + last_balance_discrepancy: None, }; state.validate()?; Ok(state) @@ -843,6 +874,7 @@ pub enum TaskType { ResubmitTransactions, WithdrawalProcessing, PollMonitoredAddresses, + RefreshRealBalance, } /// Details about a consolidation transaction, capturing the individual diff --git a/minter/src/state/tests.rs b/minter/src/state/tests.rs index e3ef55e3..100e5e66 100644 --- a/minter/src/state/tests.rs +++ b/minter/src/state/tests.rs @@ -246,6 +246,7 @@ mod state_from_init_args { consolidation_transactions: InsertionOrderedMap::new(), active_tasks: BTreeSet::new(), balance: 0, + last_balance_discrepancy: None, } ); } From 7f204204994947e25b54a44e23a4b4b69f95f7f2 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 27 Apr 2026 16:08:00 +0200 Subject: [PATCH 2/4] integration test --- integration_tests/src/fixtures.rs | 20 ++++++++++++++++ integration_tests/tests/tests.rs | 38 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/integration_tests/src/fixtures.rs b/integration_tests/src/fixtures.rs index b3bf6ba2..386c1015 100644 --- a/integration_tests/src/fixtures.rs +++ b/integration_tests/src/fixtures.rs @@ -213,6 +213,11 @@ impl MockBuilder { get_signatures_for_address_response(signatures), ) } + + /// Mock for `getBalance` returning the given lamport balance. + pub fn get_balance(self, balance: Lamport) -> Self { + self.expect(get_balance_request(), get_balance_response(balance)) + } } // ── JSON-RPC request matchers and response builders ───────────────────────── @@ -387,3 +392,18 @@ fn get_signatures_for_address_response(signatures: Vec) -> Js "id": 1 })) } + +fn get_balance_request() -> JsonRpcRequestMatcher { + JsonRpcRequestMatcher::with_method("getBalance") +} + +fn get_balance_response(balance: Lamport) -> JsonRpcResponse { + JsonRpcResponse::from(json!({ + "jsonrpc": "2.0", + "result": { + "context": { "slot": 350_000_000_u64, "apiVersion": "2.1.9" }, + "value": balance, + }, + "id": 1 + })) +} diff --git a/integration_tests/tests/tests.rs b/integration_tests/tests/tests.rs index 93c1f850..14da0cf3 100644 --- a/integration_tests/tests/tests.rs +++ b/integration_tests/tests/tests.rs @@ -32,6 +32,7 @@ const FINALIZE_TRANSACTIONS_DELAY: Duration = Duration::from_mins(2); const RESUBMIT_TRANSACTIONS_DELAY: Duration = Duration::from_mins(3); const DEPOSIT_CONSOLIDATION_DELAY: Duration = Duration::from_mins(10); const POLL_MONITORED_ADDRESSES_DELAY: Duration = Duration::from_mins(1); +const REFRESH_REAL_BALANCE_DELAY: Duration = Duration::from_secs(24 * 60 * 60); /// Deposits funds into the minter via `process_deposit`, consolidates them, /// and finalizes the consolidation so the minter's internal balance is credited. @@ -1289,6 +1290,43 @@ mod metrics_tests { .drop() .await; } + + #[tokio::test] + async fn should_report_balance_discrepancy_after_daily_refresh() { + const REAL_BALANCE: Lamport = 12_345_678; + + let setup = SetupBuilder::new().build().await; + + // Before the first refresh, both gauges should report zero. + let setup = setup + .check_metrics() + .await + .assert_contains_metric_matching(r"minter_balance_discrepancy_lamports 0 \d+") + .assert_contains_metric_matching( + r"minter_balance_discrepancy_last_refresh_timestamp_seconds 0 \d+", + ) + .into(); + + // Trigger the daily refresh and respond to the resulting `getBalance` outcall. + // The tracked balance is still 0, so the recorded discrepancy equals REAL_BALANCE. + setup.advance_time(REFRESH_REAL_BALANCE_DELAY).await; + setup + .execute_http_mocks(MockBuilder::new().get_balance(REAL_BALANCE).build()) + .await; + + setup + .check_metrics() + .await + .assert_contains_metric_matching(&format!( + r"minter_balance_discrepancy_lamports {REAL_BALANCE} \d+" + )) + .assert_contains_metric_matching( + r"minter_balance_discrepancy_last_refresh_timestamp_seconds [1-9]\d* \d+", + ) + .into() + .drop() + .await; + } } mod automated_deposit_flow_tests { From c9fb31e7c8da6f652be6dabbcae6bb1e48b73dc8 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 27 Apr 2026 16:41:27 +0200 Subject: [PATCH 3/4] also test the balance after init/upgrade --- integration_tests/tests/tests.rs | 36 ++++++++++++++++++++++++-------- minter/src/main.rs | 6 +++++- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/integration_tests/tests/tests.rs b/integration_tests/tests/tests.rs index 14da0cf3..e3b5da7f 100644 --- a/integration_tests/tests/tests.rs +++ b/integration_tests/tests/tests.rs @@ -1292,12 +1292,13 @@ mod metrics_tests { } #[tokio::test] - async fn should_report_balance_discrepancy_after_daily_refresh() { - const REAL_BALANCE: Lamport = 12_345_678; + async fn should_report_balance_discrepancy_on_install_and_after_daily_refresh() { + const INITIAL_BALANCE: Lamport = 12_345_678; + const NEXT_BALANCE: Lamport = 99_999_999; let setup = SetupBuilder::new().build().await; - // Before the first refresh, both gauges should report zero. + // Until the install-time refresh completes, both gauges report zero. let setup = setup .check_metrics() .await @@ -1307,22 +1308,39 @@ mod metrics_tests { ) .into(); - // Trigger the daily refresh and respond to the resulting `getBalance` outcall. - // The tracked balance is still 0, so the recorded discrepancy equals REAL_BALANCE. - setup.advance_time(REFRESH_REAL_BALANCE_DELAY).await; + // The minter schedules an immediate refresh on install. Respond to the + // resulting `getBalance` outcall and check the gauges update. setup - .execute_http_mocks(MockBuilder::new().get_balance(REAL_BALANCE).build()) + .execute_http_mocks(MockBuilder::new().get_balance(INITIAL_BALANCE).build()) .await; - setup + let setup = setup .check_metrics() .await .assert_contains_metric_matching(&format!( - r"minter_balance_discrepancy_lamports {REAL_BALANCE} \d+" + r"minter_balance_discrepancy_lamports {INITIAL_BALANCE} \d+" )) .assert_contains_metric_matching( r"minter_balance_discrepancy_last_refresh_timestamp_seconds [1-9]\d* \d+", ) + .into(); + + // Advance time past the daily interval; the periodic refresh runs again. + setup.advance_time(REFRESH_REAL_BALANCE_DELAY).await; + setup + .execute_http_mocks( + MockBuilder::with_start_id(4) + .get_balance(NEXT_BALANCE) + .build(), + ) + .await; + + setup + .check_metrics() + .await + .assert_contains_metric_matching(&format!( + r"minter_balance_discrepancy_lamports {NEXT_BALANCE} \d+" + )) .into() .drop() .await; diff --git a/minter/src/main.rs b/minter/src/main.rs index 8c8b6cf2..48458bef 100644 --- a/minter/src/main.rs +++ b/minter/src/main.rs @@ -352,9 +352,13 @@ fn assert_non_anonymous_account( fn setup_timers() { ic_cdk_timers::set_timer(Duration::from_secs(0), async { - // Initialize the minter's Ed25519 public key + // Initialize the minter's Ed25519 public key, then run the first balance + // refresh so the discrepancy metric is populated without waiting for the + // daily interval. Sequencing them in a single timer avoids racing + // `set_once_minter_public_key`. let runtime = IcCanisterRuntime::new(); let _ = lazy_get_schnorr_master_key(&runtime).await; + refresh_real_balance(runtime).await; }); ic_cdk_timers::set_timer_interval(DEPOSIT_CONSOLIDATION_DELAY, async || { consolidate_deposits(IcCanisterRuntime::new()).await; From b7fa271079dfb009775a3a765e3a20d0b1182a27 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 27 Apr 2026 16:51:39 +0200 Subject: [PATCH 4/4] clippy --- integration_tests/tests/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/tests/tests.rs b/integration_tests/tests/tests.rs index e3b5da7f..8dea9351 100644 --- a/integration_tests/tests/tests.rs +++ b/integration_tests/tests/tests.rs @@ -1317,7 +1317,7 @@ mod metrics_tests { let setup = setup .check_metrics() .await - .assert_contains_metric_matching(&format!( + .assert_contains_metric_matching(format!( r"minter_balance_discrepancy_lamports {INITIAL_BALANCE} \d+" )) .assert_contains_metric_matching( @@ -1338,7 +1338,7 @@ mod metrics_tests { setup .check_metrics() .await - .assert_contains_metric_matching(&format!( + .assert_contains_metric_matching(format!( r"minter_balance_discrepancy_lamports {NEXT_BALANCE} \d+" )) .into()