feat: metrics for real minter balance

integration_tests/src/fixtures.rs

@@ -213,6 +213,11 @@ impl MockBuilder {
             get_signatures_for_address_response(signatures),
         )
     }
+
+    /// Mock for `getBalance` returning the given lamport balance.
+    pub fn get_balance(self, balance: Lamport) -> Self {
+        self.expect(get_balance_request(), get_balance_response(balance))
+    }
 }
 
 // ── JSON-RPC request matchers and response builders ─────────────────────────
@@ -387,3 +392,18 @@ fn get_signatures_for_address_response(signatures: Vec<serde_json::Value>) -> Js
         "id": 1
     }))
 }
+
+fn get_balance_request() -> JsonRpcRequestMatcher {
+    JsonRpcRequestMatcher::with_method("getBalance")
+}
+
+fn get_balance_response(balance: Lamport) -> JsonRpcResponse {
+    JsonRpcResponse::from(json!({
+        "jsonrpc": "2.0",
+        "result": {
+            "context": { "slot": 350_000_000_u64, "apiVersion": "2.1.9" },
+            "value": balance,
+        },
+        "id": 1
+    }))
+}

integration_tests/tests/tests.rs

@@ -32,6 +32,7 @@ const FINALIZE_TRANSACTIONS_DELAY: Duration = Duration::from_mins(2);
 const RESUBMIT_TRANSACTIONS_DELAY: Duration = Duration::from_mins(3);
 const DEPOSIT_CONSOLIDATION_DELAY: Duration = Duration::from_mins(10);
 const POLL_MONITORED_ADDRESSES_DELAY: Duration = Duration::from_mins(1);
+const REFRESH_REAL_BALANCE_DELAY: Duration = Duration::from_secs(24 * 60 * 60);
 
 /// Deposits funds into the minter via `process_deposit`, consolidates them,
 /// and finalizes the consolidation so the minter's internal balance is credited.
@@ -1289,6 +1290,61 @@ mod metrics_tests {
             .drop()
             .await;
     }
+
+    #[tokio::test]
+    async fn should_report_balance_discrepancy_on_install_and_after_daily_refresh() {
+        const INITIAL_BALANCE: Lamport = 12_345_678;
+        const NEXT_BALANCE: Lamport = 99_999_999;
+
+        let setup = SetupBuilder::new().build().await;
+
+        // Until the install-time refresh completes, both gauges report zero.
+        let setup = setup
+            .check_metrics()
+            .await
+            .assert_contains_metric_matching(r"minter_balance_discrepancy_lamports 0 \d+")
+            .assert_contains_metric_matching(
+                r"minter_balance_discrepancy_last_refresh_timestamp_seconds 0 \d+",
+            )
+            .into();
+
+        // The minter schedules an immediate refresh on install. Respond to the
+        // resulting `getBalance` outcall and check the gauges update.
+        setup
+            .execute_http_mocks(MockBuilder::new().get_balance(INITIAL_BALANCE).build())
+            .await;
+
+        let setup = setup
+            .check_metrics()
+            .await
+            .assert_contains_metric_matching(format!(
+                r"minter_balance_discrepancy_lamports {INITIAL_BALANCE} \d+"
+            ))
+            .assert_contains_metric_matching(
+                r"minter_balance_discrepancy_last_refresh_timestamp_seconds [1-9]\d* \d+",
+            )
+            .into();
+
+        // Advance time past the daily interval; the periodic refresh runs again.
+        setup.advance_time(REFRESH_REAL_BALANCE_DELAY).await;
+        setup
+            .execute_http_mocks(
+                MockBuilder::with_start_id(4)
+                    .get_balance(NEXT_BALANCE)
+                    .build(),
+            )
+            .await;
+
+        setup
+            .check_metrics()
+            .await
+            .assert_contains_metric_matching(format!(
+                r"minter_balance_discrepancy_lamports {NEXT_BALANCE} \d+"
+            ))
+            .into()
+            .drop()
+            .await;
+    }
 }
 
 mod automated_deposit_flow_tests {

minter/src/balance_check.rs

@@ -0,0 +1,47 @@
+use crate::{
+    address::{lazy_get_schnorr_master_key, minter_address},
+    guard::TimerGuard,
+    rpc::get_balance,
+    runtime::CanisterRuntime,
+    state::{TaskType, mutate_state},
+};
+use canlog::log;
+use cksol_types_internal::log::Priority;
+use std::time::Duration;
+
+pub const REFRESH_REAL_BALANCE_DELAY: Duration = Duration::from_secs(24 * 60 * 60);
+
+/// Refresh the cached on-chain balance of the minter's main account.
+/// Each call makes an RPC request to the Solana network, so this runs at most
+/// once per day (see [`REFRESH_REAL_BALANCE_DELAY`]).
+pub async fn refresh_real_balance<R: CanisterRuntime>(runtime: R) {
+    let _guard = match TimerGuard::new(TaskType::RefreshRealBalance) {
+        Ok(guard) => guard,
+        Err(_) => return,
+    };
+
+    let master_key = lazy_get_schnorr_master_key(&runtime).await;
+    let address = minter_address(&master_key, &runtime);
+
+    match get_balance(&runtime, address).await {
+        Ok(balance) => {
+            let observed_at = runtime.time();
+            let diff = mutate_state(|s| {
+                s.record_balance_discrepancy(balance, observed_at);
+                s.last_balance_discrepancy()
+                    .expect("BUG: just-recorded discrepancy must be present")
+                    .diff_lamports
+            });
+            log!(
+                Priority::Info,
+                "Refreshed real balance for minter {address}: {balance} lamports (discrepancy vs. tracked: {diff} lamports)"
+            );
+        }
+        Err(e) => {
+            log!(
+                Priority::Info,
+                "Failed to refresh real balance for minter {address}: {e}"
+            );
+        }
+    }
+}

minter/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod address;
+pub mod balance_check;
 pub mod consolidate;
 mod constants;
 mod cycles;

minter/src/main.rs

@@ -2,6 +2,7 @@ use candid::Principal;
 use canlog::{Log, Sort};
 use cksol_minter::{
     address::lazy_get_schnorr_master_key,
+    balance_check::{REFRESH_REAL_BALANCE_DELAY, refresh_real_balance},
     consolidate::{DEPOSIT_CONSOLIDATION_DELAY, consolidate_deposits},
     deposit::automatic::{POLL_MONITORED_ADDRESSES_DELAY, poll_monitored_addresses},
     monitor::{
@@ -351,9 +352,13 @@ fn assert_non_anonymous_account(
 
 fn setup_timers() {
     ic_cdk_timers::set_timer(Duration::from_secs(0), async {
-        // Initialize the minter's Ed25519 public key
+        // Initialize the minter's Ed25519 public key, then run the first balance
+        // refresh so the discrepancy metric is populated without waiting for the
+        // daily interval. Sequencing them in a single timer avoids racing
+        // `set_once_minter_public_key`.
         let runtime = IcCanisterRuntime::new();
         let _ = lazy_get_schnorr_master_key(&runtime).await;
+        refresh_real_balance(runtime).await;
     });
     ic_cdk_timers::set_timer_interval(DEPOSIT_CONSOLIDATION_DELAY, async || {
         consolidate_deposits(IcCanisterRuntime::new()).await;
@@ -370,6 +375,9 @@ fn setup_timers() {
     ic_cdk_timers::set_timer_interval(POLL_MONITORED_ADDRESSES_DELAY, async || {
         poll_monitored_addresses(IcCanisterRuntime::new()).await;
     });
+    ic_cdk_timers::set_timer_interval(REFRESH_REAL_BALANCE_DELAY, async || {
+        refresh_real_balance(IcCanisterRuntime::new()).await;
+    });
 }
 
 fn main() {}

minter/src/metrics.rs

@@ -92,6 +92,23 @@ pub fn encode_metrics(w: &mut MetricsEncoder<Vec<u8>>, s: &State) -> std::io::Re
         s.balance().metric_value(),
         "Minter balance in Lamports.",
     )?;
+    let (discrepancy_lamports, discrepancy_refresh_seconds) = match s.last_balance_discrepancy() {
+        Some(observation) => (
+            observation.diff_lamports as f64,
+            (observation.observed_at / 1_000_000_000) as f64,
+        ),
+        None => (0.0, 0.0),
+    };
+    w.encode_gauge(
+        "minter_balance_discrepancy_lamports",
+        discrepancy_lamports,
+        "Difference (real on-chain balance minus tracked balance) of the minter's main account at the time of the last successful refresh, in lamports. Both sides are sampled together; refreshed once per day. Reports 0 until the first refresh succeeds.",
+    )?;
+    w.encode_gauge(
+        "minter_balance_discrepancy_last_refresh_timestamp_seconds",
+        discrepancy_refresh_seconds,
+        "Unix timestamp (seconds) of the last successful computation of `minter_balance_discrepancy_lamports`. Reports 0 until the first refresh succeeds.",
+    )?;
     w.encode_gauge(
         "post_upgrade_instructions_consumed",
         storage::with_unstable_metrics(|m| m.post_upgrade_instructions_consumed).metric_value(),

minter/src/rpc/mod.rs

@@ -7,9 +7,10 @@ use cksol_types::ProcessDepositError;
 use derive_more::From;
 use ic_canister_runtime::IcError;
 use sol_rpc_types::{
-    CommitmentLevel, ConfirmedTransactionStatusWithSignature, GetSignaturesForAddressParams,
-    GetTransactionEncoding, MultiRpcResult, RpcError, Slot,
+    CommitmentLevel, ConfirmedTransactionStatusWithSignature, GetBalanceParams,
+    GetSignaturesForAddressParams, GetTransactionEncoding, Lamport, MultiRpcResult, RpcError, Slot,
 };
+use solana_address::Address;
 use solana_hash::Hash;
 use solana_signature::Signature;
 use solana_transaction::Transaction;
@@ -162,3 +163,33 @@ pub enum GetSignaturesForAddressError {
     #[error("Inconsistent RPC results for getSignaturesForAddress")]
     InconsistentRpcResults,
 }
+
+pub async fn get_balance<R: CanisterRuntime>(
+    runtime: &R,
+    address: Address,
+) -> Result<Lamport, GetBalanceError> {
+    let client = read_state(|state| state.sol_rpc_client(runtime.inter_canister_call_runtime()));
+    let result = client
+        .get_balance(GetBalanceParams {
+            pubkey: address.into(),
+            commitment: Some(CommitmentLevel::Finalized),
+            min_context_slot: None,
+        })
+        .try_send()
+        .await;
+    match result? {
+        MultiRpcResult::Consistent(Ok(balance)) => Ok(balance),
+        MultiRpcResult::Consistent(Err(e)) => Err(GetBalanceError::RpcError(e)),
+        MultiRpcResult::Inconsistent(_) => Err(GetBalanceError::InconsistentRpcResults),
+    }
+}
+
+#[derive(Debug, PartialEq, Error)]
+pub enum GetBalanceError {
+    #[error("Error while calling SOL RPC canister: {0}")]
+    IcError(#[from] IcError),
+    #[error("RPC error while fetching balance: {0}")]
+    RpcError(RpcError),
+    #[error("Inconsistent RPC results for getBalance")]
+    InconsistentRpcResults,
+}

minter/src/state/mod.rs

@@ -109,6 +109,22 @@ pub struct State {
     consolidation_transactions: InsertionOrderedMap<Signature, ConsolidationTransaction>,
     active_tasks: BTreeSet<TaskType>,
     balance: Lamport,
+    /// Last observed difference (`real_balance - tracked_balance`) between the
+    /// real on-chain balance of the minter's main account and the value
+    /// tracked by the minter's state, together with the timestamp
+    /// (nanoseconds since the Unix epoch) at which it was computed.
+    ///
+    /// Both sides of the difference are sampled at the same time so that the
+    /// stored value remains meaningful even though `tracked_balance` keeps
+    /// changing afterwards. Refreshed once per day by a timer; `None` until
+    /// the first refresh succeeds.
+    last_balance_discrepancy: Option<BalanceDiscrepancyObservation>,
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct BalanceDiscrepancyObservation {
+    pub diff_lamports: i128,
+    pub observed_at: u64,
 }
 
 impl State {
@@ -246,6 +262,20 @@ impl State {
         self.balance
     }
 
+    pub fn last_balance_discrepancy(&self) -> Option<BalanceDiscrepancyObservation> {
+        self.last_balance_discrepancy
+    }
+
+    /// Records the difference between `real_balance` (just observed on-chain)
+    /// and the tracked balance, sampled at `observed_at`.
+    pub fn record_balance_discrepancy(&mut self, real_balance: Lamport, observed_at: u64) {
+        let diff_lamports = real_balance as i128 - self.balance as i128;
+        self.last_balance_discrepancy = Some(BalanceDiscrepancyObservation {
+            diff_lamports,
+            observed_at,
+        });
+    }
+
     pub fn monitored_accounts(&self) -> &InsertionOrderedSet<Account> {
         &self.monitored_accounts
     }
@@ -796,6 +826,7 @@ impl TryFrom<InitArgs> for State {
             consolidation_transactions: InsertionOrderedMap::new(),
             active_tasks: BTreeSet::new(),
             balance: 0,
+            last_balance_discrepancy: None,
         };
         state.validate()?;
         Ok(state)
@@ -843,6 +874,7 @@ pub enum TaskType {
     ResubmitTransactions,
     WithdrawalProcessing,
     PollMonitoredAddresses,
+    RefreshRealBalance,
 }
 
 /// Details about a consolidation transaction, capturing the individual

minter/src/state/tests.rs

@@ -246,6 +246,7 @@ mod state_from_init_args {
                 consolidation_transactions: InsertionOrderedMap::new(),
                 active_tasks: BTreeSet::new(),
                 balance: 0,
+                last_balance_discrepancy: None,
             }
         );
     }