diff --git a/Cargo.lock b/Cargo.lock index f4c2af570ee9..2a18e786dd33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7772,6 +7772,7 @@ dependencies = [ "ic-sha3 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "ic-stable-structures 0.6.8", "ic-state-machine-tests", + "ic-types", "ic-utils-ensure", "icrc-cbor", "icrc-ledger-client-cdk", diff --git a/rs/ethereum/cketh/minter/BUILD.bazel b/rs/ethereum/cketh/minter/BUILD.bazel index 818584c7c507..f9ec155a6013 100644 --- a/rs/ethereum/cketh/minter/BUILD.bazel +++ b/rs/ethereum/cketh/minter/BUILD.bazel @@ -207,6 +207,7 @@ rust_ic_test_suite( "//rs/state_machine_tests", "//rs/types/base_types", "//rs/types/management_canister_types", + "//rs/types/types", "@crate_index//:assert_matches", "@crate_index//:candid", "@crate_index//:ethers-core", diff --git a/rs/ethereum/cketh/minter/Cargo.toml b/rs/ethereum/cketh/minter/Cargo.toml index 651f53519680..825a09200e1b 100644 --- a/rs/ethereum/cketh/minter/Cargo.toml +++ b/rs/ethereum/cketh/minter/Cargo.toml @@ -34,6 +34,7 @@ ic-metrics-encoder = "1" ic-secp256k1 = { path = "../../../../packages/ic-secp256k1" } ic-sha3 = { workspace = true } ic-stable-structures = { workspace = true } +ic-types = { path = "../../../types/types" } ic-utils-ensure = { path = "../../../utils/ensure" } icrc-cbor = { path = "../../../../packages/icrc-cbor", features = ["u256"] } icrc-ledger-client-cdk = { path = "../../../../packages/icrc-ledger-client-cdk" } diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index 540ac0b673aa..b19ea1e99744 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -33,6 +33,7 @@ use ic_cketh_test_utils::{ MINTER_ADDRESS, }; use ic_ethereum_types::Address; +use ic_management_canister_types_private::CanisterStatusType; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc1::transfer::Memo; use icrc_ledger_types::icrc3::transactions::{Burn, Mint}; @@ -828,6 +829,132 @@ fn should_scrap_one_block_when_at_boundary_with_last_finalized_block() { .expect_rpc_calls(&cketh); } +#[test] +fn should_document_current_behavior_of_being_unstoppable_while_scraping_blocks_has_open_call_context() + { + // TODO(DEFI-2566): This test documents the current behavior, where the ckETH minter is + // unstoppable while scraping (lots of) logs on a timer. Since log scraping calls are made on + // a loop in the callback handler for log scraping responses, the scraping continues until all + // logs have been scraped. The same call context is reused, and as long as there is an open + // call context, the minter is not stoppable. + const UNSCRAPED_BLOCKS: u64 = 5_000; + const NUM_BLOCK_RANGES: usize = 10; + + let cketh = CkEthSetup::default(); + let max_eth_logs_block_range = cketh.as_ref().max_logs_block_range(); + const MAX_BLOCK: u64 = LAST_SCRAPED_BLOCK_NUMBER_AT_INSTALL + UNSCRAPED_BLOCKS; + + cketh.env.advance_time(SCRAPING_ETH_LOGS_INTERVAL); + + MockJsonRpcProviders::when(JsonRpcMethod::EthGetBlockByNumber) + .respond_for_all_with(block_response(MAX_BLOCK)) + .build() + .expect_rpc_calls(&cketh); + + // Only the first few eth_getLogs requests (e.g., 3 out of 10). + // This leaves the scraping in progress with open call contexts. + let mut from_block = BlockNumber::from(LAST_SCRAPED_BLOCK_NUMBER_AT_INSTALL + 1); + let mut to_block = from_block + .checked_add(BlockNumber::from(max_eth_logs_block_range)) + .unwrap(); + + const BLOCKS_TO_PROCESS_BEFORE_STOP: usize = 3; + for _ in 0..BLOCKS_TO_PROCESS_BEFORE_STOP { + MockJsonRpcProviders::when(JsonRpcMethod::EthGetLogs) + .with_request_params(json!([{ + "fromBlock": from_block, + "toBlock": to_block, + "address": [ETH_HELPER_CONTRACT_ADDRESS], + "topics": [cketh.received_eth_event_topic()] + }])) + .respond_for_all_with(empty_logs()) + .build() + .expect_rpc_calls(&cketh); + + from_block = to_block.checked_increment().unwrap(); + to_block = from_block + .checked_add(BlockNumber::from(max_eth_logs_block_range)) + .unwrap(); + } + + // At this point: + // - 3 block ranges have been scraped + // - The minter has made an HTTP outcall for the 4th block range + // - There's an open call context waiting for that HTTP response + // Request to stop the minter (without providing responses to pending HTTP outcalls). + // The stop will NOT complete because there's an open call context. + cketh.try_stop_minter_without_stopping_ongoing_https_outcalls(); + + // Verify the minter is in "Stopping" state (not "Stopped") + let status = cketh.tick_until_minter_canister_status(CanisterStatusType::Stopping); + assert_eq!( + status, + CanisterStatusType::Stopping, + "Expected minter to be in Stopping state due to open call contexts" + ); + + // Even while in "Stopping" state, when we provide a response to the pending HTTPS call, the + // canister does not stop. Instead, the callback continuation runs and the next loop iteration + // makes another outcall. The canister remains in "Stopping" state throughout. + for i in BLOCKS_TO_PROCESS_BEFORE_STOP..NUM_BLOCK_RANGES { + // Before providing response, verify canister is STILL in Stopping state + let status_before = cketh.minter_status(); + assert_eq!( + status_before, + CanisterStatusType::Stopping, + "Block range {}/{}: Canister should be in Stopping state before receiving response", + i + 1, + NUM_BLOCK_RANGES + ); + + // Provide response to the pending HTTPS call. + MockJsonRpcProviders::when(JsonRpcMethod::EthGetLogs) + .with_request_params(json!([{ + "fromBlock": from_block, + "toBlock": to_block, + "address": [ETH_HELPER_CONTRACT_ADDRESS], + "topics": [cketh.received_eth_event_topic()] + }])) + .respond_for_all_with(empty_logs()) + .build() + .expect_rpc_calls(&cketh); + + // After processing the response, verify the canister is still in Stopping state. + let status_after = cketh.minter_status(); + + if i < NUM_BLOCK_RANGES - 1 { + assert_eq!( + status_after, + CanisterStatusType::Stopping, + "Block range {}/{}: Canister should still be in Stopping state after receiving \ + response (it made a new HTTP call for the next block range!)", + i + 1, + NUM_BLOCK_RANGES + ); + } else { + // Last block range - canister might transition to Stopped + println!( + " Block range {}/{}: Final response received", + i + 1, + NUM_BLOCK_RANGES + ); + } + + from_block = to_block.checked_increment().unwrap(); + to_block = from_block + .checked_add(BlockNumber::from(max_eth_logs_block_range)) + .unwrap(); + } + + // After all scraping is complete, the canister should finally be Stopped. + let status = cketh.tick_until_minter_canister_status(CanisterStatusType::Stopped); + assert_eq!( + status, + CanisterStatusType::Stopped, + "Expected minter to be Stopped after all call contexts closed" + ); +} + #[test] fn should_panic_when_last_finalized_block_in_the_past() { let cketh = CkEthSetup::default(); diff --git a/rs/ethereum/cketh/test_utils/src/lib.rs b/rs/ethereum/cketh/test_utils/src/lib.rs index bce61a8ff29e..42e0d05731d5 100644 --- a/rs/ethereum/cketh/test_utils/src/lib.rs +++ b/rs/ethereum/cketh/test_utils/src/lib.rs @@ -26,6 +26,7 @@ use ic_state_machine_tests::{ }; use ic_test_utilities_load_wasm::load_wasm; use ic_types::Cycles; +use ic_types::ingress::{IngressState, IngressStatus}; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc2::approve::{ApproveArgs, ApproveError}; use num_traits::cast::ToPrimitive; @@ -495,6 +496,42 @@ impl CkEthSetup { self.start_minter(); } + /// Try to stop the minter without first stopping the ongoing HTTPS outcalls. Assert that the + /// `IngressStatus` is `Processing`. + pub fn try_stop_minter_without_stopping_ongoing_https_outcalls(&self) { + const MAX_TICKS: u64 = 10; + let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id); + let mut ingress_status = self.env.ingress_status(&stop_msg_id); + for _ in 0..MAX_TICKS { + if let IngressStatus::Known { state, .. } = &ingress_status + && state == &IngressState::Processing + { + return; + } + self.env.tick(); + ingress_status = self.env.ingress_status(&stop_msg_id); + } + panic!( + "expected minter ingress status to be `Processing`, ended up with {:?}", + ingress_status + ); + } + + pub fn tick_until_minter_canister_status( + &self, + expected_canister_status: CanisterStatusType, + ) -> CanisterStatusType { + const MAX_TICKS: u64 = 10; + let mut status = self.minter_status(); + for _ in 0..MAX_TICKS { + if status == expected_canister_status { + break; + } + status = self.minter_status(); + } + status + } + pub fn stop_minter(&self) { let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id); self.stop_ongoing_https_outcalls();