test(ckETH_Minter): DEFI-2559: Add test to verify minter cannot be stopped while it is scraping blocks

rs/ethereum/cketh/minter/BUILD.bazel

@@ -207,6 +207,7 @@ rust_ic_test_suite(
         "//rs/state_machine_tests",
         "//rs/types/base_types",
         "//rs/types/management_canister_types",
+        "//rs/types/types",
         "@crate_index//:assert_matches",
         "@crate_index//:candid",
         "@crate_index//:ethers-core",

rs/ethereum/cketh/minter/Cargo.toml

@@ -34,6 +34,7 @@ ic-metrics-encoder = "1"
 ic-secp256k1 = { path = "../../../../packages/ic-secp256k1" }
 ic-sha3 = { workspace = true }
 ic-stable-structures = { workspace = true }
+ic-types = { path = "../../../types/types" }
 ic-utils-ensure = { path = "../../../utils/ensure" }
 icrc-cbor = { path = "../../../../packages/icrc-cbor", features = ["u256"] }
 icrc-ledger-client-cdk = { path = "../../../../packages/icrc-ledger-client-cdk" }

rs/ethereum/cketh/minter/tests/cketh.rs

@@ -33,6 +33,7 @@ use ic_cketh_test_utils::{
     MINTER_ADDRESS,
 };
 use ic_ethereum_types::Address;
+use ic_management_canister_types_private::CanisterStatusType;
 use icrc_ledger_types::icrc1::account::Account;
 use icrc_ledger_types::icrc1::transfer::Memo;
 use icrc_ledger_types::icrc3::transactions::{Burn, Mint};
@@ -828,6 +829,132 @@ fn should_scrap_one_block_when_at_boundary_with_last_finalized_block() {
         .expect_rpc_calls(&cketh);
 }
 
+#[test]
+fn should_document_current_behavior_of_being_unstoppable_while_scraping_blocks_has_open_call_context()
+ {
+    // TODO(DEFI-2566): This test documents the current behavior, where the ckETH minter is
+    //  unstoppable while scraping (lots of) logs on a timer. Since log scraping calls are made on
+    //  a loop in the callback handler for log scraping responses, the scraping continues until all
+    //  logs have been scraped. The same call context is reused, and as long as there is an open
+    //  call context, the minter is not stoppable.
+    const UNSCRAPED_BLOCKS: u64 = 5_000;
+    const NUM_BLOCK_RANGES: usize = 10;
+
+    let cketh = CkEthSetup::default();
+    let max_eth_logs_block_range = cketh.as_ref().max_logs_block_range();
+    const MAX_BLOCK: u64 = LAST_SCRAPED_BLOCK_NUMBER_AT_INSTALL + UNSCRAPED_BLOCKS;
+
+    cketh.env.advance_time(SCRAPING_ETH_LOGS_INTERVAL);
+
+    MockJsonRpcProviders::when(JsonRpcMethod::EthGetBlockByNumber)
+        .respond_for_all_with(block_response(MAX_BLOCK))
+        .build()
+        .expect_rpc_calls(&cketh);
+
+    // Only the first few eth_getLogs requests (e.g., 3 out of 10).
+    // This leaves the scraping in progress with open call contexts.
+    let mut from_block = BlockNumber::from(LAST_SCRAPED_BLOCK_NUMBER_AT_INSTALL + 1);
+    let mut to_block = from_block
+        .checked_add(BlockNumber::from(max_eth_logs_block_range))
+        .unwrap();
+
+    const BLOCKS_TO_PROCESS_BEFORE_STOP: usize = 3;
+    for _ in 0..BLOCKS_TO_PROCESS_BEFORE_STOP {
+        MockJsonRpcProviders::when(JsonRpcMethod::EthGetLogs)
+            .with_request_params(json!([{
+                "fromBlock": from_block,
+                "toBlock": to_block,
+                "address": [ETH_HELPER_CONTRACT_ADDRESS],
+                "topics": [cketh.received_eth_event_topic()]
+            }]))
+            .respond_for_all_with(empty_logs())
+            .build()
+            .expect_rpc_calls(&cketh);
+
+        from_block = to_block.checked_increment().unwrap();
+        to_block = from_block
+            .checked_add(BlockNumber::from(max_eth_logs_block_range))
+            .unwrap();
+    }
+
+    // At this point:
+    // - 3 block ranges have been scraped
+    // - The minter has made an HTTP outcall for the 4th block range
+    // - There's an open call context waiting for that HTTP response
+    // Request to stop the minter (without providing responses to pending HTTP outcalls).
+    // The stop will NOT complete because there's an open call context.
+    cketh.try_stop_minter_without_stopping_ongoing_https_outcalls();
+
+    // Verify the minter is in "Stopping" state (not "Stopped")
+    let status = cketh.tick_until_minter_canister_status(CanisterStatusType::Stopping);
+    assert_eq!(
+        status,
+        CanisterStatusType::Stopping,
+        "Expected minter to be in Stopping state due to open call contexts"
+    );
+
+    // Even while in "Stopping" state, when we provide a response to the pending HTTPS call, the
+    // canister does not stop. Instead, the callback continuation runs and the next loop iteration
+    // makes another outcall. The canister remains in "Stopping" state throughout.
+    for i in BLOCKS_TO_PROCESS_BEFORE_STOP..NUM_BLOCK_RANGES {
+        // Before providing response, verify canister is STILL in Stopping state
+        let status_before = cketh.minter_status();
+        assert_eq!(
+            status_before,
+            CanisterStatusType::Stopping,
+            "Block range {}/{}: Canister should be in Stopping state before receiving response",
+            i + 1,
+            NUM_BLOCK_RANGES
+        );
+
+        // Provide response to the pending HTTPS call.
+        MockJsonRpcProviders::when(JsonRpcMethod::EthGetLogs)
+            .with_request_params(json!([{
+                "fromBlock": from_block,
+                "toBlock": to_block,
+                "address": [ETH_HELPER_CONTRACT_ADDRESS],
+                "topics": [cketh.received_eth_event_topic()]
+            }]))
+            .respond_for_all_with(empty_logs())
+            .build()
+            .expect_rpc_calls(&cketh);
+
+        // After processing the response, verify the canister is still in Stopping state.
+        let status_after = cketh.minter_status();
+
+        if i < NUM_BLOCK_RANGES - 1 {
+            assert_eq!(
+                status_after,
+                CanisterStatusType::Stopping,
+                "Block range {}/{}: Canister should still be in Stopping state after receiving \
+                 response (it made a new HTTP call for the next block range!)",
+                i + 1,
+                NUM_BLOCK_RANGES
+            );
+        } else {
+            // Last block range - canister might transition to Stopped
+            println!(
+                "  Block range {}/{}: Final response received",
+                i + 1,
+                NUM_BLOCK_RANGES
+            );
+        }
+
+        from_block = to_block.checked_increment().unwrap();
+        to_block = from_block
+            .checked_add(BlockNumber::from(max_eth_logs_block_range))
+            .unwrap();
+    }
+
+    // After all scraping is complete, the canister should finally be Stopped.
+    let status = cketh.tick_until_minter_canister_status(CanisterStatusType::Stopped);
+    assert_eq!(
+        status,
+        CanisterStatusType::Stopped,
+        "Expected minter to be Stopped after all call contexts closed"
+    );
+}
+
 #[test]
 fn should_panic_when_last_finalized_block_in_the_past() {
     let cketh = CkEthSetup::default();

rs/ethereum/cketh/test_utils/src/lib.rs

@@ -26,6 +26,7 @@ use ic_state_machine_tests::{
 };
 use ic_test_utilities_load_wasm::load_wasm;
 use ic_types::Cycles;
+use ic_types::ingress::{IngressState, IngressStatus};
 use icrc_ledger_types::icrc1::account::Account;
 use icrc_ledger_types::icrc2::approve::{ApproveArgs, ApproveError};
 use num_traits::cast::ToPrimitive;
@@ -495,6 +496,42 @@ impl CkEthSetup {
         self.start_minter();
     }
 
+    /// Try to stop the minter without first stopping the ongoing HTTPS outcalls. Assert that the
+    /// `IngressStatus` is `Processing`.
+    pub fn try_stop_minter_without_stopping_ongoing_https_outcalls(&self) {
+        const MAX_TICKS: u64 = 10;
+        let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id);
+        let mut ingress_status = self.env.ingress_status(&stop_msg_id);
+        for _ in 0..MAX_TICKS {
+            if let IngressStatus::Known { state, .. } = &ingress_status
+                && state == &IngressState::Processing
+            {
+                return;
+            }
+            self.env.tick();
+            ingress_status = self.env.ingress_status(&stop_msg_id);
+        }
+        panic!(
+            "expected minter ingress status to be `Processing`, ended up with {:?}",
+            ingress_status
+        );
+    }
+
+    pub fn tick_until_minter_canister_status(
+        &self,
+        expected_canister_status: CanisterStatusType,
+    ) -> CanisterStatusType {
+        const MAX_TICKS: u64 = 10;
+        let mut status = self.minter_status();
+        for _ in 0..MAX_TICKS {
+            if status == expected_canister_status {
+                break;
+            }
+            status = self.minter_status();
+        }
+        status
+    }
+
     pub fn stop_minter(&self) {
         let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id);
         self.stop_ongoing_https_outcalls();