From 236b77d1f8d95130346b1c5ab603cc099325ba02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Fri, 5 Dec 2025 14:29:25 +0100 Subject: [PATCH 1/8] Upgrade minter to the same wasm to verify it can be stopped --- rs/ethereum/cketh/minter/tests/cketh.rs | 2 ++ rs/ethereum/cketh/test_utils/src/lib.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index 540ac0b673aa..51a120aadeaa 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -826,6 +826,8 @@ fn should_scrap_one_block_when_at_boundary_with_last_finalized_block() { .respond_for_all_with(empty_logs()) .build() .expect_rpc_calls(&cketh); + + cketh.upgrade_minter_with_same_wasm_and_without_upgrade_args(); } #[test] diff --git a/rs/ethereum/cketh/test_utils/src/lib.rs b/rs/ethereum/cketh/test_utils/src/lib.rs index bce61a8ff29e..9b3b2885123c 100644 --- a/rs/ethereum/cketh/test_utils/src/lib.rs +++ b/rs/ethereum/cketh/test_utils/src/lib.rs @@ -533,6 +533,11 @@ impl CkEthSetup { .status() } + pub fn upgrade_minter_with_same_wasm_and_without_upgrade_args(self) -> Self { + self.upgrade_minter(UpgradeArg::default()); + self + } + pub fn upgrade_minter_to_add_orchestrator_id(self, orchestrator_id: Principal) -> Self { self.upgrade_minter(UpgradeArg { ledger_suite_orchestrator_id: Some(orchestrator_id), From c36a6d3e67b733822adc8ef00700207b85edce12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Mon, 8 Dec 2025 11:30:30 +0100 Subject: [PATCH 2/8] Add test that shows that minter cannot be stopped while call context is open --- Cargo.lock | 1 + rs/ethereum/cketh/minter/BUILD.bazel | 1 + rs/ethereum/cketh/minter/Cargo.toml | 1 + rs/ethereum/cketh/minter/tests/cketh.rs | 145 +++++++++++++++++++++++- rs/ethereum/cketh/test_utils/src/lib.rs | 10 ++ 5 files changed, 157 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f4c2af570ee9..2a18e786dd33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7772,6 +7772,7 @@ dependencies = [ "ic-sha3 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "ic-stable-structures 0.6.8", "ic-state-machine-tests", + "ic-types", "ic-utils-ensure", "icrc-cbor", "icrc-ledger-client-cdk", diff --git a/rs/ethereum/cketh/minter/BUILD.bazel b/rs/ethereum/cketh/minter/BUILD.bazel index 818584c7c507..f9ec155a6013 100644 --- a/rs/ethereum/cketh/minter/BUILD.bazel +++ b/rs/ethereum/cketh/minter/BUILD.bazel @@ -207,6 +207,7 @@ rust_ic_test_suite( "//rs/state_machine_tests", "//rs/types/base_types", "//rs/types/management_canister_types", + "//rs/types/types", "@crate_index//:assert_matches", "@crate_index//:candid", "@crate_index//:ethers-core", diff --git a/rs/ethereum/cketh/minter/Cargo.toml b/rs/ethereum/cketh/minter/Cargo.toml index 651f53519680..825a09200e1b 100644 --- a/rs/ethereum/cketh/minter/Cargo.toml +++ b/rs/ethereum/cketh/minter/Cargo.toml @@ -34,6 +34,7 @@ ic-metrics-encoder = "1" ic-secp256k1 = { path = "../../../../packages/ic-secp256k1" } ic-sha3 = { workspace = true } ic-stable-structures = { workspace = true } +ic-types = { path = "../../../types/types" } ic-utils-ensure = { path = "../../../utils/ensure" } icrc-cbor = { path = "../../../../packages/icrc-cbor", features = ["u256"] } icrc-ledger-client-cdk = { path = "../../../../packages/icrc-ledger-client-cdk" } diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index 51a120aadeaa..c7413dd97df6 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -33,6 +33,8 @@ use ic_cketh_test_utils::{ MINTER_ADDRESS, }; use ic_ethereum_types::Address; +use ic_management_canister_types_private::CanisterStatusType; +use ic_types::ingress::{IngressState, IngressStatus}; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc1::transfer::Memo; use icrc_ledger_types::icrc3::transactions::{Burn, Mint}; @@ -826,8 +828,149 @@ fn should_scrap_one_block_when_at_boundary_with_last_finalized_block() { .respond_for_all_with(empty_logs()) .build() .expect_rpc_calls(&cketh); +} + +#[test] +fn should_be_unstoppable_while_scraping_blocks_has_open_call_context() { + const UNSCRAPED_BLOCKS: u64 = 5_000; + const NUM_BLOCK_RANGES: usize = 10; + + let cketh = CkEthSetup::default(); + let max_eth_logs_block_range = cketh.as_ref().max_logs_block_range(); + const MAX_BLOCK: u64 = LAST_SCRAPED_BLOCK_NUMBER_AT_INSTALL + UNSCRAPED_BLOCKS; + + cketh.env.advance_time(SCRAPING_ETH_LOGS_INTERVAL); + + MockJsonRpcProviders::when(JsonRpcMethod::EthGetBlockByNumber) + .respond_for_all_with(block_response(MAX_BLOCK)) + .build() + .expect_rpc_calls(&cketh); + + // Only the first few eth_getLogs requests (e.g., 3 out of 10). + // This leaves the scraping in progress with open call contexts. + let mut from_block = BlockNumber::from(LAST_SCRAPED_BLOCK_NUMBER_AT_INSTALL + 1); + let mut to_block = from_block + .checked_add(BlockNumber::from(max_eth_logs_block_range)) + .unwrap(); + + const BLOCKS_TO_PROCESS_BEFORE_STOP: usize = 3; + for _ in 0..BLOCKS_TO_PROCESS_BEFORE_STOP { + MockJsonRpcProviders::when(JsonRpcMethod::EthGetLogs) + .with_request_params(json!([{ + "fromBlock": from_block, + "toBlock": to_block, + "address": [ETH_HELPER_CONTRACT_ADDRESS], + "topics": [cketh.received_eth_event_topic()] + }])) + .respond_for_all_with(empty_logs()) + .build() + .expect_rpc_calls(&cketh); + + from_block = to_block.checked_increment().unwrap(); + to_block = from_block + .checked_add(BlockNumber::from(max_eth_logs_block_range)) + .unwrap(); + } - cketh.upgrade_minter_with_same_wasm_and_without_upgrade_args(); + // At this point: + // - 3 block ranges have been scraped + // - The minter has made an HTTP outcall for the 4th block range + // - There's an open call context waiting for that HTTP response + // Tick once to ensure any pending requests are visible + cketh.env.tick(); + + // Request to stop the minter (without providing responses to pending HTTP outcalls). + // The stop will NOT complete because there's an open call context. + let stop_status = cketh.try_stop_minter_without_stopping_ongoing_https_outcalls(); + + // Verify the stop request is still Processing (not Completed) + match &stop_status { + IngressStatus::Known { state, .. } => { + assert!( + matches!(state, IngressState::Processing), + "Expected stop to still be Processing due to open call contexts, but got: {:?}", + state + ); + } + other => panic!( + "Expected IngressStatus::Known with Processing state, got: {:?}", + other + ), + } + + // Verify the minter is in "Stopping" state (not "Stopped") + let status = cketh.minter_status(); + assert_eq!( + status, + CanisterStatusType::Stopping, + "Expected minter to be in Stopping state due to open call contexts" + ); + + // Even while in "Stopping" state, when we provide a response to the pending HTTPS call, the + // canister does not stop. Instead, the callback continuation runs and the next loop iteration + // makes another outcall. The canister remains in "Stopping" state throughout. + for i in BLOCKS_TO_PROCESS_BEFORE_STOP..NUM_BLOCK_RANGES { + // Before providing response, verify canister is STILL in Stopping state + let status_before = cketh.minter_status(); + assert_eq!( + status_before, + CanisterStatusType::Stopping, + "Block range {}/{}: Canister should be in Stopping state before receiving response", + i + 1, + NUM_BLOCK_RANGES + ); + + // Provide response to the pending HTTPS call. + MockJsonRpcProviders::when(JsonRpcMethod::EthGetLogs) + .with_request_params(json!([{ + "fromBlock": from_block, + "toBlock": to_block, + "address": [ETH_HELPER_CONTRACT_ADDRESS], + "topics": [cketh.received_eth_event_topic()] + }])) + .respond_for_all_with(empty_logs()) + .build() + .expect_rpc_calls(&cketh); + + // After processing the response, verify the canister is still in Stopping state. + let status_after = cketh.minter_status(); + + if i < NUM_BLOCK_RANGES - 1 { + assert_eq!( + status_after, + CanisterStatusType::Stopping, + "Block range {}/{}: Canister should still be in Stopping state after receiving \ + response (it made a new HTTP call for the next block range!)", + i + 1, + NUM_BLOCK_RANGES + ); + } else { + // Last block range - canister might transition to Stopped + println!( + " Block range {}/{}: Final response received", + i + 1, + NUM_BLOCK_RANGES + ); + } + + from_block = to_block.checked_increment().unwrap(); + to_block = from_block + .checked_add(BlockNumber::from(max_eth_logs_block_range)) + .unwrap(); + } + + // After all scraping is complete, give a few ticks for the canister to stop. + for _ in 0..10 { + cketh.env.tick(); + } + + // Now the canister should finally be Stopped + let final_status = cketh.minter_status(); + assert_eq!( + final_status, + CanisterStatusType::Stopped, + "Expected minter to be Stopped after all call contexts closed" + ); } #[test] diff --git a/rs/ethereum/cketh/test_utils/src/lib.rs b/rs/ethereum/cketh/test_utils/src/lib.rs index 9b3b2885123c..55d2644668ea 100644 --- a/rs/ethereum/cketh/test_utils/src/lib.rs +++ b/rs/ethereum/cketh/test_utils/src/lib.rs @@ -26,6 +26,7 @@ use ic_state_machine_tests::{ }; use ic_test_utilities_load_wasm::load_wasm; use ic_types::Cycles; +use ic_types::ingress::IngressStatus; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc2::approve::{ApproveArgs, ApproveError}; use num_traits::cast::ToPrimitive; @@ -495,6 +496,15 @@ impl CkEthSetup { self.start_minter(); } + pub fn try_stop_minter_without_stopping_ongoing_https_outcalls(&self) -> IngressStatus { + const MAX_TICKS: u64 = 100; + let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id); + for _ in 0..MAX_TICKS { + self.env.tick(); + } + self.env.ingress_status(&stop_msg_id) + } + pub fn stop_minter(&self) { let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id); self.stop_ongoing_https_outcalls(); From 773ec613b559edf67b926ae610aa3f61a9c51081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Mon, 15 Dec 2025 09:12:02 +0100 Subject: [PATCH 3/8] Remove unused helper --- rs/ethereum/cketh/test_utils/src/lib.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rs/ethereum/cketh/test_utils/src/lib.rs b/rs/ethereum/cketh/test_utils/src/lib.rs index 55d2644668ea..d6706cf7ff7c 100644 --- a/rs/ethereum/cketh/test_utils/src/lib.rs +++ b/rs/ethereum/cketh/test_utils/src/lib.rs @@ -543,11 +543,6 @@ impl CkEthSetup { .status() } - pub fn upgrade_minter_with_same_wasm_and_without_upgrade_args(self) -> Self { - self.upgrade_minter(UpgradeArg::default()); - self - } - pub fn upgrade_minter_to_add_orchestrator_id(self, orchestrator_id: Principal) -> Self { self.upgrade_minter(UpgradeArg { ledger_suite_orchestrator_id: Some(orchestrator_id), From f001972ffb2a513f62615e0a6b11cdf2306a8cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Mon, 15 Dec 2025 09:15:01 +0100 Subject: [PATCH 4/8] Add TODO comment with pointer to ticket for adjusting the current behavior --- rs/ethereum/cketh/minter/tests/cketh.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index c7413dd97df6..e54279164573 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -832,6 +832,11 @@ fn should_scrap_one_block_when_at_boundary_with_last_finalized_block() { #[test] fn should_be_unstoppable_while_scraping_blocks_has_open_call_context() { + // TODO(DEFI-2566): This test documents the current behavior, where the ckETH minter is + // unstoppable while scraping (lots of) logs on a timer. Since log scraping calls are made on + // a loop in the callback handler for log scraping responses, the scraping continues until all + // logs have been scraped. The same call context is reused, and as long as there is an open + // call context, the minter is not stoppable. const UNSCRAPED_BLOCKS: u64 = 5_000; const NUM_BLOCK_RANGES: usize = 10; From 95aa00887c7afa0fa32e451bfa3a88770f1a6201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Mon, 15 Dec 2025 10:00:45 +0100 Subject: [PATCH 5/8] Clean up test, only tick until condition is met --- rs/ethereum/cketh/minter/tests/cketh.rs | 34 ++++-------------------- rs/ethereum/cketh/test_utils/src/lib.rs | 35 ++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index e54279164573..2e1f33c99931 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -34,7 +34,6 @@ use ic_cketh_test_utils::{ }; use ic_ethereum_types::Address; use ic_management_canister_types_private::CanisterStatusType; -use ic_types::ingress::{IngressState, IngressStatus}; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc1::transfer::Memo; use icrc_ledger_types::icrc3::transactions::{Burn, Mint}; @@ -881,30 +880,12 @@ fn should_be_unstoppable_while_scraping_blocks_has_open_call_context() { // - 3 block ranges have been scraped // - The minter has made an HTTP outcall for the 4th block range // - There's an open call context waiting for that HTTP response - // Tick once to ensure any pending requests are visible - cketh.env.tick(); - // Request to stop the minter (without providing responses to pending HTTP outcalls). // The stop will NOT complete because there's an open call context. - let stop_status = cketh.try_stop_minter_without_stopping_ongoing_https_outcalls(); - - // Verify the stop request is still Processing (not Completed) - match &stop_status { - IngressStatus::Known { state, .. } => { - assert!( - matches!(state, IngressState::Processing), - "Expected stop to still be Processing due to open call contexts, but got: {:?}", - state - ); - } - other => panic!( - "Expected IngressStatus::Known with Processing state, got: {:?}", - other - ), - } + cketh.try_stop_minter_without_stopping_ongoing_https_outcalls(); // Verify the minter is in "Stopping" state (not "Stopped") - let status = cketh.minter_status(); + let status = cketh.tick_until_minter_canister_status(CanisterStatusType::Stopping); assert_eq!( status, CanisterStatusType::Stopping, @@ -964,15 +945,10 @@ fn should_be_unstoppable_while_scraping_blocks_has_open_call_context() { .unwrap(); } - // After all scraping is complete, give a few ticks for the canister to stop. - for _ in 0..10 { - cketh.env.tick(); - } - - // Now the canister should finally be Stopped - let final_status = cketh.minter_status(); + // After all scraping is complete, the canister should finally be Stopped. + let status = cketh.tick_until_minter_canister_status(CanisterStatusType::Stopped); assert_eq!( - final_status, + status, CanisterStatusType::Stopped, "Expected minter to be Stopped after all call contexts closed" ); diff --git a/rs/ethereum/cketh/test_utils/src/lib.rs b/rs/ethereum/cketh/test_utils/src/lib.rs index d6706cf7ff7c..c0babd9b6bd0 100644 --- a/rs/ethereum/cketh/test_utils/src/lib.rs +++ b/rs/ethereum/cketh/test_utils/src/lib.rs @@ -26,7 +26,7 @@ use ic_state_machine_tests::{ }; use ic_test_utilities_load_wasm::load_wasm; use ic_types::Cycles; -use ic_types::ingress::IngressStatus; +use ic_types::ingress::{IngressState, IngressStatus}; use icrc_ledger_types::icrc1::account::Account; use icrc_ledger_types::icrc2::approve::{ApproveArgs, ApproveError}; use num_traits::cast::ToPrimitive; @@ -496,13 +496,40 @@ impl CkEthSetup { self.start_minter(); } - pub fn try_stop_minter_without_stopping_ongoing_https_outcalls(&self) -> IngressStatus { - const MAX_TICKS: u64 = 100; + /// Try to stop the minter without first stopping the ongoing HTTPS outcalls. Assert that the + /// `IngressStatus` is `Processing`. + pub fn try_stop_minter_without_stopping_ongoing_https_outcalls(&self) { + const MAX_TICKS: u64 = 10; let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id); + let mut ingress_status = self.env.ingress_status(&stop_msg_id); for _ in 0..MAX_TICKS { + if let IngressStatus::Known { state, .. } = &ingress_status { + if state == &IngressState::Processing { + return; + } + } self.env.tick(); + ingress_status = self.env.ingress_status(&stop_msg_id); + } + panic!( + "expected minter ingress status to be `Processing`, ended up with {:?}", + ingress_status + ); + } + + pub fn tick_until_minter_canister_status( + &self, + expected_canister_status: CanisterStatusType, + ) -> CanisterStatusType { + const MAX_TICKS: u64 = 10; + let mut status = self.minter_status(); + for _ in 0..MAX_TICKS { + if status == expected_canister_status { + break; + } + status = self.minter_status(); } - self.env.ingress_status(&stop_msg_id) + status } pub fn stop_minter(&self) { From fa565c2a1841cc8802f13f00cbaa05feebcc7b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Mon, 15 Dec 2025 10:03:23 +0100 Subject: [PATCH 6/8] Rename test --- rs/ethereum/cketh/minter/tests/cketh.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index 2e1f33c99931..b438205620de 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -830,7 +830,7 @@ fn should_scrap_one_block_when_at_boundary_with_last_finalized_block() { } #[test] -fn should_be_unstoppable_while_scraping_blocks_has_open_call_context() { +fn should_document_current_behavior_of_being_unstoppable_while_scraping_blocks_has_open_call_context() { // TODO(DEFI-2566): This test documents the current behavior, where the ckETH minter is // unstoppable while scraping (lots of) logs on a timer. Since log scraping calls are made on // a loop in the callback handler for log scraping responses, the scraping continues until all From 97cd2770dcdb93dca55b4892046d080758e1a2a0 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Mon, 15 Dec 2025 09:08:30 +0000 Subject: [PATCH 7/8] Automatically fixing code for linting and formatting issues --- rs/ethereum/cketh/minter/tests/cketh.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index b438205620de..b19ea1e99744 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -830,7 +830,8 @@ fn should_scrap_one_block_when_at_boundary_with_last_finalized_block() { } #[test] -fn should_document_current_behavior_of_being_unstoppable_while_scraping_blocks_has_open_call_context() { +fn should_document_current_behavior_of_being_unstoppable_while_scraping_blocks_has_open_call_context() + { // TODO(DEFI-2566): This test documents the current behavior, where the ckETH minter is // unstoppable while scraping (lots of) logs on a timer. Since log scraping calls are made on // a loop in the callback handler for log scraping responses, the scraping continues until all From c2bb7a31826edeee23514c6f1f35bd49dffffbdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Mon, 15 Dec 2025 10:40:47 +0100 Subject: [PATCH 8/8] Clippy --- rs/ethereum/cketh/test_utils/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/ethereum/cketh/test_utils/src/lib.rs b/rs/ethereum/cketh/test_utils/src/lib.rs index c0babd9b6bd0..42e0d05731d5 100644 --- a/rs/ethereum/cketh/test_utils/src/lib.rs +++ b/rs/ethereum/cketh/test_utils/src/lib.rs @@ -503,10 +503,10 @@ impl CkEthSetup { let stop_msg_id = self.env.stop_canister_non_blocking(self.minter_id); let mut ingress_status = self.env.ingress_status(&stop_msg_id); for _ in 0..MAX_TICKS { - if let IngressStatus::Known { state, .. } = &ingress_status { - if state == &IngressState::Processing { - return; - } + if let IngressStatus::Known { state, .. } = &ingress_status + && state == &IngressState::Processing + { + return; } self.env.tick(); ingress_status = self.env.ingress_status(&stop_msg_id);