From c8bfcd87c5a6b8f0a506af2df68fc050cf07f455 Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Sun, 24 Mar 2024 23:41:37 -0700 Subject: [PATCH 01/11] Implement it --- ipc/provider/src/checkpoint.rs | 97 +++++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 24 deletions(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 2c6d9daae2..f09fe56521 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -5,13 +5,16 @@ use crate::config::Subnet; use crate::manager::{BottomUpCheckpointRelayer, EthSubnetManager}; use anyhow::{anyhow, Result}; +use futures_util::future::join_all; use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; +use ipc_api::checkpoint::{BottomUpCheckpointBundle, QuorumReachedEvent}; use ipc_wallet::{EthKeyAddress, PersistentKeyStore}; use std::cmp::max; use std::fmt::{Display, Formatter}; use std::sync::{Arc, RwLock}; use std::time::Duration; +use tokio::sync::Semaphore; /// Tracks the config required for bottom up checkpoint submissions /// parent/child subnet and checkpoint period. @@ -26,10 +29,11 @@ pub struct CheckpointConfig { /// Then it will submit at the next submission height for the new checkpoint. pub struct BottomUpCheckpointManager { metadata: CheckpointConfig, - parent_handler: T, + parent_handler: Arc, child_handler: T, /// The number of blocks away from the chain head that is considered final finalization_blocks: ChainEpoch, + submission_semaphore: Arc, } impl BottomUpCheckpointManager { @@ -49,9 +53,10 @@ impl BottomUpCheckpointManager { child, period, }, - parent_handler, + parent_handler: Arc::new(parent_handler), child_handler, finalization_blocks: 0, + submission_semaphore: Arc::new(Semaphore::new(4)), }) } @@ -106,16 +111,16 @@ impl BottomUpCheckpointMan log::info!("launching {self} for {submitter}"); loop { - if let Err(e) = self.submit_next_epoch(&submitter).await { + if let Err(e) = self.submit_next_epoch(submitter.clone()).await { log::error!("cannot submit checkpoint for submitter: {submitter} due to {e}"); } - + log::debug!("JIEJIE: Sleeping for {:?}", submission_interval); tokio::time::sleep(submission_interval).await; } } /// Checks if the relayer has already submitted at the next submission epoch, if not it submits it. - async fn submit_next_epoch(&self, submitter: &Address) -> Result<()> { + async fn submit_next_epoch(&self, submitter: Address) -> Result<()> { let last_checkpoint_epoch = self .parent_handler .last_bottom_up_checkpoint_height(&self.metadata.child.id) @@ -137,6 +142,10 @@ impl BottomUpCheckpointMan let start = last_checkpoint_epoch + 1; log::debug!("start querying quorum reached events from : {start} to {finalized_height}"); + log::debug!("JIEJIE: Start a round of submission!"); + let mut count = 0; + let mut all_submit_tasks = vec![]; + for h in start..=finalized_height { let events = self.child_handler.quorum_reached_events(h).await?; if events.is_empty() { @@ -162,30 +171,70 @@ impl BottomUpCheckpointMan .await?; log::debug!("bottom up bundle: {bundle:?}"); - let epoch = self - .parent_handler - .submit_checkpoint( - submitter, - bundle.checkpoint, - bundle.signatures, - bundle.signatories, + log::debug!("JIEJIE: Trying to acquire a permit for submission"); + log::debug!("JIEJIE: ... available permits: {:}", self.submission_semaphore.available_permits()); + // We support parallel checkpoint submission using FIFO order with a limited parallelism (controlled by + // the size of submission_semaphore). + // We need to acquire a permit (from a limited permit pool) before submitting a checkpoint. + // We may wait here until a permit is available. + let parent_handler_clone = Arc::clone(&self.parent_handler); + let submission_permit = self.submission_semaphore.clone().acquire_owned().await.unwrap(); + log::debug!("JIEJIE: GOT A PERMIT. GOING TO SUBMIT A CHECKPOINT NOW!"); + all_submit_tasks.push(tokio::task::spawn(async move { + Self::submit_checkpoint( + parent_handler_clone, + submitter.clone(), + bundle, + event, ) - .await - .map_err(|e| { - anyhow!( - "cannot submit bottom up checkpoint at height {} due to: {e:}", - event.height - ) - })?; - - log::info!( - "submitted bottom up checkpoint({}) in parent at height {}", - event.height, - epoch + .await; + drop(submission_permit); + })); + + count += 1; + log::debug!( + "JIEJIE: This round has asynchronously submitted {:} checkpoints!", + count ); } } + log::debug!("JIEJIE: Waiting for all submissions to finish theirs execution"); + join_all(all_submit_tasks).await; + log::debug!( + "JIEJIE: End a round of submission, {:} submit tasks finished!", + count + ); + Ok(()) } + + async fn submit_checkpoint( + parent_handler: Arc, // Can't clone. Use Arc + submitter: Address, // Just clone + bundle: BottomUpCheckpointBundle, // Moved here + event: QuorumReachedEvent, // Moved here + ) { + let epoch = parent_handler + .submit_checkpoint( + &submitter, + bundle.checkpoint, + bundle.signatures, + bundle.signatories, + ) + .await + .map_err(|e| { + anyhow!( + "cannot submit bottom up checkpoint at height {} due to: {e:}", + event.height + ) + }) + .unwrap(); + + log::info!( + "submitted bottom up checkpoint({}) in parent at height {}", + event.height, + epoch + ); + } } From 9b4a6431febb83487e2804a984a355ad09138bbc Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Sun, 24 Mar 2024 23:43:35 -0700 Subject: [PATCH 02/11] Remove debug log --- ipc/provider/src/checkpoint.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index f09fe56521..3720528548 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -114,7 +114,6 @@ impl BottomUpCheckpointMan if let Err(e) = self.submit_next_epoch(submitter.clone()).await { log::error!("cannot submit checkpoint for submitter: {submitter} due to {e}"); } - log::debug!("JIEJIE: Sleeping for {:?}", submission_interval); tokio::time::sleep(submission_interval).await; } } @@ -142,7 +141,6 @@ impl BottomUpCheckpointMan let start = last_checkpoint_epoch + 1; log::debug!("start querying quorum reached events from : {start} to {finalized_height}"); - log::debug!("JIEJIE: Start a round of submission!"); let mut count = 0; let mut all_submit_tasks = vec![]; @@ -171,15 +169,12 @@ impl BottomUpCheckpointMan .await?; log::debug!("bottom up bundle: {bundle:?}"); - log::debug!("JIEJIE: Trying to acquire a permit for submission"); - log::debug!("JIEJIE: ... available permits: {:}", self.submission_semaphore.available_permits()); // We support parallel checkpoint submission using FIFO order with a limited parallelism (controlled by // the size of submission_semaphore). // We need to acquire a permit (from a limited permit pool) before submitting a checkpoint. // We may wait here until a permit is available. let parent_handler_clone = Arc::clone(&self.parent_handler); let submission_permit = self.submission_semaphore.clone().acquire_owned().await.unwrap(); - log::debug!("JIEJIE: GOT A PERMIT. GOING TO SUBMIT A CHECKPOINT NOW!"); all_submit_tasks.push(tokio::task::spawn(async move { Self::submit_checkpoint( parent_handler_clone, @@ -193,18 +188,14 @@ impl BottomUpCheckpointMan count += 1; log::debug!( - "JIEJIE: This round has asynchronously submitted {:} checkpoints!", + "This round has asynchronously submitted {:} checkpoints", count ); } } - log::debug!("JIEJIE: Waiting for all submissions to finish theirs execution"); + log::debug!("Waiting for all submissions to finish"); join_all(all_submit_tasks).await; - log::debug!( - "JIEJIE: End a round of submission, {:} submit tasks finished!", - count - ); Ok(()) } From 2abd2c0a8f0f2eeaa9d26a8ac23bfe587ef8c8d5 Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Sun, 24 Mar 2024 23:47:49 -0700 Subject: [PATCH 03/11] Lint fix --- ipc/provider/src/checkpoint.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 3720528548..0f03dd34a6 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -111,7 +111,7 @@ impl BottomUpCheckpointMan log::info!("launching {self} for {submitter}"); loop { - if let Err(e) = self.submit_next_epoch(submitter.clone()).await { + if let Err(e) = self.submit_next_epoch(submitter).await { log::error!("cannot submit checkpoint for submitter: {submitter} due to {e}"); } tokio::time::sleep(submission_interval).await; @@ -174,15 +174,14 @@ impl BottomUpCheckpointMan // We need to acquire a permit (from a limited permit pool) before submitting a checkpoint. // We may wait here until a permit is available. let parent_handler_clone = Arc::clone(&self.parent_handler); - let submission_permit = self.submission_semaphore.clone().acquire_owned().await.unwrap(); + let submission_permit = self + .submission_semaphore + .clone() + .acquire_owned() + .await + .unwrap(); all_submit_tasks.push(tokio::task::spawn(async move { - Self::submit_checkpoint( - parent_handler_clone, - submitter.clone(), - bundle, - event, - ) - .await; + Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event).await; drop(submission_permit); })); @@ -201,10 +200,10 @@ impl BottomUpCheckpointMan } async fn submit_checkpoint( - parent_handler: Arc, // Can't clone. Use Arc - submitter: Address, // Just clone - bundle: BottomUpCheckpointBundle, // Moved here - event: QuorumReachedEvent, // Moved here + parent_handler: Arc, + submitter: Address, + bundle: BottomUpCheckpointBundle, + event: QuorumReachedEvent, ) { let epoch = parent_handler .submit_checkpoint( From 84d83d74ad423e9ceecf6ae8d0b9b148c884c074 Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Mon, 25 Mar 2024 00:04:50 -0700 Subject: [PATCH 04/11] Expose max parallelism as cmd parameter --- ipc/cli/src/commands/checkpoint/relayer.rs | 3 +++ ipc/provider/src/checkpoint.rs | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ipc/cli/src/commands/checkpoint/relayer.rs b/ipc/cli/src/commands/checkpoint/relayer.rs index 816c60ff0d..96c1aa2855 100644 --- a/ipc/cli/src/commands/checkpoint/relayer.rs +++ b/ipc/cli/src/commands/checkpoint/relayer.rs @@ -55,6 +55,7 @@ impl CommandLineHandler for BottomUpRelayer { parent.clone(), child.clone(), Arc::new(RwLock::new(keystore)), + arguments.max_parallel_submission, ) .await?; @@ -88,4 +89,6 @@ pub(crate) struct BottomUpRelayerArgs { pub finalization_blocks: Option, #[arg(long, help = "The hex encoded address of the submitter")] pub submitter: Option, + #[arg(long, default_value = "4", help = "The max parallelism for submitting checkpoint")] + pub max_parallel_submission: usize, } diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 0f03dd34a6..8555a30b94 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -42,6 +42,7 @@ impl BottomUpCheckpointManager { child: Subnet, parent_handler: T, child_handler: T, + max_parallel_submission: usize, ) -> Result { let period = parent_handler .checkpoint_period(&child.id) @@ -56,7 +57,7 @@ impl BottomUpCheckpointManager { parent_handler: Arc::new(parent_handler), child_handler, finalization_blocks: 0, - submission_semaphore: Arc::new(Semaphore::new(4)), + submission_semaphore: Arc::new(Semaphore::new(max_parallel_submission)), }) } @@ -71,12 +72,13 @@ impl BottomUpCheckpointManager { parent: Subnet, child: Subnet, keystore: Arc>>, + max_parallel_submission: usize, ) -> Result { let parent_handler = EthSubnetManager::from_subnet_with_wallet_store(&parent, Some(keystore.clone()))?; let child_handler = EthSubnetManager::from_subnet_with_wallet_store(&child, Some(keystore))?; - Self::new(parent, child, parent_handler, child_handler).await + Self::new(parent, child, parent_handler, child_handler, max_parallel_submission).await } } From 4cd54de8c8ad195dfcb7f709ca7e7fcdc8e9a2c8 Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Mon, 25 Mar 2024 00:08:38 -0700 Subject: [PATCH 05/11] More lint --- ipc/cli/src/commands/checkpoint/relayer.rs | 6 +++++- ipc/provider/src/checkpoint.rs | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ipc/cli/src/commands/checkpoint/relayer.rs b/ipc/cli/src/commands/checkpoint/relayer.rs index 96c1aa2855..1c264fd42a 100644 --- a/ipc/cli/src/commands/checkpoint/relayer.rs +++ b/ipc/cli/src/commands/checkpoint/relayer.rs @@ -89,6 +89,10 @@ pub(crate) struct BottomUpRelayerArgs { pub finalization_blocks: Option, #[arg(long, help = "The hex encoded address of the submitter")] pub submitter: Option, - #[arg(long, default_value = "4", help = "The max parallelism for submitting checkpoint")] + #[arg( + long, + default_value = "4", + help = "The max parallelism for submitting checkpoint" + )] pub max_parallel_submission: usize, } diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 8555a30b94..25d1633b74 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -78,7 +78,14 @@ impl BottomUpCheckpointManager { EthSubnetManager::from_subnet_with_wallet_store(&parent, Some(keystore.clone()))?; let child_handler = EthSubnetManager::from_subnet_with_wallet_store(&child, Some(keystore))?; - Self::new(parent, child, parent_handler, child_handler, max_parallel_submission).await + Self::new( + parent, + child, + parent_handler, + child_handler, + max_parallel_submission, + ) + .await } } From 8137c77182cddfff13f14fab859a441e6829e8b0 Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Mon, 25 Mar 2024 14:00:55 -0700 Subject: [PATCH 06/11] Make sure we always drop permit before finish async task --- ipc/provider/src/checkpoint.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 25d1633b74..d56489b645 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -190,7 +190,14 @@ impl BottomUpCheckpointMan .await .unwrap(); all_submit_tasks.push(tokio::task::spawn(async move { - Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event).await; + match Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event).await { + Ok(()) => { + log::debug!("Successfully submitted checkpoint"); + } + Err(_err) => { + log::debug!("Failed to submit checkpoint"); + } + } drop(submission_permit); })); @@ -213,7 +220,7 @@ impl BottomUpCheckpointMan submitter: Address, bundle: BottomUpCheckpointBundle, event: QuorumReachedEvent, - ) { + ) -> Result<(), anyhow::Error> { let epoch = parent_handler .submit_checkpoint( &submitter, @@ -227,13 +234,13 @@ impl BottomUpCheckpointMan "cannot submit bottom up checkpoint at height {} due to: {e:}", event.height ) - }) - .unwrap(); + })?; log::info!( "submitted bottom up checkpoint({}) in parent at height {}", event.height, epoch ); + Ok(()) } } From 6c9a93e08296bf27db0dc52127dad4cf2ce02a65 Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Mon, 25 Mar 2024 14:07:06 -0700 Subject: [PATCH 07/11] Lint fix --- ipc/provider/src/checkpoint.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index d56489b645..fafc0febe4 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -190,7 +190,9 @@ impl BottomUpCheckpointMan .await .unwrap(); all_submit_tasks.push(tokio::task::spawn(async move { - match Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event).await { + match Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) + .await + { Ok(()) => { log::debug!("Successfully submitted checkpoint"); } From a46a84bb4061677807e08e618f4ff88515b42d9e Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Thu, 28 Mar 2024 08:57:23 -0700 Subject: [PATCH 08/11] Address Akosh's comments --- ipc/provider/src/checkpoint.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index fafc0febe4..36688d9ad4 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -5,7 +5,7 @@ use crate::config::Subnet; use crate::manager::{BottomUpCheckpointRelayer, EthSubnetManager}; use anyhow::{anyhow, Result}; -use futures_util::future::join_all; +use futures_util::future::try_join_all; use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use ipc_api::checkpoint::{BottomUpCheckpointBundle, QuorumReachedEvent}; @@ -190,29 +190,30 @@ impl BottomUpCheckpointMan .await .unwrap(); all_submit_tasks.push(tokio::task::spawn(async move { - match Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) - .await + let height = event.height; + if let Err(e) = + Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) + .await { - Ok(()) => { - log::debug!("Successfully submitted checkpoint"); - } - Err(_err) => { - log::debug!("Failed to submit checkpoint"); - } + log::error!("Fail to submit checkpoint at height {height}: {e}"); + drop(submission_permit); + Err(e) + } else { + drop(submission_permit); + Ok(()) } - drop(submission_permit); })); count += 1; log::debug!( - "This round has asynchronously submitted {:} checkpoints", - count + "This round has asynchronously submitted {count} checkpoints", ); } } log::debug!("Waiting for all submissions to finish"); - join_all(all_submit_tasks).await; + // Return error if any of the submit task failed. + try_join_all(all_submit_tasks).await?; Ok(()) } @@ -233,7 +234,7 @@ impl BottomUpCheckpointMan .await .map_err(|e| { anyhow!( - "cannot submit bottom up checkpoint at height {} due to: {e:}", + "cannot submit bottom up checkpoint at height {} due to: {e}", event.height ) })?; From 47527149ef450ce7ef562247e5c3dc5fe5d59fa4 Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Thu, 28 Mar 2024 09:12:07 -0700 Subject: [PATCH 09/11] Lint --- ipc/provider/src/checkpoint.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 36688d9ad4..95594379c3 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -205,9 +205,7 @@ impl BottomUpCheckpointMan })); count += 1; - log::debug!( - "This round has asynchronously submitted {count} checkpoints", - ); + log::debug!("This round has asynchronously submitted {count} checkpoints",); } } From 143b5395b9926a439e12c02f11483a9ad4eafd31 Mon Sep 17 00:00:00 2001 From: raulk Date: Mon, 1 Apr 2024 19:22:46 +0100 Subject: [PATCH 10/11] rename cli flag to --max-parallelism. --- ipc/cli/src/commands/checkpoint/relayer.rs | 6 +++--- ipc/provider/src/checkpoint.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ipc/cli/src/commands/checkpoint/relayer.rs b/ipc/cli/src/commands/checkpoint/relayer.rs index 1c264fd42a..43d927b409 100644 --- a/ipc/cli/src/commands/checkpoint/relayer.rs +++ b/ipc/cli/src/commands/checkpoint/relayer.rs @@ -55,7 +55,7 @@ impl CommandLineHandler for BottomUpRelayer { parent.clone(), child.clone(), Arc::new(RwLock::new(keystore)), - arguments.max_parallel_submission, + arguments.max_parallelism, ) .await?; @@ -92,7 +92,7 @@ pub(crate) struct BottomUpRelayerArgs { #[arg( long, default_value = "4", - help = "The max parallelism for submitting checkpoint" + help = "The max parallelism for submitting checkpoints" )] - pub max_parallel_submission: usize, + pub max_parallelism: usize, } diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 95594379c3..4ee6206e31 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -42,7 +42,7 @@ impl BottomUpCheckpointManager { child: Subnet, parent_handler: T, child_handler: T, - max_parallel_submission: usize, + max_parallelism: usize, ) -> Result { let period = parent_handler .checkpoint_period(&child.id) @@ -57,7 +57,7 @@ impl BottomUpCheckpointManager { parent_handler: Arc::new(parent_handler), child_handler, finalization_blocks: 0, - submission_semaphore: Arc::new(Semaphore::new(max_parallel_submission)), + submission_semaphore: Arc::new(Semaphore::new(max_parallelism)), }) } @@ -72,7 +72,7 @@ impl BottomUpCheckpointManager { parent: Subnet, child: Subnet, keystore: Arc>>, - max_parallel_submission: usize, + max_parallelism: usize, ) -> Result { let parent_handler = EthSubnetManager::from_subnet_with_wallet_store(&parent, Some(keystore.clone()))?; @@ -83,7 +83,7 @@ impl BottomUpCheckpointManager { child, parent_handler, child_handler, - max_parallel_submission, + max_parallelism, ) .await } From 207d6e0ddde22c238c73be2ec31f52cfe22d8c55 Mon Sep 17 00:00:00 2001 From: raulk Date: Mon, 1 Apr 2024 19:34:33 +0100 Subject: [PATCH 11/11] use inspect_err. --- ipc/provider/src/checkpoint.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 4ee6206e31..7066844c98 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -191,17 +191,14 @@ impl BottomUpCheckpointMan .unwrap(); all_submit_tasks.push(tokio::task::spawn(async move { let height = event.height; - if let Err(e) = + let result = Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) .await - { - log::error!("Fail to submit checkpoint at height {height}: {e}"); - drop(submission_permit); - Err(e) - } else { - drop(submission_permit); - Ok(()) - } + .inspect_err(|err| { + log::error!("Fail to submit checkpoint at height {height}: {err}"); + }); + drop(submission_permit); + result })); count += 1;