Extrinsic Submission Timeout

bittensor/core/extrinsics/registration.py

@@ -13,11 +13,8 @@
 from bittensor.utils import format_error_message, unlock_key
 from bittensor.utils.btlogging import logging
 from bittensor.utils.networking import ensure_connected
-from bittensor.utils.registration import (
-    create_pow,
-    torch,
-    log_no_torch_error,
-)
+from bittensor.utils.registration import create_pow, torch, log_no_torch_error
+from bittensor.core.extrinsics.utils import submit_extrinsic
 
 # For annotation and lazy import purposes
 if TYPE_CHECKING:
@@ -68,8 +65,9 @@ def _do_pow_register(
         },
     )
     extrinsic = self.substrate.create_signed_extrinsic(call=call, keypair=wallet.hotkey)
-    response = self.substrate.submit_extrinsic(
-        extrinsic,
+    response = submit_extrinsic(
+        substrate=self.substrate,
+        extrinsic=extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
     )
@@ -298,7 +296,8 @@ def _do_burned_register(
     extrinsic = self.substrate.create_signed_extrinsic(
         call=call, keypair=wallet.coldkey
     )
-    response = self.substrate.submit_extrinsic(
+    response = submit_extrinsic(
+        self.substrate,
         extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,

bittensor/core/extrinsics/root.py

@@ -5,6 +5,7 @@
 from numpy.typing import NDArray
 
 from bittensor.core.settings import version_as_int
+from bittensor.core.extrinsics.utils import submit_extrinsic
 from bittensor.utils import format_error_message, weight_utils, unlock_key
 from bittensor.utils.btlogging import logging
 from bittensor.utils.networking import ensure_connected
@@ -31,7 +32,8 @@ def _do_root_register(
     extrinsic = self.substrate.create_signed_extrinsic(
         call=call, keypair=wallet.coldkey
     )
-    response = self.substrate.submit_extrinsic(
+    response = submit_extrinsic(
+        self.substrate,
         extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
@@ -156,7 +158,8 @@ def _do_set_root_weights(
         keypair=wallet.coldkey,
         era={"period": period},
     )
-    response = self.substrate.submit_extrinsic(
+    response = submit_extrinsic(
+        self.substrate,
         extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,

bittensor/core/extrinsics/serving.py

@@ -288,7 +288,8 @@ def publish_metadata(
         )
 
         extrinsic = substrate.create_signed_extrinsic(call=call, keypair=wallet.hotkey)
-        response = substrate.submit_extrinsic(
+        response = submit_extrinsic(
+            substrate,
             extrinsic,
             wait_for_inclusion=wait_for_inclusion,
             wait_for_finalization=wait_for_finalization,

bittensor/core/extrinsics/utils.py

@@ -1,21 +1,33 @@
 """Module with helper functions for extrinsics."""
 
+import threading
 from typing import TYPE_CHECKING
-from substrateinterface.exceptions import SubstrateRequestException
+
+from substrateinterface.exceptions import SubstrateRequestException, ExtrinsicNotFound
+
 from bittensor.utils.btlogging import logging
 from bittensor.utils import format_error_message
 
 if TYPE_CHECKING:
-    from substrateinterface import SubstrateInterface
+    from substrateinterface import SubstrateInterface, ExtrinsicReceipt
     from scalecodec.types import GenericExtrinsic
 
 
+class _ThreadingTimeoutException(Exception):
+    """
+    Exception raised for timeout. Different from TimeoutException because this also triggers
+    a websocket failure. This exception should only be used with `threading` timer..
+    """
+
+    pass
+
+
 def submit_extrinsic(
     substrate: "SubstrateInterface",
     extrinsic: "GenericExtrinsic",
     wait_for_inclusion: bool,
     wait_for_finalization: bool,
-):
+) -> "ExtrinsicReceipt":
     """
     Submits an extrinsic to the substrate blockchain and handles potential exceptions.
 
@@ -35,7 +47,22 @@ def submit_extrinsic(
     Raises:
         SubstrateRequestException: If the submission of the extrinsic fails, the error is logged and re-raised.
     """
+    extrinsic_hash = extrinsic.extrinsic_hash
+    starting_block = substrate.get_block()
+
+    def _handler():
+        """
+        Timeout handler for threading. Will raise a TimeoutError if timeout is exceeded.
+        """
+        logging.error("Timed out waiting for extrinsic submission.")
+        raise _ThreadingTimeoutException
+
+    # sets a timeout timer for the next call to 200 seconds
+    # will raise a _ThreadingTimeoutException if it reaches this point
+    timer = threading.Timer(200, _handler)
+
     try:
+        timer.start()
         response = substrate.submit_extrinsic(
             extrinsic,
             wait_for_inclusion=wait_for_inclusion,
@@ -46,4 +73,32 @@ def submit_extrinsic(
         # Re-rise the exception for retrying of the extrinsic call. If we remove the retry logic, the raise will need
         # to be removed.
         raise
+
+    except _ThreadingTimeoutException:
+        after_timeout_block = substrate.get_block()
+
+        response = None
+        for block_num in range(
+            starting_block["header"]["number"],
+            after_timeout_block["header"]["number"] + 1,
+        ):
+            block_hash = substrate.get_block_hash(block_num)
+            try:
+                response = substrate.retrieve_extrinsic_by_hash(
+                    block_hash, f"0x{extrinsic_hash.hex()}"
+                )
+            except ExtrinsicNotFound:
+                continue
+            if response:
+                break
+    finally:
+        timer.cancel()
+
+    if response is None:
+        logging.error(
+            f"Extrinsic '0x{extrinsic_hash.hex()}' not submitted. "
+            f"Initially attempted to submit at block {starting_block['header']['number']}."
+        )
+        raise SubstrateRequestException
+
     return response

tests/e2e_tests/utils/e2e_test_utils.py

@@ -50,6 +50,7 @@ def clone_or_update_templates(specific_commit=None):
     os.chdir(install_dir)
 
     for repo, git_link in repo_mapping.items():
+        print(os.path.abspath(repo))
         if not os.path.exists(repo):
             print(f"\033[94mCloning {repo}...\033[0m")
             subprocess.run(["git", "clone", git_link, repo], check=True)