From b3781bb375248e589fdfd98731f76d30d98049fd Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 15 Jan 2025 17:53:09 +0200 Subject: [PATCH 01/52] Uses a different asyncio loop caller. --- bittensor/core/subtensor.py | 38 +++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 4fc0b5b9c5..c0e26cdfa3 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1,3 +1,5 @@ +import asyncio +import threading import warnings from functools import lru_cache from typing import TYPE_CHECKING, Any, Iterable, Optional, Union @@ -12,7 +14,6 @@ from bittensor.utils import ( execute_coroutine, torch, - get_event_loop, event_loop_is_running, ) @@ -34,6 +35,28 @@ from scalecodec.types import ScaleType +class SynchronousAsyncCaller: + def __init__(self): + self.loop = None + self.thread = threading.Thread(target=self._start_loop, args=()) + self.thread.start() + + def _start_loop(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + self.loop.run_forever() + + def run_coroutine(self, coro): + while self.loop is None: + pass + future = asyncio.run_coroutine_threadsafe(coro, self.loop) + return future.result() + + def stop(self): + self.loop.call_soon_threadsafe(self.loop.stop) + self.thread.join() + + class Subtensor: """ Represents a synchronous interface for `bittensor.core.async_subtensor.AsyncSubtensor`. @@ -63,7 +86,8 @@ def __init__( "You are calling this from an already running event loop. Some features may not work correctly. You " "should instead use `AsyncSubtensor`." ) - self.event_loop = get_event_loop() + self.caller = SynchronousAsyncCaller() + self.event_loop = self.caller.loop self.network = network self._config = config self.log_verbose = log_verbose @@ -88,11 +112,17 @@ def __str__(self): def __repr__(self): return self.async_subtensor.__repr__() + def __del__(self): + try: + self.caller.stop() + except AttributeError: + pass + def execute_coroutine(self, coroutine) -> Any: - return execute_coroutine(coroutine, self.event_loop) + return self.caller.run_coroutine(coroutine) def close(self): - execute_coroutine(self.async_subtensor.close()) + self.execute_coroutine(self.async_subtensor.close()) # Subtensor queries =========================================================================================== From 1701d7c03f37656a737b1ba7af4ea266c91dcaa3 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 15 Jan 2025 22:55:17 +0200 Subject: [PATCH 02/52] Applies EventLoopManager broadly, adds factory functions for creating initialized async objects. --- bittensor/core/async_subtensor.py | 44 +++++++++++++++-- bittensor/core/extrinsics/commit_reveal.py | 6 +-- bittensor/core/extrinsics/commit_weights.py | 11 ++--- bittensor/core/extrinsics/registration.py | 11 ++--- bittensor/core/extrinsics/root.py | 11 ++--- bittensor/core/extrinsics/serving.py | 33 ++++++------- bittensor/core/extrinsics/set_weights.py | 6 +-- bittensor/core/extrinsics/staking.py | 11 ++--- bittensor/core/extrinsics/transfer.py | 6 +-- bittensor/core/extrinsics/unstaking.py | 11 ++--- bittensor/core/metagraph.py | 53 +++++++++++++-------- bittensor/core/subtensor.py | 38 ++++----------- bittensor/utils/__init__.py | 2 - tests/unit_tests/extrinsics/test_serving.py | 6 +-- 14 files changed, 124 insertions(+), 125 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 0e41597aa8..f9f913b640 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -68,7 +68,7 @@ ss58_to_vec_u8, torch, u16_normalized_float, - execute_coroutine, + event_loop_is_running, ) from bittensor.utils import networking from bittensor.utils.balance import Balance @@ -119,7 +119,6 @@ def __init__( config: Optional["Config"] = None, _mock: bool = False, log_verbose: bool = False, - event_loop: asyncio.AbstractEventLoop = None, ): """ Initializes an instance of the AsyncSubtensor class. @@ -155,7 +154,6 @@ def __init__( type_registry=TYPE_REGISTRY, use_remote_preset=True, chain_name="Bittensor", - event_loop=event_loop, _mock=_mock, ) if self.log_verbose: @@ -170,7 +168,8 @@ def __repr__(self): return self.__str__() def __del__(self): - execute_coroutine(self.close()) + if loop := event_loop_is_running(): + loop.create_task(self.close()) def _check_and_log_network_settings(self): if self.network == settings.NETWORKS[3]: # local @@ -358,6 +357,26 @@ async def close(self): if self.substrate: await self.substrate.close() + async def initialize(self): + logging.info( + f"[magenta]Connecting to Substrate:[/magenta] [blue]{self}[/blue][magenta]...[/magenta]" + ) + try: + await self.substrate.initialize() + return self + except TimeoutError: + logging.error( + f"[red]Error[/red]: Timeout occurred connecting to substrate." + f" Verify your chain and network settings: {self}" + ) + raise ConnectionError + except (ConnectionRefusedError, ssl.SSLError) as error: + logging.error( + f"[red]Error[/red]: Connection refused when connecting to substrate. " + f"Verify your chain and network settings: {self}. Error: {error}" + ) + raise ConnectionError + async def __aenter__(self): logging.info( f"[magenta]Connecting to Substrate:[/magenta] [blue]{self}[/blue][magenta]...[/magenta]" @@ -3470,3 +3489,20 @@ async def unstake_multiple( wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) + + +async def async_subtensor( + network: Optional[str] = None, + config: Optional["Config"] = None, + _mock: bool = False, + log_verbose: bool = False, +) -> "AsyncSubtensor": + """ + Factory method to create an initialized AsyncSubtensor. Mainly useful for when you don't want to run + `await subtensor.initialize()` after instantiation. + """ + sub = AsyncSubtensor( + network=network, config=config, _mock=_mock, log_verbose=log_verbose + ) + await sub.initialize() + return sub diff --git a/bittensor/core/extrinsics/commit_reveal.py b/bittensor/core/extrinsics/commit_reveal.py index eac792f897..739e6dbff2 100644 --- a/bittensor/core/extrinsics/commit_reveal.py +++ b/bittensor/core/extrinsics/commit_reveal.py @@ -9,7 +9,6 @@ commit_reveal_v3_extrinsic as async_commit_reveal_v3_extrinsic, ) from bittensor.core.settings import version_as_int -from bittensor.utils import execute_coroutine if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -27,7 +26,7 @@ def commit_reveal_v3_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_commit_reveal_v3_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -37,6 +36,5 @@ def commit_reveal_v3_extrinsic( version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/commit_weights.py b/bittensor/core/extrinsics/commit_weights.py index bd98f32ecc..289918839b 100644 --- a/bittensor/core/extrinsics/commit_weights.py +++ b/bittensor/core/extrinsics/commit_weights.py @@ -6,7 +6,6 @@ reveal_weights_extrinsic as async_reveal_weights_extrinsic, commit_weights_extrinsic as async_commit_weights_extrinsic, ) -from bittensor.utils import execute_coroutine if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -21,7 +20,7 @@ def commit_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_commit_weights_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -29,8 +28,7 @@ def commit_weights_extrinsic( commit_hash=commit_hash, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) @@ -45,7 +43,7 @@ def reveal_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_reveal_weights_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -56,6 +54,5 @@ def reveal_weights_extrinsic( version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/registration.py b/bittensor/core/extrinsics/registration.py index 5fd231fa35..fc43f94795 100644 --- a/bittensor/core/extrinsics/registration.py +++ b/bittensor/core/extrinsics/registration.py @@ -12,7 +12,6 @@ burned_register_extrinsic as async_burned_register_extrinsic, register_extrinsic as async_register_extrinsic, ) -from bittensor.utils import execute_coroutine # For annotation and lazy import purposes if TYPE_CHECKING: @@ -27,15 +26,14 @@ def burned_register_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_burned_register_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, netuid=netuid, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) @@ -54,7 +52,7 @@ def register_extrinsic( update_interval: Optional[int] = None, log_verbose: bool = False, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_register_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -69,6 +67,5 @@ def register_extrinsic( num_processes=num_processes, update_interval=update_interval, log_verbose=log_verbose, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/root.py b/bittensor/core/extrinsics/root.py index a0312a4211..8631e652c0 100644 --- a/bittensor/core/extrinsics/root.py +++ b/bittensor/core/extrinsics/root.py @@ -7,7 +7,6 @@ root_register_extrinsic as async_root_register_extrinsic, set_root_weights_extrinsic as async_set_root_weights_extrinsic, ) -from bittensor.utils import execute_coroutine from bittensor.utils.registration import torch if TYPE_CHECKING: @@ -21,15 +20,14 @@ def root_register_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_root_register_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, netuid=0, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) @@ -42,7 +40,7 @@ def set_root_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_set_root_weights_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -51,6 +49,5 @@ def set_root_weights_extrinsic( version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/serving.py b/bittensor/core/extrinsics/serving.py index 8a8d9c82d1..04c124bfe0 100644 --- a/bittensor/core/extrinsics/serving.py +++ b/bittensor/core/extrinsics/serving.py @@ -6,7 +6,6 @@ publish_metadata as async_publish_metadata, get_metadata as async_get_metadata, ) -from bittensor.utils import execute_coroutine if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -17,21 +16,20 @@ def do_serve_axon( - self: "Subtensor", + subtensor: "Subtensor", wallet: "Wallet", call_params: "AxonServeCallParams", wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> tuple[bool, Optional[dict]]: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_do_serve_axon( - subtensor=self.async_subtensor, + subtensor=subtensor.async_subtensor, wallet=wallet, call_params=call_params, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=self.event_loop, + ) ) @@ -43,7 +41,7 @@ def serve_axon_extrinsic( wait_for_finalization: bool = True, certificate: Optional["Certificate"] = None, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_serve_axon_extrinsic( subtensor=subtensor.async_subtensor, netuid=netuid, @@ -51,13 +49,12 @@ def serve_axon_extrinsic( wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, certificate=certificate, - ), - event_loop=subtensor.event_loop, + ) ) def publish_metadata( - self: "Subtensor", + subtensor: "Subtensor", wallet: "Wallet", netuid: int, data_type: str, @@ -65,29 +62,27 @@ def publish_metadata( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_publish_metadata( - subtensor=self.async_subtensor, + subtensor=subtensor.async_subtensor, wallet=wallet, netuid=netuid, data_type=data_type, data=data, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=self.event_loop, + ) ) def get_metadata( - self: "Subtensor", netuid: int, hotkey: str, block: Optional[int] = None + subtensor: "Subtensor", netuid: int, hotkey: str, block: Optional[int] = None ) -> str: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_get_metadata( - subtensor=self.async_subtensor, + subtensor=subtensor.async_subtensor, netuid=netuid, hotkey=hotkey, block=block, - ), - event_loop=self.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index 913ae29a8b..a908e59c09 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -8,7 +8,6 @@ from bittensor.core.extrinsics.asyncex.weights import ( set_weights_extrinsic as async_set_weights_extrinsic, ) -from bittensor.utils import execute_coroutine from bittensor.utils.registration import torch if TYPE_CHECKING: @@ -26,7 +25,7 @@ def set_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_set_weights_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -36,6 +35,5 @@ def set_weights_extrinsic( version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/staking.py b/bittensor/core/extrinsics/staking.py index 7245ce98e9..ea127327ed 100644 --- a/bittensor/core/extrinsics/staking.py +++ b/bittensor/core/extrinsics/staking.py @@ -4,7 +4,6 @@ add_stake_extrinsic as async_add_stake_extrinsic, add_stake_multiple_extrinsic as async_add_stake_multiple_extrinsic, ) -from bittensor.utils import execute_coroutine if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -20,7 +19,7 @@ def add_stake_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_add_stake_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -28,8 +27,7 @@ def add_stake_extrinsic( amount=amount, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) @@ -41,7 +39,7 @@ def add_stake_multiple_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_add_stake_multiple_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -49,6 +47,5 @@ def add_stake_multiple_extrinsic( amounts=amounts, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/transfer.py b/bittensor/core/extrinsics/transfer.py index fbf6267b19..c4a0b73072 100644 --- a/bittensor/core/extrinsics/transfer.py +++ b/bittensor/core/extrinsics/transfer.py @@ -3,7 +3,6 @@ from bittensor.core.extrinsics.asyncex.transfer import ( transfer_extrinsic as async_transfer_extrinsic, ) -from bittensor.utils import execute_coroutine if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -21,7 +20,7 @@ def transfer_extrinsic( wait_for_finalization: bool = False, keep_alive: bool = True, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_transfer_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -31,6 +30,5 @@ def transfer_extrinsic( wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, keep_alive=keep_alive, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/extrinsics/unstaking.py b/bittensor/core/extrinsics/unstaking.py index 0b7a425f34..c647daef3f 100644 --- a/bittensor/core/extrinsics/unstaking.py +++ b/bittensor/core/extrinsics/unstaking.py @@ -4,7 +4,6 @@ unstake_extrinsic as async_unstake_extrinsic, unstake_multiple_extrinsic as async_unstake_multiple_extrinsic, ) -from bittensor.utils import execute_coroutine from bittensor.utils.balance import Balance if TYPE_CHECKING: @@ -20,7 +19,7 @@ def unstake_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_unstake_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -28,8 +27,7 @@ def unstake_extrinsic( amount=amount, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) @@ -41,7 +39,7 @@ def unstake_multiple_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return execute_coroutine( + return subtensor.execute_coroutine( coroutine=async_unstake_multiple_extrinsic( subtensor=subtensor.async_subtensor, wallet=wallet, @@ -49,6 +47,5 @@ def unstake_multiple_extrinsic( amounts=amounts, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), - event_loop=subtensor.event_loop, + ) ) diff --git a/bittensor/core/metagraph.py b/bittensor/core/metagraph.py index 20c18db41c..4455ee9754 100644 --- a/bittensor/core/metagraph.py +++ b/bittensor/core/metagraph.py @@ -8,6 +8,7 @@ from os.path import join from typing import Optional, Union +from async_substrate_interface.utils import EventLoopManager import numpy as np from numpy.typing import NDArray @@ -20,7 +21,6 @@ ) from bittensor.core import settings from bittensor.core.chain_data import AxonInfo -from bittensor.utils import execute_coroutine # For annotation purposes if typing.TYPE_CHECKING: @@ -475,7 +475,7 @@ def __init__( Example: Initializing a metagraph object for the Bittensor network with a specific network UID:: - metagraph = metagraph(netuid=123, network="finney", lite=True, sync=True) + metagraph = Metagraph(netuid=123, network="finney", lite=True, sync=True) """ @@ -1134,9 +1134,6 @@ def __init__( self.subtensor = subtensor self.should_sync = sync - if self.should_sync: - execute_coroutine(self.sync(block=None, lite=lite, subtensor=subtensor)) - async def __aenter__(self): if self.should_sync: await self.sync(block=None, lite=self.lite, subtensor=self.subtensor) @@ -1338,9 +1335,6 @@ def __init__( self.subtensor = subtensor self.should_sync = sync - if self.should_sync: - execute_coroutine(self.sync(block=None, lite=lite, subtensor=subtensor)) - async def __aenter__(self): if self.should_sync: await self.sync(block=None, lite=self.lite, subtensor=self.subtensor) @@ -1516,6 +1510,13 @@ def __init__( sync=sync, subtensor=subtensor.async_subtensor if subtensor else None, ) + if self.subtensor: + self.event_loop_mgr = self.subtensor.event_loop_mgr + else: + self.event_loop_mgr = EventLoopManager() + if sync: + if self.subtensor: + self.subtensor.event_loop_mgr.run(self._async_metagraph.sync()) def sync( self, @@ -1525,18 +1526,17 @@ def sync( ): """Synchronizes the metagraph to the specified block, lite, and subtensor instance if available.""" if subtensor: - event_loop = subtensor.event_loop + event_loop_mgr = subtensor.event_loop_mgr elif self.subtensor: - event_loop = self.subtensor.event_loop + event_loop_mgr = self.subtensor.event_loop_mgr else: - event_loop = None - execute_coroutine( + event_loop_mgr = self.event_loop_mgr + event_loop_mgr.run( self._async_metagraph.sync( block=block, lite=lite, subtensor=subtensor.async_subtensor if subtensor else None, - ), - event_loop=event_loop, + ) ) def __getattr__(self, name): @@ -1545,12 +1545,25 @@ def __getattr__(self, name): if asyncio.iscoroutinefunction(attr): def wrapper(*args, **kwargs): - return execute_coroutine( - attr(*args, **kwargs), - event_loop=self.subtensor.event_loop - if self.subtensor - else None, - ) + return self.event_loop_mgr.run(attr(*args, **kwargs)) return wrapper return attr + + +async def async_metagraph( + netuid: int, + network: str = settings.DEFAULT_NETWORK, + lite: bool = True, + sync: bool = True, + subtensor: "AsyncSubtensor" = None, +) -> "AsyncMetagraph": + """ + Factory function to create an instantiated AsyncMetagraph, mainly for the ability to use sync at instantiation. + """ + metagraph_ = AsyncMetagraph( + netuid=netuid, network=network, lite=lite, sync=sync, subtensor=subtensor + ) + if sync: + await metagraph_.sync() + return metagraph_ diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index c0e26cdfa3..0978518aae 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1,11 +1,10 @@ -import asyncio -import threading import warnings from functools import lru_cache from typing import TYPE_CHECKING, Any, Iterable, Optional, Union import numpy as np from async_substrate_interface import SubstrateInterface +from async_substrate_interface.utils import EventLoopManager from numpy.typing import NDArray from bittensor.core.async_subtensor import AsyncSubtensor @@ -35,28 +34,6 @@ from scalecodec.types import ScaleType -class SynchronousAsyncCaller: - def __init__(self): - self.loop = None - self.thread = threading.Thread(target=self._start_loop, args=()) - self.thread.start() - - def _start_loop(self): - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - self.loop.run_forever() - - def run_coroutine(self, coro): - while self.loop is None: - pass - future = asyncio.run_coroutine_threadsafe(coro, self.loop) - return future.result() - - def stop(self): - self.loop.call_soon_threadsafe(self.loop.stop) - self.thread.join() - - class Subtensor: """ Represents a synchronous interface for `bittensor.core.async_subtensor.AsyncSubtensor`. @@ -73,6 +50,7 @@ class Subtensor: determine_chain_endpoint_and_network = ( AsyncSubtensor.determine_chain_endpoint_and_network ) + event_loop_mgr: EventLoopManager def __init__( self, @@ -86,8 +64,8 @@ def __init__( "You are calling this from an already running event loop. Some features may not work correctly. You " "should instead use `AsyncSubtensor`." ) - self.caller = SynchronousAsyncCaller() - self.event_loop = self.caller.loop + self.event_loop_mgr = EventLoopManager() + self.event_loop = self.event_loop_mgr.loop self.network = network self._config = config self.log_verbose = log_verbose @@ -114,12 +92,12 @@ def __repr__(self): def __del__(self): try: - self.caller.stop() + self.event_loop_mgr.stop() except AttributeError: pass def execute_coroutine(self, coroutine) -> Any: - return self.caller.run_coroutine(coroutine) + return self.event_loop_mgr.run(coroutine) def close(self): self.execute_coroutine(self.async_subtensor.close()) @@ -786,12 +764,12 @@ def root_register( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return execute_coroutine( + return self.execute_coroutine( self.async_subtensor.root_register( wallet=wallet, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, - ), + ) ) def root_set_weights( diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index ac9341d461..9de7ba75bd 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -9,7 +9,6 @@ event_loop_is_running, hex_to_bytes, get_event_loop, - execute_coroutine, ) from bittensor_wallet import Keypair from bittensor_wallet.errors import KeyFileError, PasswordError @@ -35,7 +34,6 @@ event_loop_is_running = event_loop_is_running hex_to_bytes = hex_to_bytes get_event_loop = get_event_loop -execute_coroutine = execute_coroutine RAOPERTAO = 1e9 diff --git a/tests/unit_tests/extrinsics/test_serving.py b/tests/unit_tests/extrinsics/test_serving.py index 27b11ede1f..9b3e2393ea 100644 --- a/tests/unit_tests/extrinsics/test_serving.py +++ b/tests/unit_tests/extrinsics/test_serving.py @@ -16,7 +16,7 @@ def test_do_serve_axon(mocker): # Call result = serving.do_serve_axon( - self=fake_subtensor, + subtensor=fake_subtensor, wallet=fake_wallet, call_params=call_params, wait_for_inclusion=wait_for_inclusion, @@ -97,7 +97,7 @@ def test_publish_metadata(mocker): # Call result = serving.publish_metadata( - self=fake_subtensor, + subtensor=fake_subtensor, wallet=fake_wallet, netuid=netuid, data_type=data_type, @@ -138,7 +138,7 @@ def test_get_metadata(mocker): # Call result = serving.get_metadata( - self=fake_subtensor, + subtensor=fake_subtensor, netuid=netuid, hotkey=hotkey, block=block, From ba231199e4c50b91cb7317fcd151b503baaaf467 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 15 Jan 2025 23:51:04 +0200 Subject: [PATCH 03/52] Renaming --- bittensor/core/async_subtensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index f9f913b640..2fecd1fe4e 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -3491,7 +3491,7 @@ async def unstake_multiple( ) -async def async_subtensor( +async def get_async_subtensor( network: Optional[str] = None, config: Optional["Config"] = None, _mock: bool = False, From f79f77b6b7e4d2a57de97b60666b13a7559700eb Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 16 Jan 2025 01:06:14 +0200 Subject: [PATCH 04/52] Pass the EventLoopManager to substrate --- bittensor/core/subtensor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 0978518aae..21b8a6ea35 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -73,14 +73,13 @@ def __init__( network=network, config=config, log_verbose=log_verbose, - event_loop=self.event_loop, _mock=_mock, ) - self.substrate = SubstrateInterface( url=self.async_subtensor.chain_endpoint, _mock=_mock, substrate=self.async_subtensor.substrate, + event_loop_manager=self.event_loop_mgr ) self.chain_endpoint = self.async_subtensor.chain_endpoint From 54591b945ff7d78a9fabbd79ebd0a05dce05bbae Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 16 Jan 2025 16:02:32 +0200 Subject: [PATCH 05/52] Remove execute_coroutine import --- bittensor/core/subtensor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 21b8a6ea35..db0b13b896 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -11,7 +11,6 @@ from bittensor.core.metagraph import Metagraph from bittensor.core.settings import version_as_int from bittensor.utils import ( - execute_coroutine, torch, event_loop_is_running, ) @@ -79,7 +78,7 @@ def __init__( url=self.async_subtensor.chain_endpoint, _mock=_mock, substrate=self.async_subtensor.substrate, - event_loop_manager=self.event_loop_mgr + event_loop_manager=self.event_loop_mgr, ) self.chain_endpoint = self.async_subtensor.chain_endpoint From 8ce006eac563830b009f9ec09e3992750bd7dac8 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 16 Jan 2025 20:58:41 +0200 Subject: [PATCH 06/52] Remove attr --- bittensor/core/subtensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index db0b13b896..9b804c8db6 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -64,7 +64,6 @@ def __init__( "should instead use `AsyncSubtensor`." ) self.event_loop_mgr = EventLoopManager() - self.event_loop = self.event_loop_mgr.loop self.network = network self._config = config self.log_verbose = log_verbose @@ -99,6 +98,7 @@ def execute_coroutine(self, coroutine) -> Any: def close(self): self.execute_coroutine(self.async_subtensor.close()) + self.event_loop_mgr.stop() # Subtensor queries =========================================================================================== From 680127075a1985cca387d7c95d05d66971b4dafc Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Sat, 18 Jan 2025 18:08:32 +0200 Subject: [PATCH 07/52] [WIP] Check-in --- bittensor/core/__init__.py | 201 ++++++++++++++ bittensor/core/async_subtensor.py | 199 +------------- bittensor/core/metagraph.py | 5 - bittensor/core/subtensor.py | 432 +++++++++++++++++++++++------- 4 files changed, 537 insertions(+), 300 deletions(-) diff --git a/bittensor/core/__init__.py b/bittensor/core/__init__.py index e69de29bb2..e8411b35c2 100644 --- a/bittensor/core/__init__.py +++ b/bittensor/core/__init__.py @@ -0,0 +1,201 @@ +from abc import ABC +import argparse +from typing import Optional + +from bittensor.utils import networking +from bittensor.utils.btlogging import logging +from bittensor.core import settings +from bittensor.core.config import Config + + +class SubtensorMixin(ABC): + network: str + chain_endpoint: str + log_verbose: bool + + def __str__(self): + return f"Network: {self.network}, Chain: {self.chain_endpoint}" + + def __repr__(self): + return self.__str__() + + def _check_and_log_network_settings(self): + if self.network == settings.NETWORKS[3]: # local + logging.warning( + ":warning: Verify your local subtensor is running on port [blue]9944[/blue]." + ) + + if ( + self.network == "finney" + or self.chain_endpoint == settings.FINNEY_ENTRYPOINT + ) and self.log_verbose: + logging.info( + f"You are connecting to {self.network} network with endpoint {self.chain_endpoint}." + ) + logging.debug( + "We strongly encourage running a local subtensor node whenever possible. " + "This increases decentralization and resilience of the network." + ) + # TODO: remove or apply this warning as updated default endpoint? + logging.debug( + "In a future release, local subtensor will become the default endpoint. " + "To get ahead of this change, please run a local subtensor node and point to it." + ) + + @staticmethod # TODO can this be a class method? + def config() -> "Config": + """ + Creates and returns a Bittensor configuration object. + + Returns: + config (bittensor.core.config.Config): A Bittensor configuration object configured with arguments added by + the `subtensor.add_args` method. + """ + parser = argparse.ArgumentParser() + SubtensorMixin.add_args(parser) + return Config(parser) + + @staticmethod + def setup_config(network: Optional[str], config: "Config"): + """ + Sets up and returns the configuration for the Subtensor network and endpoint. + + This method determines the appropriate network and chain endpoint based on the provided network string or + configuration object. It evaluates the network and endpoint in the following order of precedence: + 1. Provided network string. + 2. Configured chain endpoint in the `config` object. + 3. Configured network in the `config` object. + 4. Default chain endpoint. + 5. Default network. + + Arguments: + network (Optional[str]): The name of the Subtensor network. If None, the network and endpoint will be + determined from the `config` object. + config (bittensor.core.config.Config): The configuration object containing the network and chain endpoint + settings. + + Returns: + tuple: A tuple containing the formatted WebSocket endpoint URL and the evaluated network name. + """ + if network is None: + candidates = [ + ( + config.is_set("subtensor.chain_endpoint"), + config.subtensor.chain_endpoint, + ), + (config.is_set("subtensor.network"), config.subtensor.network), + ( + config.subtensor.get("chain_endpoint"), + config.subtensor.chain_endpoint, + ), + (config.subtensor.get("network"), config.subtensor.network), + ] + for check, config_network in candidates: + if check: + network = config_network + + evaluated_network, evaluated_endpoint = ( + SubtensorMixin.determine_chain_endpoint_and_network(network) + ) + + return networking.get_formatted_ws_endpoint_url( + evaluated_endpoint + ), evaluated_network + + @classmethod + def help(cls): + """Print help to stdout.""" + parser = argparse.ArgumentParser() + cls.add_args(parser) + print(cls.__new__.__doc__) + parser.print_help() + + @classmethod + def add_args(cls, parser: "argparse.ArgumentParser", prefix: Optional[str] = None): + """ + Adds command-line arguments to the provided ArgumentParser for configuring the Subtensor settings. + + Arguments: + parser (argparse.ArgumentParser): The ArgumentParser object to which the Subtensor arguments will be added. + prefix (Optional[str]): An optional prefix for the argument names. If provided, the prefix is prepended to + each argument name. + + Arguments added: + --subtensor.network: The Subtensor network flag. Possible values are 'finney', 'test', 'archive', and + 'local'. Overrides the chain endpoint if set. + --subtensor.chain_endpoint: The Subtensor chain endpoint flag. If set, it overrides the network flag. + --subtensor._mock: If true, uses a mocked connection to the chain. + + Example: + parser = argparse.ArgumentParser() + Subtensor.add_args(parser) + """ + prefix_str = "" if prefix is None else f"{prefix}." + try: + default_network = settings.DEFAULT_NETWORK + default_chain_endpoint = settings.FINNEY_ENTRYPOINT + + parser.add_argument( + f"--{prefix_str}subtensor.network", + default=default_network, + type=str, + help="""The subtensor network flag. The likely choices are: + -- finney (main network) + -- test (test network) + -- archive (archive network +300 blocks) + -- local (local running network) + If this option is set it overloads subtensor.chain_endpoint with + an entry point node from that network. + """, + ) + parser.add_argument( + f"--{prefix_str}subtensor.chain_endpoint", + default=default_chain_endpoint, + type=str, + help="""The subtensor endpoint flag. If set, overrides the --network flag.""", + ) + parser.add_argument( + f"--{prefix_str}subtensor._mock", + default=False, + type=bool, + help="""If true, uses a mocked connection to the chain.""", + ) + + except argparse.ArgumentError: + # re-parsing arguments. + pass + + @staticmethod + def determine_chain_endpoint_and_network( + network: str, + ) -> tuple[Optional[str], Optional[str]]: + """Determines the chain endpoint and network from the passed network or chain_endpoint. + + Arguments: + network (str): The network flag. The choices are: ``finney`` (main network), ``archive`` (archive network + +300 blocks), ``local`` (local running network), ``test`` (test network). + + Returns: + tuple[Optional[str], Optional[str]]: The network and chain endpoint flag. If passed, overrides the + ``network`` argument. + """ + + if network is None: + return None, None + if network in settings.NETWORKS: + return network, settings.NETWORK_MAP[network] + + substrings_map = { + "entrypoint-finney.opentensor.ai": ("finney", settings.FINNEY_ENTRYPOINT), + "test.finney.opentensor.ai": ("test", settings.FINNEY_TEST_ENTRYPOINT), + "archive.chain.opentensor.ai": ("archive", settings.ARCHIVE_ENTRYPOINT), + "subvortex": ("subvortex", settings.SUBVORTEX_ENTRYPOINT), + "127.0.0.1": ("local", settings.LOCAL_ENTRYPOINT), + "localhost": ("local", settings.LOCAL_ENTRYPOINT), + } + + for substring, result in substrings_map.items(): + if substring in network: + return result + + return "unknown", network diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 2fecd1fe4e..318b1a4e82 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -1,4 +1,3 @@ -import argparse import asyncio import copy import ssl @@ -15,7 +14,7 @@ from scalecodec.base import RuntimeConfiguration from scalecodec.type_registry import load_type_registry_preset -from bittensor.core import settings +from bittensor.core import SubtensorMixin from bittensor.core.chain_data import ( DelegateInfo, StakeInfo, @@ -68,9 +67,7 @@ ss58_to_vec_u8, torch, u16_normalized_float, - event_loop_is_running, ) -from bittensor.utils import networking from bittensor.utils.balance import Balance from bittensor.utils.btlogging import logging from bittensor.utils.delegates_details import DelegatesDetails @@ -110,7 +107,7 @@ def _decode_hex_identity_dict(info_dictionary: dict[str, Any]) -> dict[str, Any] return info_dictionary -class AsyncSubtensor: +class AsyncSubtensor(SubtensorMixin): """Thin layer for interacting with Substrate Interface. Mostly a collection of frequently-used calls.""" def __init__( @@ -128,7 +125,6 @@ def __init__( config (Optional[Config]): Configuration object for the AsyncSubtensor instance. _mock: Whether this is a mock instance. Mainly just for use in testing. log_verbose (bool): Enables or disables verbose logging. - event_loop (Optional[asyncio.AbstractEventLoop]): Custom asyncio event loop. Raises: Any exceptions raised during the setup, configuration, or connection process. @@ -161,197 +157,6 @@ def __init__( f"Connected to {self.network} network and {self.chain_endpoint}." ) - def __str__(self): - return f"Network: {self.network}, Chain: {self.chain_endpoint}" - - def __repr__(self): - return self.__str__() - - def __del__(self): - if loop := event_loop_is_running(): - loop.create_task(self.close()) - - def _check_and_log_network_settings(self): - if self.network == settings.NETWORKS[3]: # local - logging.warning( - ":warning: Verify your local subtensor is running on port [blue]9944[/blue]." - ) - - if ( - self.network == "finney" - or self.chain_endpoint == settings.FINNEY_ENTRYPOINT - ) and self.log_verbose: - logging.info( - f"You are connecting to {self.network} network with endpoint {self.chain_endpoint}." - ) - logging.debug( - "We strongly encourage running a local subtensor node whenever possible. " - "This increases decentralization and resilience of the network." - ) - # TODO: remove or apply this warning as updated default endpoint? - logging.debug( - "In a future release, local subtensor will become the default endpoint. " - "To get ahead of this change, please run a local subtensor node and point to it." - ) - - @staticmethod - def config() -> "Config": - """ - Creates and returns a Bittensor configuration object. - - Returns: - config (bittensor.core.config.Config): A Bittensor configuration object configured with arguments added by - the `subtensor.add_args` method. - """ - parser = argparse.ArgumentParser() - AsyncSubtensor.add_args(parser) - return Config(parser) - - @staticmethod - def setup_config(network: Optional[str], config: "Config"): - """ - Sets up and returns the configuration for the Subtensor network and endpoint. - - This method determines the appropriate network and chain endpoint based on the provided network string or - configuration object. It evaluates the network and endpoint in the following order of precedence: - 1. Provided network string. - 2. Configured chain endpoint in the `config` object. - 3. Configured network in the `config` object. - 4. Default chain endpoint. - 5. Default network. - - Arguments: - network (Optional[str]): The name of the Subtensor network. If None, the network and endpoint will be - determined from the `config` object. - config (bittensor.core.config.Config): The configuration object containing the network and chain endpoint - settings. - - Returns: - tuple: A tuple containing the formatted WebSocket endpoint URL and the evaluated network name. - """ - if network is None: - candidates = [ - ( - config.is_set("subtensor.chain_endpoint"), - config.subtensor.chain_endpoint, - ), - (config.is_set("subtensor.network"), config.subtensor.network), - ( - config.subtensor.get("chain_endpoint"), - config.subtensor.chain_endpoint, - ), - (config.subtensor.get("network"), config.subtensor.network), - ] - for check, config_network in candidates: - if check: - network = config_network - - evaluated_network, evaluated_endpoint = ( - AsyncSubtensor.determine_chain_endpoint_and_network(network) - ) - - return networking.get_formatted_ws_endpoint_url( - evaluated_endpoint - ), evaluated_network - - @classmethod - def help(cls): - """Print help to stdout.""" - parser = argparse.ArgumentParser() - cls.add_args(parser) - print(cls.__new__.__doc__) - parser.print_help() - - @classmethod - def add_args(cls, parser: "argparse.ArgumentParser", prefix: Optional[str] = None): - """ - Adds command-line arguments to the provided ArgumentParser for configuring the Subtensor settings. - - Arguments: - parser (argparse.ArgumentParser): The ArgumentParser object to which the Subtensor arguments will be added. - prefix (Optional[str]): An optional prefix for the argument names. If provided, the prefix is prepended to - each argument name. - - Arguments added: - --subtensor.network: The Subtensor network flag. Possible values are 'finney', 'test', 'archive', and - 'local'. Overrides the chain endpoint if set. - --subtensor.chain_endpoint: The Subtensor chain endpoint flag. If set, it overrides the network flag. - --subtensor._mock: If true, uses a mocked connection to the chain. - - Example: - parser = argparse.ArgumentParser() - Subtensor.add_args(parser) - """ - prefix_str = "" if prefix is None else f"{prefix}." - try: - default_network = settings.DEFAULT_NETWORK - default_chain_endpoint = settings.FINNEY_ENTRYPOINT - - parser.add_argument( - f"--{prefix_str}subtensor.network", - default=default_network, - type=str, - help="""The subtensor network flag. The likely choices are: - -- finney (main network) - -- test (test network) - -- archive (archive network +300 blocks) - -- local (local running network) - If this option is set it overloads subtensor.chain_endpoint with - an entry point node from that network. - """, - ) - parser.add_argument( - f"--{prefix_str}subtensor.chain_endpoint", - default=default_chain_endpoint, - type=str, - help="""The subtensor endpoint flag. If set, overrides the --network flag.""", - ) - parser.add_argument( - f"--{prefix_str}subtensor._mock", - default=False, - type=bool, - help="""If true, uses a mocked connection to the chain.""", - ) - - except argparse.ArgumentError: - # re-parsing arguments. - pass - - @staticmethod - def determine_chain_endpoint_and_network( - network: str, - ) -> tuple[Optional[str], Optional[str]]: - """Determines the chain endpoint and network from the passed network or chain_endpoint. - - Arguments: - network (str): The network flag. The choices are: ``finney`` (main network), ``archive`` (archive network - +300 blocks), ``local`` (local running network), ``test`` (test network). - - Returns: - tuple[Optional[str], Optional[str]]: The network and chain endpoint flag. If passed, overrides the - ``network`` argument. - """ - - if network is None: - return None, None - if network in settings.NETWORKS: - return network, settings.NETWORK_MAP[network] - - substrings_map = { - "entrypoint-finney.opentensor.ai": ("finney", settings.FINNEY_ENTRYPOINT), - "test.finney.opentensor.ai": ("test", settings.FINNEY_TEST_ENTRYPOINT), - "archive.chain.opentensor.ai": ("archive", settings.ARCHIVE_ENTRYPOINT), - "subvortex": ("subvortex", settings.SUBVORTEX_ENTRYPOINT), - "127.0.0.1": ("local", settings.LOCAL_ENTRYPOINT), - "localhost": ("local", settings.LOCAL_ENTRYPOINT), - } - - for substring, result in substrings_map.items(): - if substring in network: - return result - - return "unknown", network - async def close(self): """Close the connection.""" if self.substrate: diff --git a/bittensor/core/metagraph.py b/bittensor/core/metagraph.py index 4455ee9754..ce696c536e 100644 --- a/bittensor/core/metagraph.py +++ b/bittensor/core/metagraph.py @@ -8,7 +8,6 @@ from os.path import join from typing import Optional, Union -from async_substrate_interface.utils import EventLoopManager import numpy as np from numpy.typing import NDArray @@ -1510,10 +1509,6 @@ def __init__( sync=sync, subtensor=subtensor.async_subtensor if subtensor else None, ) - if self.subtensor: - self.event_loop_mgr = self.subtensor.event_loop_mgr - else: - self.event_loop_mgr = EventLoopManager() if sync: if self.subtensor: self.subtensor.event_loop_mgr.run(self._async_metagraph.sync()) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 9b804c8db6..de9c9219ab 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1,19 +1,21 @@ -import warnings +import copy from functools import lru_cache from typing import TYPE_CHECKING, Any, Iterable, Optional, Union import numpy as np -from async_substrate_interface import SubstrateInterface -from async_substrate_interface.utils import EventLoopManager +from async_substrate_interface.sync_substrate import SubstrateInterface from numpy.typing import NDArray +import scalecodec +from scalecodec.base import RuntimeConfiguration +from scalecodec.type_registry import load_type_registry_preset -from bittensor.core.async_subtensor import AsyncSubtensor +from bittensor.core import SubtensorMixin +from bittensor.core.chain_data import custom_rpc_type_registry from bittensor.core.metagraph import Metagraph -from bittensor.core.settings import version_as_int -from bittensor.utils import ( - torch, - event_loop_is_running, -) +from bittensor.core.settings import version_as_int, SS58_FORMAT, TYPE_REGISTRY +from bittensor.core.types import ParamWithTypes +from bittensor.utils import torch +from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -28,29 +30,16 @@ from bittensor.core.chain_data.subnet_info import SubnetInfo from bittensor.utils.balance import Balance from bittensor.utils import Certificate - from async_substrate_interface import QueryMapResult + from async_substrate_interface.sync_substrate import QueryMapResult from bittensor.utils.delegates_details import DelegatesDetails from scalecodec.types import ScaleType -class Subtensor: +class Subtensor(SubtensorMixin): """ - Represents a synchronous interface for `bittensor.core.async_subtensor.AsyncSubtensor`. - - If you want to get the description of any method from the `bittensor.core.subtensor.Subtensor` class, then simply - get the corresponding method from the `bittensor.core.async_subtensor.AsyncSubtensor` class. + TODO docstring """ - # get static methods from AsyncSubtensor - config = AsyncSubtensor.config - setup_config = AsyncSubtensor.setup_config - help = AsyncSubtensor.help - add_args = AsyncSubtensor.add_args - determine_chain_endpoint_and_network = ( - AsyncSubtensor.determine_chain_endpoint_and_network - ) - event_loop_mgr: EventLoopManager - def __init__( self, network: Optional[str] = None, @@ -58,57 +47,76 @@ def __init__( _mock: bool = False, log_verbose: bool = False, ): - if event_loop_is_running(): - warnings.warn( - "You are calling this from an already running event loop. Some features may not work correctly. You " - "should instead use `AsyncSubtensor`." - ) - self.event_loop_mgr = EventLoopManager() - self.network = network - self._config = config + """ + Initializes an instance of the AsyncSubtensor class. + + Arguments: + network (str): The network name or type to connect to. + config (Optional[Config]): Configuration object for the AsyncSubtensor instance. + _mock: Whether this is a mock instance. Mainly just for use in testing. + log_verbose (bool): Enables or disables verbose logging. + + Raises: + Any exceptions raised during the setup, configuration, or connection process. + """ + if config is None: + config = self.config() + self._config = copy.deepcopy(config) + self.chain_endpoint, self.network = self.setup_config(network, self._config) + self._mock = _mock + self.log_verbose = log_verbose - self.async_subtensor = AsyncSubtensor( - network=network, - config=config, - log_verbose=log_verbose, - _mock=_mock, + self._check_and_log_network_settings() + + logging.debug( + f"Connecting to ..." ) self.substrate = SubstrateInterface( - url=self.async_subtensor.chain_endpoint, + url=self.chain_endpoint, + ss58_format=SS58_FORMAT, + type_registry=TYPE_REGISTRY, + use_remote_preset=True, + chain_name="Bittensor", _mock=_mock, - substrate=self.async_subtensor.substrate, - event_loop_manager=self.event_loop_mgr, ) - self.chain_endpoint = self.async_subtensor.chain_endpoint - - def __str__(self): - return self.async_subtensor.__str__() - - def __repr__(self): - return self.async_subtensor.__repr__() - - def __del__(self): - try: - self.event_loop_mgr.stop() - except AttributeError: - pass - - def execute_coroutine(self, coroutine) -> Any: - return self.event_loop_mgr.run(coroutine) + if self.log_verbose: + logging.info( + f"Connected to {self.network} network and {self.chain_endpoint}." + ) def close(self): - self.execute_coroutine(self.async_subtensor.close()) - self.event_loop_mgr.stop() + """ + Does nothing. Exists for backwards compatibility purposes. + """ + pass # Subtensor queries =========================================================================================== def query_constant( self, module_name: str, constant_name: str, block: Optional[int] = None ) -> Optional["ScaleType"]: - return self.execute_coroutine( - self.async_subtensor.query_constant( - module_name=module_name, constant_name=constant_name, block=block - ) + """ + Retrieves a constant from the specified module on the Bittensor blockchain. This function is used to access + fixed parameters or values defined within the blockchain's modules, which are essential for understanding + the network's configuration and rules. + + Args: + module_name: The name of the module containing the constant. + constant_name: The name of the constant to retrieve. + block: The blockchain block number at which to query the constant. + + Returns: + Optional[scalecodec.ScaleType]: The value of the constant if found, `None` otherwise. + + Constants queried through this function can include critical network parameters such as inflation rates, + consensus rules, or validation thresholds, providing a deeper understanding of the Bittensor network's + operational parameters. + """ + return self.substrate.get_constant( + module_name=module_name, + constant_name=constant_name, + block_hash=self.determine_block_hash(block), ) def query_map( @@ -118,19 +126,54 @@ def query_map( block: Optional[int] = None, params: Optional[list] = None, ) -> "QueryMapResult": - return self.execute_coroutine( - self.async_subtensor.query_map( - module=module, name=name, block=block, params=params - ) - ) + """ + Queries map storage from any module on the Bittensor blockchain. This function retrieves data structures that + represent key-value mappings, essential for accessing complex and structured data within the blockchain + modules. + + Args: + module: The name of the module from which to query the map storage. + name: The specific storage function within the module to query. + block: The blockchain block number at which to perform the query. + params: Parameters to be passed to the query. + + Returns: + result: A data structure representing the map storage if found, `None` otherwise. + + This function is particularly useful for retrieving detailed and structured data from various blockchain + modules, offering insights into the network's state and the relationships between its different components. + """ + result = self.substrate.query_map( + module=module, + storage_function=name, + params=params, + block_hash=self.determine_block_hash(block=block), + ) + return getattr(result, "value", None) def query_map_subtensor( self, name: str, block: Optional[int] = None, params: Optional[list] = None ) -> "QueryMapResult": - return self.execute_coroutine( - self.async_subtensor.query_map_subtensor( - name=name, block=block, params=params - ) + """ + Queries map storage from the Subtensor module on the Bittensor blockchain. This function is designed to retrieve + a map-like data structure, which can include various neuron-specific details or network-wide attributes. + + Args: + name: The name of the map storage function to query. + block: The blockchain block number at which to perform the query. + params: A list of parameters to pass to the query function. + + Returns: + An object containing the map-like data structure, or `None` if not found. + + This function is particularly useful for analyzing and understanding complex network structures and + relationships within the Bittensor ecosystem, such as interneuronal connections and stake distributions. + """ + return self.substrate.query_map( + module="SubtensorModule", + storage_function=name, + params=params, + block_hash=self.determine_block_hash(block), ) def query_module( @@ -140,13 +183,28 @@ def query_module( block: Optional[int] = None, params: Optional[list] = None, ) -> "ScaleType": - return self.execute_coroutine( - self.async_subtensor.query_module( - module=module, - name=name, - block=block, - params=params, - ) + """ + Queries any module storage on the Bittensor blockchain with the specified parameters and block number. This + function is a generic query interface that allows for flexible and diverse data retrieval from various + blockchain modules. + + Args: + module (str): The name of the module from which to query data. + name (str): The name of the storage function within the module. + block (Optional[int]): The blockchain block number at which to perform the query. + params (Optional[list[object]]): A list of parameters to pass to the query function. + + Returns: + An object containing the requested data if found, `None` otherwise. + + This versatile query function is key to accessing a wide range of data and insights from different parts of the + Bittensor blockchain, enhancing the understanding and analysis of the network's state and dynamics. + """ + return self.substrate.query( + module=module, + storage_function=name, + params=params, + block_hash=self.determine_block_hash(block), ) def query_runtime_api( @@ -156,27 +214,103 @@ def query_runtime_api( params: Optional[Union[list[int], dict[str, int]]] = None, block: Optional[int] = None, ) -> Optional[str]: - return self.execute_coroutine( - coroutine=self.async_subtensor.query_runtime_api( - runtime_api=runtime_api, - method=method, - params=params, - block=block, - ) + """ + Queries the runtime API of the Bittensor blockchain, providing a way to interact with the underlying runtime and + retrieve data encoded in Scale Bytes format. This function is essential for advanced users who need to + interact with specific runtime methods and decode complex data types. + + Args: + runtime_api: The name of the runtime API to query. + method: The specific method within the runtime API to call. + params: The parameters to pass to the method call. + block: the block number for this query. + + Returns: + The Scale Bytes encoded result from the runtime API call, or `None` if the call fails. + + This function enables access to the deeper layers of the Bittensor blockchain, allowing for detailed and + specific interactions with the network's runtime environment. + """ + block_hash = self.determine_block_hash(block) + + call_definition = TYPE_REGISTRY["runtime_api"][runtime_api]["methods"][method] + + data = ( + "0x" + if params is None + else self.encode_params(call_definition=call_definition, params=params) + ) + api_method = f"{runtime_api}_{method}" + + json_result = self.substrate.rpc_request( + method="state_call", + params=[api_method, data, block_hash] if block_hash else [api_method, data], ) + if json_result is None: + return None + + return_type = call_definition["type"] + + as_scale_bytes = scalecodec.ScaleBytes(json_result["result"]) # type: ignore + + rpc_runtime_config = RuntimeConfiguration() + rpc_runtime_config.update_type_registry(load_type_registry_preset("legacy")) + rpc_runtime_config.update_type_registry(custom_rpc_type_registry) + + obj = rpc_runtime_config.create_scale_object(return_type, as_scale_bytes) + if obj.data.to_hex() == "0x0400": # RPC returned None result + return None + + return obj.decode() + def query_subtensor( self, name: str, block: Optional[int] = None, params: Optional[list] = None ) -> "ScaleType": - return self.execute_coroutine( - self.async_subtensor.query_subtensor(name=name, block=block, params=params) + """ + Queries named storage from the Subtensor module on the Bittensor blockchain. This function is used to retrieve + specific data or parameters from the blockchain, such as stake, rank, or other neuron-specific attributes. + + Args: + name: The name of the storage function to query. + block: The blockchain block number at which to perform the query. + params: A list of parameters to pass to the query function. + + Returns: + query_response (scalecodec.ScaleType): An object containing the requested data. + + This query function is essential for accessing detailed information about the network and its neurons, providing + valuable insights into the state and dynamics of the Bittensor ecosystem. + """ + return self.substrate.query( + module="SubtensorModule", + storage_function=name, + params=params, + block_hash=self.determine_block_hash(block), ) def state_call( self, method: str, data: str, block: Optional[int] = None ) -> dict[Any, Any]: - return self.execute_coroutine( - self.async_subtensor.state_call(method=method, data=data, block=block) + """ + Makes a state call to the Bittensor blockchain, allowing for direct queries of the blockchain's state. This + function is typically used for advanced queries that require specific method calls and data inputs. + + Args: + method: The method name for the state call. + data: The data to be passed to the method. + block: The blockchain block number at which to perform the state call. + + Returns: + result (dict[Any, Any]): The result of the rpc call. + + The state call function provides a more direct and flexible way of querying blockchain data, useful for specific + use cases where standard queries are insufficient. + """ + block_hash = self.determine_block_hash(block) + return self.substrate.rpc_request( + method="state_call", + params=[method, data, block_hash] if block_hash else [method, data], ) # Common subtensor calls =========================================================================================== @@ -186,20 +320,68 @@ def block(self) -> int: return self.get_current_block() def blocks_since_last_update(self, netuid: int, uid: int) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.blocks_since_last_update(netuid=netuid, uid=uid) - ) + """ + Returns the number of blocks since the last update for a specific UID in the subnetwork. + + Arguments: + netuid (int): The unique identifier of the subnetwork. + uid (int): The unique identifier of the neuron. + + Returns: + Optional[int]: The number of blocks since the last update, or ``None`` if the subnetwork or UID does not + exist. + """ + call = self.get_hyperparameter(param_name="LastUpdate", netuid=netuid) + return None if call is None else (self.get_current_block() - int(call[uid])) def bonds( self, netuid: int, block: Optional[int] = None ) -> list[tuple[int, list[tuple[int, int]]]]: - return self.execute_coroutine( - self.async_subtensor.bonds(netuid=netuid, block=block), - ) + """ + Retrieves the bond distribution set by neurons within a specific subnet of the Bittensor network. + Bonds represent the investments or commitments made by neurons in one another, indicating a level of trust + and perceived value. This bonding mechanism is integral to the network's market-based approach to + measuring and rewarding machine intelligence. + + Args: + netuid: The network UID of the subnet to query. + block: the block number for this query. + + Returns: + List of tuples mapping each neuron's UID to its bonds with other neurons. + + Understanding bond distributions is crucial for analyzing the trust dynamics and market behavior within the + subnet. It reflects how neurons recognize and invest in each other's intelligence and contributions, + supporting diverse and niche systems within the Bittensor ecosystem. + """ + b_map_encoded = self.substrate.query_map( + module="SubtensorModule", + storage_function="Bonds", + params=[netuid], + block_hash=self.determine_block_hash(block), + ) + b_map = [] + for uid, b in b_map_encoded: + b_map.append((uid, b.value)) + + return b_map def commit(self, wallet, netuid: int, data: str) -> bool: - return self.execute_coroutine( - self.async_subtensor.commit(wallet=wallet, netuid=netuid, data=data) + """ + Commits arbitrary data to the Bittensor network by publishing metadata. + + Arguments: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the data. + netuid (int): The unique identifier of the subnetwork. + data (str): The data to be committed to the network. + """ + # TODO add + return publish_metadata( + subtensor=self, + wallet=wallet, + netuid=netuid, + data_type=f"Raw{len(data)}", + data=data.encode(), ) def commit_reveal_enabled( @@ -249,6 +431,60 @@ def get_block_hash(self, block: Optional[int] = None) -> str: coroutine=self.async_subtensor.get_block_hash(block=block), ) + def determine_block_hash(self, block: Optional[int]) -> Optional[str]: + if block is None: + return None + else: + return self.get_block_hash(block=block) + + def encode_params( + self, + call_definition: dict[str, list["ParamWithTypes"]], + params: Union[list[Any], dict[str, Any]], + ) -> str: + """Returns a hex encoded string of the params using their types.""" + param_data = scalecodec.ScaleBytes(b"") + + for i, param in enumerate(call_definition["params"]): + scale_obj = self.substrate.create_scale_object(param["type"]) + if isinstance(params, list): + param_data += scale_obj.encode(params[i]) + else: + if param["name"] not in params: + raise ValueError(f"Missing param {param['name']} in params dict.") + + param_data += scale_obj.encode(params[param["name"]]) + + return param_data.to_hex() + + def get_hyperparameter( + self, param_name: str, netuid: int, block: Optional[int] = None + ) -> Optional[Any]: + """ + Retrieves a specified hyperparameter for a specific subnet. + + Arguments: + param_name (str): The name of the hyperparameter to retrieve. + netuid (int): The unique identifier of the subnet. + block: the block number at which to retrieve the hyperparameter. + + Returns: + The value of the specified hyperparameter if the subnet exists, or None + """ + block_hash = self.determine_block_hash(block) + if not self.subnet_exists(netuid, block=block): + logging.error(f"subnet {netuid} does not exist") + return None + + result = self.substrate.query( + module="SubtensorModule", + storage_function=param_name, + params=[netuid], + block_hash=block_hash, + ) + + return getattr(result, "value", result) + def get_children( self, hotkey: str, netuid: int, block: Optional[int] = None ) -> tuple[bool, list, str]: From 82fa80d538b704d51b24664e65a86b69a0de3e0c Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Sat, 18 Jan 2025 20:24:59 +0200 Subject: [PATCH 08/52] All non-extrinsics methods ported. --- bittensor/core/async_subtensor.py | 49 +- bittensor/core/subtensor.py | 2029 ++++++++++++++++++---- bittensor/utils/__init__.py | 27 + requirements/prod.txt | 1 + tests/unit_tests/test_async_subtensor.py | 2 +- 5 files changed, 1776 insertions(+), 332 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 318b1a4e82..a7bdeeb076 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -67,6 +67,7 @@ ss58_to_vec_u8, torch, u16_normalized_float, + _decode_hex_identity_dict, ) from bittensor.utils.balance import Balance from bittensor.utils.btlogging import logging @@ -81,32 +82,6 @@ from async_substrate_interface import QueryMapResult -def _decode_hex_identity_dict(info_dictionary: dict[str, Any]) -> dict[str, Any]: - """Decodes a dictionary of hexadecimal identities.""" - for k, v in info_dictionary.items(): - if isinstance(v, dict): - item = next(iter(v.values())) - else: - item = v - if isinstance(item, tuple) and item: - if len(item) > 1: - try: - info_dictionary[k] = ( - bytes(item).hex(sep=" ", bytes_per_sep=2).upper() - ) - except UnicodeDecodeError: - logging.error(f"Could not decode: {k}: {item}.") - else: - try: - info_dictionary[k] = bytes(item[0]).decode("utf-8") - except UnicodeDecodeError: - logging.error(f"Could not decode: {k}: {item}.") - else: - info_dictionary[k] = item - - return info_dictionary - - class AsyncSubtensor(SubtensorMixin): """Thin layer for interacting with Substrate Interface. Mostly a collection of frequently-used calls.""" @@ -464,6 +439,7 @@ async def query_runtime_api( This function enables access to the deeper layers of the Bittensor blockchain, allowing for detailed and specific interactions with the network's runtime environment. """ + # TODO why doesn't this just use SubstrateInterface.runtime_call ? block_hash = await self.determine_block_hash(block, block_hash, reuse_block) call_definition = TYPE_REGISTRY["runtime_api"][runtime_api]["methods"][method] @@ -1425,7 +1401,10 @@ async def get_neuron_for_pubkey_and_subnet( params = [netuid, uid.value] json_body = await self.substrate.rpc_request( - method="neuronInfo_getNeuron", params=params, reuse_block_hash=reuse_block + method="neuronInfo_getNeuron", + params=params, + block_hash=block_hash, + reuse_block_hash=reuse_block, ) if not (result := json_body.get("result", None)): @@ -2559,7 +2538,9 @@ async def weights( block_hash=block_hash, reuse_block_hash=reuse_block, ) - w_map = [(uid, w.value or []) async for uid, w in w_map_encoded] + w_map = [] + async for uid, w in w_map_encoded: + w_map.append((uid, w.value)) return w_map @@ -2784,7 +2765,8 @@ async def commit_weights( message = "No attempt made. Perhaps it is too soon to commit weights!" logging.info( - f"Committing weights with params: netuid={netuid}, uids={uids}, weights={weights}, version_key={version_key}" + f"Committing weights with params: netuid={netuid}, uids={uids}, weights={weights}, " + f"version_key={version_key}" ) # Generate the hash of the weights @@ -2977,9 +2959,6 @@ async def root_register( except TypeError as e: logging.error(f"Unable to retrieve current recycle. {e}") return False - except KeyError: - logging.error("Unable to retrieve current balance.") - return False current_recycle = Balance.from_rao(int(recycle_call)) @@ -3187,7 +3166,7 @@ async def serve_axon( async def transfer( self, wallet: "Wallet", - destination: str, + dest: str, amount: Union["Balance", float], transfer_all: bool = False, wait_for_inclusion: bool = True, @@ -3199,7 +3178,7 @@ async def transfer( Arguments: wallet (bittensor_wallet.Wallet): Source wallet for the transfer. - destination (str): Destination address for the transfer. + dest (str): Destination address for the transfer. amount (float): Amount of tokens to transfer. transfer_all (bool): Flag to transfer all tokens. Default is ``False``. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``True``. @@ -3216,7 +3195,7 @@ async def transfer( return await transfer_extrinsic( subtensor=self, wallet=wallet, - destination=destination, + destination=dest, amount=amount, transfer_all=transfer_all, wait_for_inclusion=wait_for_inclusion, diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index de9c9219ab..e4a7278380 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -3,19 +3,40 @@ from typing import TYPE_CHECKING, Any, Iterable, Optional, Union import numpy as np +import ujson +from async_substrate_interface.errors import SubstrateRequestException from async_substrate_interface.sync_substrate import SubstrateInterface +from async_substrate_interface.utils import hex_to_bytes from numpy.typing import NDArray +import requests import scalecodec from scalecodec.base import RuntimeConfiguration from scalecodec.type_registry import load_type_registry_preset from bittensor.core import SubtensorMixin -from bittensor.core.chain_data import custom_rpc_type_registry +from bittensor.core.chain_data import ( + custom_rpc_type_registry, + decode_account_id, + WeightCommitInfo, +) from bittensor.core.metagraph import Metagraph -from bittensor.core.settings import version_as_int, SS58_FORMAT, TYPE_REGISTRY +from bittensor.core.settings import ( + version_as_int, + SS58_FORMAT, + TYPE_REGISTRY, + DELEGATES_DETAILS_URL, +) from bittensor.core.types import ParamWithTypes -from bittensor.utils import torch +from bittensor.utils import ( + torch, + format_error_message, + ss58_to_vec_u8, + decode_hex_identity_dict, + u16_normalized_float, + _decode_hex_identity_dict, +) from bittensor.utils.btlogging import logging +from bittensor.utils.weight_utils import generate_weight_hash if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -231,6 +252,7 @@ def query_runtime_api( This function enables access to the deeper layers of the Bittensor blockchain, allowing for detailed and specific interactions with the network's runtime environment. """ + # TODO why doesn't this just use SubstrateInterface.runtime_call ? block_hash = self.determine_block_hash(block) call_definition = TYPE_REGISTRY["runtime_api"][runtime_api]["methods"][method] @@ -387,49 +409,180 @@ def commit(self, wallet, netuid: int, data: str) -> bool: def commit_reveal_enabled( self, netuid: int, block: Optional[int] = None ) -> Optional[bool]: - return self.execute_coroutine( - self.async_subtensor.commit_reveal_enabled(netuid=netuid, block=block) + """ + Check if commit-reveal mechanism is enabled for a given network at a specific block. + + Arguments: + netuid: The network identifier for which to check the commit-reveal mechanism. + block: The block number to query. + + Returns: + Returns the integer value of the hyperparameter if available; otherwise, returns None. + """ + call = self.get_hyperparameter( + param_name="CommitRevealWeightsEnabled", block=block, netuid=netuid ) + return True if call is True else False def difficulty(self, netuid: int, block: Optional[int] = None) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.difficulty(netuid=netuid, block=block), + """ + Retrieves the 'Difficulty' hyperparameter for a specified subnet in the Bittensor network. + + This parameter is instrumental in determining the computational challenge required for neurons to participate in + consensus and validation processes. + + Arguments: + netuid: The unique identifier of the subnet. + block: The blockchain block number for the query. + + Returns: + Optional[int]: The value of the 'Difficulty' hyperparameter if the subnet exists, ``None`` otherwise. + + The 'Difficulty' parameter directly impacts the network's security and integrity by setting the computational + effort required for validating transactions and participating in the network's consensus mechanism. + """ + call = self.get_hyperparameter( + param_name="Difficulty", netuid=netuid, block=block ) + if call is None: + return None + return int(call) def does_hotkey_exist(self, hotkey_ss58: str, block: Optional[int] = None) -> bool: - return self.execute_coroutine( - self.async_subtensor.does_hotkey_exist(hotkey_ss58=hotkey_ss58, block=block) + """ + Returns true if the hotkey is known by the chain and there are accounts. + + Args: + hotkey_ss58: The SS58 address of the hotkey. + block: the block number for this query. + + Returns: + `True` if the hotkey is known by the chain and there are accounts, `False` otherwise. + """ + _result = self.substrate.query( + module="SubtensorModule", + storage_function="Owner", + params=[hotkey_ss58], + block_hash=self.determine_block_hash(block), ) + result = decode_account_id(_result.value[0]) + return_val = ( + False + if result is None + else result != "5C4hrfjw9DjXZTzV3MwzrrAr9P1MJhSrvWGWqi1eSuyUpnhM" + ) + return return_val def get_all_subnets_info(self, block: Optional[int] = None) -> list["SubnetInfo"]: - return self.execute_coroutine( - self.async_subtensor.get_all_subnets_info(block=block), + """ + Retrieves detailed information about all subnets within the Bittensor network. This function provides + comprehensive data on each subnet, including its characteristics and operational parameters. + + Arguments: + block: The blockchain block number for the query. + + Returns: + list[SubnetInfo]: A list of SubnetInfo objects, each containing detailed information about a subnet. + + Gaining insights into the subnets' details assists in understanding the network's composition, the roles of + different subnets, and their unique features. + """ + hex_bytes_result = self.query_runtime_api( + "SubnetInfoRuntimeApi", "get_subnets_info", params=[], block=block ) + if not hex_bytes_result: + return [] + else: + return SubnetInfo.list_from_vec_u8(hex_to_bytes(hex_bytes_result)) def get_balance(self, address: str, block: Optional[int] = None) -> "Balance": - return self.execute_coroutine( - self.async_subtensor.get_balance(address, block=block), + """ + Retrieves the balance for given coldkey. + + Arguments: + address (str): coldkey address. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Balance object. + """ + balance = self.substrate.query( + module="System", + storage_function="Account", + params=[address], + block_hash=self.determine_block_hash(block), ) + return Balance(balance["data"]["free"]) def get_balances( self, *addresses: str, block: Optional[int] = None, ) -> dict[str, "Balance"]: - return self.execute_coroutine( - self.async_subtensor.get_balances(*addresses, block=block), - ) + """ + Retrieves the balance for given coldkey(s) + + Arguments: + addresses (str): coldkey addresses(s). + block (Optional[int]): The blockchain block number for the query. + + Returns: + Dict of {address: Balance objects}. + """ + if not (block_hash := self.determine_block_hash(block)): + block_hash = self.substrate.get_chain_head() + calls = [ + ( + self.substrate.create_storage_key( + "System", "Account", [address], block_hash=block_hash + ) + ) + for address in addresses + ] + batch_call = self.substrate.query_multi(calls, block_hash=block_hash) + results = {} + for item in batch_call: + value = item[1] or {"data": {"free": 0}} + results.update({item[0].params[0]: Balance(value["data"]["free"])}) + return results def get_current_block(self) -> int: - return self.execute_coroutine( - coroutine=self.async_subtensor.get_current_block(), - ) + """ + Returns the current block number on the Bittensor blockchain. This function provides the latest block number, + indicating the most recent state of the blockchain. + + Returns: + int: The current chain block number. + + Knowing the current block number is essential for querying real-time data and performing time-sensitive + operations on the blockchain. It serves as a reference point for network activities and data + synchronization. + """ + return self.substrate.get_block_number(None) @lru_cache(maxsize=128) - def get_block_hash(self, block: Optional[int] = None) -> str: - return self.execute_coroutine( - coroutine=self.async_subtensor.get_block_hash(block=block), - ) + def _get_block_hash(self, block_id: int): + return self.substrate.get_block_hash(block_id) + + def get_block_hash(self, block: Optional[int] = None): + """ + Retrieves the hash of a specific block on the Bittensor blockchain. The block hash is a unique identifier + representing the cryptographic hash of the block's content, ensuring its integrity and immutability. + + Arguments: + block (int): The block number for which the hash is to be retrieved. + + Returns: + str: The cryptographic hash of the specified block. + + The block hash is a fundamental aspect of blockchain technology, providing a secure reference to each block's + data. It is crucial for verifying transactions, ensuring data consistency, and maintaining the + trustworthiness of the blockchain. + """ + if block: + return self._get_block_hash(block) + else: + return self.substrate.get_chain_head() def determine_block_hash(self, block: Optional[int]) -> Optional[str]: if block is None: @@ -488,216 +641,798 @@ def get_hyperparameter( def get_children( self, hotkey: str, netuid: int, block: Optional[int] = None ) -> tuple[bool, list, str]: - return self.execute_coroutine( - self.async_subtensor.get_children( - hotkey=hotkey, netuid=netuid, block=block - ), - ) + """ + This method retrieves the children of a given hotkey and netuid. It queries the SubtensorModule's ChildKeys + storage function to get the children and formats them before returning as a tuple. + + Arguments: + hotkey (str): The hotkey value. + netuid (int): The netuid value. + block (Optional[int]): The block number for which the children are to be retrieved. + + Returns: + A tuple containing a boolean indicating success or failure, a list of formatted children, and an error + message (if applicable) + """ + try: + children = self.substrate.query( + module="SubtensorModule", + storage_function="ChildKeys", + params=[hotkey, netuid], + block_hash=self.determine_block_hash(block), + ) + if children: + formatted_children = [] + for proportion, child in children.value: + # Convert U64 to int + formatted_child = decode_account_id(child[0]) + int_proportion = int(proportion) + formatted_children.append((int_proportion, formatted_child)) + return True, formatted_children, "" + else: + return True, [], "" + except SubstrateRequestException as e: + return False, [], format_error_message(e) def get_commitment(self, netuid: int, uid: int, block: Optional[int] = None) -> str: - return self.execute_coroutine( - self.async_subtensor.get_commitment(netuid=netuid, uid=uid, block=block), - ) + """ + Retrieves the on-chain commitment for a specific neuron in the Bittensor network. + + Arguments: + netuid (int): The unique identifier of the subnetwork. + uid (int): The unique identifier of the neuron. + block (Optional[int]): The block number to retrieve the commitment from. If None, the latest block is used. + Default is ``None``. + + Returns: + str: The commitment data as a string. + """ + metagraph = self.metagraph(netuid) + try: + hotkey = metagraph.hotkeys[uid] # type: ignore + except IndexError: + logging.error( + "Your uid is not in the hotkeys. Please double-check your UID." + ) + return "" + + metadata = get_metadata(self, netuid, hotkey, block) # TODO add + try: + commitment = metadata["info"]["fields"][0] # type: ignore + hex_data = commitment[list(commitment.keys())[0]][2:] # type: ignore + return bytes.fromhex(hex_data).decode() + + except TypeError: + return "" def get_current_weight_commit_info( self, netuid: int, block: Optional[int] = None ) -> list: - return self.execute_coroutine( - self.async_subtensor.get_current_weight_commit_info( - netuid=netuid, block=block - ), + """ + Retrieves CRV3 weight commit information for a specific subnet. + + Arguments: + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. Default is ``None``. + + Returns: + list: A list of commit details, where each entry is a dictionary with keys 'who', 'serialized_commit', and + 'reveal_round', or an empty list if no data is found. + """ + result = self.substrate.query_map( + module="SubtensorModule", + storage_function="CRV3WeightCommits", + params=[netuid], + block_hash=self.determine_block_hash(block), ) + commits = result.records[0][1] if result.records else [] + return [WeightCommitInfo.from_vec_u8(commit) for commit in commits] + def get_delegate_by_hotkey( self, hotkey_ss58: str, block: Optional[int] = None ) -> Optional["DelegateInfo"]: - return self.execute_coroutine( - self.async_subtensor.get_delegate_by_hotkey( - hotkey_ss58=hotkey_ss58, block=block - ), + """ + Retrieves detailed information about a delegate neuron based on its hotkey. This function provides a + comprehensive view of the delegate's status, including its stakes, nominators, and reward distribution. + + Arguments: + hotkey_ss58 (str): The ``SS58`` address of the delegate's hotkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[DelegateInfo]: Detailed information about the delegate neuron, ``None`` if not found. + + This function is essential for understanding the roles and influence of delegate neurons within the Bittensor + network's consensus and governance structures. + """ + + block_hash = self.determine_block_hash(block) + encoded_hotkey = ss58_to_vec_u8(hotkey_ss58) + + json_body = self.substrate.rpc_request( + method="delegateInfo_getDelegate", # custom rpc method + params=([encoded_hotkey, block_hash] if block_hash else [encoded_hotkey]), ) + if not (result := json_body.get("result", None)): + return None + + return DelegateInfo.from_vec_u8(bytes(result)) + def get_delegate_identities( self, block: Optional[int] = None ) -> dict[str, "DelegatesDetails"]: - return self.execute_coroutine( - self.async_subtensor.get_delegate_identities(block=block), - ) + """ + Fetches delegates identities from the chain and GitHub. Preference is given to chain data, and missing info is + filled-in by the info from GitHub. At some point, we want to totally move away from fetching this info from + GitHub, but chain data is still limited in that regard. + + Arguments: + block (Optional[int]): The blockchain block number for the query. + + Returns: + Dict {ss58: DelegatesDetails, ...} + + """ + block_hash = self.determine_block_hash(block) + response = requests.get(DELEGATES_DETAILS_URL) + identities_info = self.substrate.query_map( + module="Registry", storage_function="IdentityOf", block_hash=block_hash + ) + + all_delegates_details = {} + for ss58_address, identity in identities_info: + all_delegates_details.update( + { + decode_account_id( + ss58_address[0] + ): DelegatesDetails.from_chain_data( + decode_hex_identity_dict(identity.value["info"]) + ) + } + ) + if response.ok: + all_delegates: dict[str, Any] = ujson.loads(response.content) + + for delegate_hotkey, delegate_details in all_delegates.items(): + delegate_info = all_delegates_details.setdefault( + delegate_hotkey, + DelegatesDetails( + display=delegate_details.get("name", ""), + web=delegate_details.get("url", ""), + additional=delegate_details.get("description", ""), + pgp_fingerprint=delegate_details.get("fingerprint", ""), + ), + ) + delegate_info.display = delegate_info.display or delegate_details.get( + "name", "" + ) + delegate_info.web = delegate_info.web or delegate_details.get("url", "") + delegate_info.additional = ( + delegate_info.additional or delegate_details.get("description", "") + ) + delegate_info.pgp_fingerprint = ( + delegate_info.pgp_fingerprint + or delegate_details.get("fingerprint", "") + ) + + return all_delegates_details def get_delegate_take( self, hotkey_ss58: str, block: Optional[int] = None ) -> Optional[float]: - return self.execute_coroutine( - self.async_subtensor.get_delegate_take( - hotkey_ss58=hotkey_ss58, block=block - ), + """ + Retrieves the delegate 'take' percentage for a neuron identified by its hotkey. The 'take' represents the + percentage of rewards that the delegate claims from its nominators' stakes. + + Arguments: + hotkey_ss58 (str): The ``SS58`` address of the neuron's hotkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[float]: The delegate take percentage, None if not available. + + The delegate take is a critical parameter in the network's incentive structure, influencing the distribution of + rewards among neurons and their nominators. + """ + result = self.query_subtensor( + name="Delegates", + block=block, + params=[hotkey_ss58], + ) + return ( + None + if result is None + else u16_normalized_float(getattr(result, "value", 0)) ) def get_delegated( self, coldkey_ss58: str, block: Optional[int] = None ) -> list[tuple["DelegateInfo", "Balance"]]: - return self.execute_coroutine( - self.async_subtensor.get_delegated(coldkey_ss58=coldkey_ss58, block=block), + """ + Retrieves a list of delegates and their associated stakes for a given coldkey. This function identifies the + delegates that a specific account has staked tokens on. + + Arguments: + coldkey_ss58 (str): The `SS58` address of the account's coldkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + A list of tuples, each containing a delegate's information and staked amount. + + This function is important for account holders to understand their stake allocations and their involvement in + the network's delegation and consensus mechanisms. + """ + + block_hash = self.determine_block_hash(block) + encoded_coldkey = ss58_to_vec_u8(coldkey_ss58) + json_body = self.substrate.rpc_request( + method="delegateInfo_getDelegated", + params=([block_hash, encoded_coldkey] if block_hash else [encoded_coldkey]), ) + if not (result := json_body.get("result")): + return [] + + return DelegateInfo.delegated_list_from_vec_u8(bytes(result)) + def get_delegates(self, block: Optional[int] = None) -> list["DelegateInfo"]: - return self.execute_coroutine( - self.async_subtensor.get_delegates(block=block), - ) + """ + Fetches all delegates on the chain + + Arguments: + block (Optional[int]): The blockchain block number for the query. + + Returns: + List of DelegateInfo objects, or an empty list if there are no delegates. + """ + hex_bytes_result = self.query_runtime_api( + runtime_api="DelegateInfoRuntimeApi", + method="get_delegates", + params=[], + block=block, + ) + if hex_bytes_result: + return DelegateInfo.list_from_vec_u8(hex_to_bytes(hex_bytes_result)) + else: + return [] def get_existential_deposit( self, block: Optional[int] = None ) -> Optional["Balance"]: - return self.execute_coroutine( - self.async_subtensor.get_existential_deposit(block=block), + """ + Retrieves the existential deposit amount for the Bittensor blockchain. + The existential deposit is the minimum amount of TAO required for an account to exist on the blockchain. + Accounts with balances below this threshold can be reaped to conserve network resources. + + Arguments: + block (Optional[int]): The blockchain block number for the query. + + Returns: + The existential deposit amount. + + The existential deposit is a fundamental economic parameter in the Bittensor network, ensuring efficient use of + storage and preventing the proliferation of dust accounts. + """ + result = self.substrate.get_constant( + module_name="Balances", + constant_name="ExistentialDeposit", + block_hash=self.determine_block_hash(block), ) + if result is None: + raise Exception("Unable to retrieve existential deposit amount.") + + return Balance.from_rao(getattr(result, "value", 0)) + def get_hotkey_owner( self, hotkey_ss58: str, block: Optional[int] = None ) -> Optional[str]: - return self.execute_coroutine( - self.async_subtensor.get_hotkey_owner(hotkey_ss58=hotkey_ss58, block=block), + """ + Retrieves the owner of the given hotkey at a specific block hash. + This function queries the blockchain for the owner of the provided hotkey. If the hotkey does not exist at the + specified block hash, it returns None. + + Arguments: + hotkey_ss58 (str): The SS58 address of the hotkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[str]: The SS58 address of the owner if the hotkey exists, or None if it doesn't. + """ + hk_owner_query = self.substrate.query( + module="SubtensorModule", + storage_function="Owner", + params=[hotkey_ss58], + block_hash=self.determine_block_hash(block), ) + exists = False + val = None + if hasattr(hk_owner_query, "value"): + val = decode_account_id(hk_owner_query.value[0]) + if val: + exists = self.does_hotkey_exist(hotkey_ss58, block=block) + hotkey_owner = val if exists else None + return hotkey_owner def get_minimum_required_stake(self) -> "Balance": - return self.execute_coroutine( - self.async_subtensor.get_minimum_required_stake(), + """ + Returns the minimum required stake for nominators in the Subtensor network. + This method retries the substrate call up to three times with exponential backoff in case of failures. + + Returns: + Balance: The minimum required stake as a Balance object. + + Raises: + Exception: If the substrate call fails after the maximum number of retries. + """ + result = self.substrate.query( + module="SubtensorModule", storage_function="NominatorMinRequiredStake" ) + return Balance.from_rao(getattr(result, "value", 0)) + def get_netuids_for_hotkey( - self, hotkey_ss58: str, block: Optional[int] = None, reuse_block: bool = False + self, hotkey_ss58: str, block: Optional[int] = None ) -> list[int]: - return self.execute_coroutine( - self.async_subtensor.get_netuids_for_hotkey( - hotkey_ss58=hotkey_ss58, block=block, reuse_block=reuse_block - ), + """ + Retrieves a list of subnet UIDs (netuids) for which a given hotkey is a member. This function identifies the + specific subnets within the Bittensor network where the neuron associated with the hotkey is active. + + Arguments: + hotkey_ss58 (str): The ``SS58`` address of the neuron's hotkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + A list of netuids where the neuron is a member. + """ + result = self.substrate.query_map( + module="SubtensorModule", + storage_function="IsNetworkMember", + params=[hotkey_ss58], + block_hash=self.determine_block_hash(block), ) + netuids = [] + if result.records: + async for record in result: + if record[1].value: + netuids.append(record[0]) + return netuids def get_neuron_certificate( self, hotkey: str, netuid: int, block: Optional[int] = None ) -> Optional["Certificate"]: - return self.execute_coroutine( - self.async_subtensor.get_neuron_certificate(hotkey, netuid, block=block), + """ + Retrieves the TLS certificate for a specific neuron identified by its unique identifier (UID) within a + specified subnet (netuid) of the Bittensor network. + + Arguments: + hotkey: The hotkey to query. + netuid: The unique identifier of the subnet. + block: The blockchain block number for the query. + + Returns: + the certificate of the neuron if found, `None` otherwise. + + This function is used for certificate discovery for setting up mutual tls communication between neurons. + """ + certificate = self.query_module( + module="SubtensorModule", + name="NeuronCertificates", + block=block, + params=[netuid, hotkey], ) + try: + if certificate: + public_key = bytes(certificate["public_key"][0]).hex() + return chr(certificate["algorithm"]) + f"0x{public_key}" + + except AttributeError: + return None + return None def get_neuron_for_pubkey_and_subnet( self, hotkey_ss58: str, netuid: int, block: Optional[int] = None ) -> Optional["NeuronInfo"]: - return self.execute_coroutine( - self.async_subtensor.get_neuron_for_pubkey_and_subnet( - hotkey_ss58, netuid, block=block - ), + """ + Retrieves information about a neuron based on its public key (hotkey SS58 address) and the specific subnet UID + (netuid). This function provides detailed neuron information for a particular subnet within the Bittensor + network. + + Arguments: + hotkey_ss58 (str): The ``SS58`` address of the neuron's hotkey. + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[bittensor.core.chain_data.neuron_info.NeuronInfo]: Detailed information about the neuron if found, + ``None`` otherwise. + + This function is crucial for accessing specific neuron data and understanding its status, stake, and other + attributes within a particular subnet of the Bittensor ecosystem. + """ + block_hash = self.determine_block_hash(block) + uid = self.substrate.query( + module="SubtensorModule", + storage_function="Uids", + params=[netuid, hotkey_ss58], + block_hash=block_hash, ) + if uid is None: + return NeuronInfo.get_null_neuron() + + params = [netuid, uid.value] + json_body = self.substrate.rpc_request( + method="neuronInfo_getNeuron", params=params, block_hash=block_hash + ) + + if not (result := json_body.get("result", None)): + return NeuronInfo.get_null_neuron() + + return NeuronInfo.from_vec_u8(bytes(result)) def get_stake_for_coldkey_and_hotkey( self, hotkey_ss58: str, coldkey_ss58: str, block: Optional[int] = None ) -> Optional["Balance"]: - return self.execute_coroutine( - self.async_subtensor.get_stake_for_coldkey_and_hotkey( - hotkey_ss58=hotkey_ss58, coldkey_ss58=coldkey_ss58, block=block - ), + """ + Retrieves stake information associated with a specific coldkey and hotkey. + + Arguments: + hotkey_ss58 (str): the hotkey SS58 address to query + coldkey_ss58 (str): the coldkey SS58 address to query + block (Optional[int]): the block number to query + + Returns: + Stake Balance for the given coldkey and hotkey + """ + result = self.substrate.query( + module="SubtensorModule", + storage_function="Stake", + params=[hotkey_ss58, coldkey_ss58], + block_hash=self.determine_block_hash(block), ) + return Balance.from_rao(getattr(result, "value", 0)) def get_stake_info_for_coldkey( self, coldkey_ss58: str, block: Optional[int] = None ) -> list["StakeInfo"]: - return self.execute_coroutine( - self.async_subtensor.get_stake_info_for_coldkey( - coldkey_ss58=coldkey_ss58, block=block - ), + """ + Retrieves stake information associated with a specific coldkey. This function provides details about the stakes + held by an account, including the staked amounts and associated delegates. + + Arguments: + coldkey_ss58 (str): The ``SS58`` address of the account's coldkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + A list of StakeInfo objects detailing the stake allocations for the account. + + Stake information is vital for account holders to assess their investment and participation in the network's + delegation and consensus processes. + """ + encoded_coldkey = ss58_to_vec_u8(coldkey_ss58) + + hex_bytes_result = self.query_runtime_api( + runtime_api="StakeInfoRuntimeApi", + method="get_stake_info_for_coldkey", + params=[encoded_coldkey], + block=block, ) + if not hex_bytes_result: + return [] + + return StakeInfo.list_from_vec_u8(hex_to_bytes(hex_bytes_result)) + def get_subnet_burn_cost(self, block: Optional[int] = None) -> Optional[str]: - return self.execute_coroutine( - self.async_subtensor.get_subnet_burn_cost(block=block), + """ + Retrieves the burn cost for registering a new subnet within the Bittensor network. This cost represents the + amount of Tao that needs to be locked or burned to establish a new subnet. + + Arguments: + block (Optional[int]): The blockchain block number for the query. + + Returns: + int: The burn cost for subnet registration. + + The subnet burn cost is an important economic parameter, reflecting the network's mechanisms for controlling + the proliferation of subnets and ensuring their commitment to the network's long-term viability. + """ + lock_cost = self.query_runtime_api( + runtime_api="SubnetRegistrationRuntimeApi", + method="get_network_registration_cost", + params=[], + block=block, ) + return lock_cost + def get_subnet_hyperparameters( self, netuid: int, block: Optional[int] = None ) -> Optional[Union[list, "SubnetHyperparameters"]]: - return self.execute_coroutine( - self.async_subtensor.get_subnet_hyperparameters(netuid=netuid, block=block), + """ + Retrieves the hyperparameters for a specific subnet within the Bittensor network. These hyperparameters define + the operational settings and rules governing the subnet's behavior. + + Arguments: + netuid (int): The network UID of the subnet to query. + block (Optional[int]): The blockchain block number for the query. + + Returns: + The subnet's hyperparameters, or `None` if not available. + + Understanding the hyperparameters is crucial for comprehending how subnets are configured and managed, and how + they interact with the network's consensus and incentive mechanisms. + """ + hex_bytes_result = self.query_runtime_api( + runtime_api="SubnetInfoRuntimeApi", + method="get_subnet_hyperparams", + params=[netuid], + block=block, ) + if not hex_bytes_result: + return None + + return SubnetHyperparameters.from_vec_u8(hex_to_bytes(hex_bytes_result)) + def get_subnet_reveal_period_epochs( self, netuid: int, block: Optional[int] = None ) -> int: - return self.execute_coroutine( - self.async_subtensor.get_subnet_reveal_period_epochs( - netuid=netuid, block=block - ), + """Retrieve the SubnetRevealPeriodEpochs hyperparameter.""" + return self.get_hyperparameter( + param_name="RevealPeriodEpochs", block=block, netuid=netuid ) def get_subnets(self, block: Optional[int] = None) -> list[int]: - return self.execute_coroutine( - self.async_subtensor.get_subnets(block=block), + """ + Retrieves the list of all subnet unique identifiers (netuids) currently present in the Bittensor network. + + Arguments: + block (Optional[int]): The blockchain block number for the query. + + Returns: + A list of subnet netuids. + + This function provides a comprehensive view of the subnets within the Bittensor network, + offering insights into its diversity and scale. + """ + result = self.substrate.query_map( + module="SubtensorModule", + storage_function="NetworksAdded", + block_hash=self.determine_block_hash(block), ) + subnets = [] + if result.records: + for netuid, exists in result: + if exists: + subnets.append(netuid) + return subnets def get_total_stake_for_coldkey( self, ss58_address: str, block: Optional[int] = None ) -> "Balance": - result = self.execute_coroutine( - self.async_subtensor.get_total_stake_for_coldkey(ss58_address, block=block), + """ + Returns the total stake held on a coldkey. + + Arguments: + ss58_address (str): The SS58 address of the coldkey + block (Optional[int]): The blockchain block number for the query. + + Returns: + Balance of the stake held on the coldkey. + """ + result = self.substrate.query( + module="SubtensorModule", + storage_function="TotalColdkeyStake", + params=[ss58_address], + block_hash=self.determine_block_hash(block), ) - return result + return Balance.from_rao(getattr(result, "value", 0)) def get_total_stake_for_coldkeys( self, *ss58_addresses: str, block: Optional[int] = None ) -> dict[str, "Balance"]: - return self.execute_coroutine( - self.async_subtensor.get_total_stake_for_coldkeys( - *ss58_addresses, block=block - ), - ) + """ + Returns the total stake held on multiple coldkeys. + + Arguments: + ss58_addresses (tuple[str]): The SS58 address(es) of the coldkey(s) + block (Optional[int]): The blockchain block number for the query. + + Returns: + Dict in view {address: Balance objects}. + """ + if not (block_hash := self.determine_block_hash(block)): + block_hash = self.substrate.get_chain_head() + calls = [ + ( + self.substrate.create_storage_key( + "SubtensorModule", + "TotalColdkeyStake", + [address], + block_hash=block_hash, + ) + ) + for address in ss58_addresses + ] + batch_call = self.substrate.query_multi(calls, block_hash=block_hash) + results = {} + for item in batch_call: + results.update({item[0].params[0]: Balance.from_rao(item[1] or 0)}) + return results def get_total_stake_for_hotkey( self, ss58_address: str, block: Optional[int] = None ) -> "Balance": - result = self.execute_coroutine( - self.async_subtensor.get_total_stake_for_hotkey(ss58_address, block=block), + """ + Returns the total stake held on a hotkey. + + Arguments: + ss58_address (str): The SS58 address of the hotkey + block (Optional[int]): The blockchain block number for the query. + + Returns: + Balance of the stake held on the hotkey. + """ + result = self.substrate.query( + module="SubtensorModule", + storage_function="TotalHotkeyStake", + params=[ss58_address], + block_hash=self.determine_block_hash(block), ) - return result + return Balance.from_rao(getattr(result, "value", 0)) def get_total_stake_for_hotkeys( self, *ss58_addresses: str, block: Optional[int] = None ) -> dict[str, "Balance"]: - return self.execute_coroutine( - self.async_subtensor.get_total_stake_for_hotkeys( - *ss58_addresses, block=block - ), + """ + Returns the total stake held on hotkeys. + + Arguments: + ss58_addresses (tuple[str]): The SS58 address(es) of the hotkey(s) + block (Optional[int]): The blockchain block number for the query. + + Returns: + Dict {address: Balance objects}. + """ + results = self.substrate.query_multiple( + params=[s for s in ss58_addresses], + module="SubtensorModule", + storage_function="TotalHotkeyStake", + block_hash=self.determine_block_hash(block), ) + return {k: Balance.from_rao(r or 0) for (k, r) in results.items()} def get_total_subnets(self, block: Optional[int] = None) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.get_total_subnets(block=block), + """ + Retrieves the total number of subnets within the Bittensor network as of a specific blockchain block. + + Arguments: + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[str]: The total number of subnets in the network. + + Understanding the total number of subnets is essential for assessing the network's growth and the extent of its + decentralized infrastructure. + """ + result = self.substrate.query( + module="SubtensorModule", + storage_function="TotalNetworks", + params=[], + block_hash=self.determine_block_hash(block), ) + return getattr(result, "value", None) def get_transfer_fee( self, wallet: "Wallet", dest: str, value: Union["Balance", float, int] ) -> "Balance": - return self.execute_coroutine( - self.async_subtensor.get_transfer_fee( - wallet=wallet, dest=dest, value=value - ), - ) + """ + Calculates the transaction fee for transferring tokens from a wallet to a specified destination address. This + function simulates the transfer to estimate the associated cost, taking into account the current network + conditions and transaction complexity. + + Arguments: + wallet (bittensor_wallet.Wallet): The wallet from which the transfer is initiated. + dest (str): The ``SS58`` address of the destination account. + value (Union[bittensor.utils.balance.Balance, float, int]): The amount of tokens to be transferred, + specified as a Balance object, or in Tao (float) or Rao (int) units. + + Returns: + bittensor.utils.balance.Balance: The estimated transaction fee for the transfer, represented as a Balance + object. + + Estimating the transfer fee is essential for planning and executing token transactions, ensuring that the wallet + has sufficient funds to cover both the transfer amount and the associated costs. This function provides a + crucial tool for managing financial operations within the Bittensor network. + """ + if isinstance(value, float): + value = Balance.from_tao(value) + elif isinstance(value, int): + value = Balance.from_rao(value) + + if isinstance(value, Balance): + call = self.substrate.compose_call( + call_module="Balances", + call_function="transfer_allow_death", + call_params={"dest": dest, "value": str(value.rao)}, + ) + + try: + payment_info = self.substrate.get_payment_info( + call=call, keypair=wallet.coldkeypub + ) + except Exception as e: + logging.error( + f":cross_mark: [red]Failed to get payment info: [/red]{e}" + ) + payment_info = {"partialFee": int(2e7)} # assume 0.02 Tao + + return Balance.from_rao(payment_info["partialFee"]) + else: + fee = Balance.from_rao(int(2e7)) + logging.error( + "To calculate the transaction fee, the value must be Balance, float, or int. Received type: %s. Fee " + "is %s", + type(value), + 2e7, + ) + return fee def get_vote_data( self, proposal_hash: str, block: Optional[int] = None ) -> Optional["ProposalVoteData"]: - return self.execute_coroutine( - self.async_subtensor.get_vote_data( - proposal_hash=proposal_hash, block=block - ), + """ + Retrieves the voting data for a specific proposal on the Bittensor blockchain. This data includes information + about how senate members have voted on the proposal. + + Arguments: + proposal_hash (str): The hash of the proposal for which voting data is requested. + block (Optional[int]): The blockchain block number for the query. + + Returns: + An object containing the proposal's voting data, or `None` if not found. + + This function is important for tracking and understanding the decision-making processes within the Bittensor + network, particularly how proposals are received and acted upon by the governing body. + """ + vote_data = self.substrate.query( + module="Triumvirate", + storage_function="Voting", + params=[proposal_hash], + block_hash=self.determine_block_hash(block), ) + if vote_data is None: + return None + else: + return ProposalVoteData(vote_data) def get_uid_for_hotkey_on_subnet( self, hotkey_ss58: str, netuid: int, block: Optional[int] = None ) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.get_uid_for_hotkey_on_subnet( - hotkey_ss58=hotkey_ss58, netuid=netuid, block=block - ), + """ + Retrieves the unique identifier (UID) for a neuron's hotkey on a specific subnet. + + Arguments: + hotkey_ss58 (str): The ``SS58`` address of the neuron's hotkey. + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[int]: The UID of the neuron if it is registered on the subnet, ``None`` otherwise. + + The UID is a critical identifier within the network, linking the neuron's hotkey to its operational and + governance activities on a particular subnet. + """ + result = self.substrate.query( + module="SubtensorModule", + storage_function="Uids", + params=[netuid, hotkey_ss58], + block_hash=self.determine_block_hash(block), ) + return getattr(result, "value", result) def filter_netuids_by_registered_hotkeys( self, @@ -706,28 +1441,91 @@ def filter_netuids_by_registered_hotkeys( all_hotkeys: Iterable["Wallet"], block: Optional[int], ) -> list[int]: - return self.execute_coroutine( - self.async_subtensor.filter_netuids_by_registered_hotkeys( - all_netuids=all_netuids, - filter_for_netuids=filter_for_netuids, - all_hotkeys=all_hotkeys, - block=block, - ), - ) + """ + Filters a given list of all netuids for certain specified netuids and hotkeys + + Arguments: + all_netuids (Iterable[int]): A list of netuids to filter. + filter_for_netuids (Iterable[int]): A subset of all_netuids to filter from the main list. + all_hotkeys (Iterable[Wallet]): Hotkeys to filter from the main list. + block (Optional[int]): The blockchain block number for the query. + + Returns: + The filtered list of netuids. + """ + # TODO add non-async async stuff here + self._get_block_hash(block) # just used to cache the block hash + netuids_with_registered_hotkeys = [ + item + for sublist in [ + self.get_netuids_for_hotkey( + wallet.hotkey.ss58_address, + block=block, + ) + for wallet in all_hotkeys + ] + for item in sublist + ] + + if not filter_for_netuids: + all_netuids = netuids_with_registered_hotkeys + + else: + filtered_netuids = [ + netuid for netuid in all_netuids if netuid in filter_for_netuids + ] + + registered_hotkeys_filtered = [ + netuid + for netuid in netuids_with_registered_hotkeys + if netuid in filter_for_netuids + ] + + # Combine both filtered lists + all_netuids = filtered_netuids + registered_hotkeys_filtered + + return list(set(all_netuids)) def immunity_period( self, netuid: int, block: Optional[int] = None ) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.immunity_period(netuid=netuid, block=block), + """ + Retrieves the 'ImmunityPeriod' hyperparameter for a specific subnet. This parameter defines the duration during + which new neurons are protected from certain network penalties or restrictions. + + Args: + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[int]: The value of the 'ImmunityPeriod' hyperparameter if the subnet exists, ``None`` otherwise. + + The 'ImmunityPeriod' is a critical aspect of the network's governance system, ensuring that new participants + have a grace period to establish themselves and contribute to the network without facing immediate + punitive actions. + """ + call = self.get_hyperparameter( + param_name="ImmunityPeriod", netuid=netuid, block=block ) + return None if call is None else int(call) def is_hotkey_delegate(self, hotkey_ss58: str, block: Optional[int] = None) -> bool: - return self.execute_coroutine( - self.async_subtensor.is_hotkey_delegate( - hotkey_ss58=hotkey_ss58, block=block - ), - ) + """ + Determines whether a given hotkey (public key) is a delegate on the Bittensor network. This function checks if + the neuron associated with the hotkey is part of the network's delegation system. + + Arguments: + hotkey_ss58 (str): The SS58 address of the neuron's hotkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + `True` if the hotkey is a delegate, `False` otherwise. + + Being a delegate is a significant status within the Bittensor network, indicating a neuron's involvement in + consensus and governance processes. + """ + delegates = self.get_delegates(block) + return hotkey_ss58 in [info.hotkey_ss58 for info in delegates] def is_hotkey_registered( self, @@ -735,40 +1533,90 @@ def is_hotkey_registered( netuid: Optional[int] = None, block: Optional[int] = None, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.is_hotkey_registered( - hotkey_ss58=hotkey_ss58, netuid=netuid, block=block - ), - ) + """ + Determines whether a given hotkey (public key) is registered in the Bittensor network, either globally across + any subnet or specifically on a specified subnet. This function checks the registration status of a neuron + identified by its hotkey, which is crucial for validating its participation and activities within the + network. + + Args: + hotkey_ss58: The SS58 address of the neuron's hotkey. + netuid: The unique identifier of the subnet to check the registration. If `None`, the + registration is checked across all subnets. + block: The blockchain block number at which to perform the query. + + Returns: + bool: `True` if the hotkey is registered in the specified context (either any subnet or a specific subnet), + `False` otherwise. + + This function is important for verifying the active status of neurons in the Bittensor network. It aids in + understanding whether a neuron is eligible to participate in network processes such as consensus, + validation, and incentive distribution based on its registration status. + """ + if netuid is None: + return self.is_hotkey_registered_any(hotkey_ss58, block) + else: + return self.is_hotkey_registered_on_subnet(hotkey_ss58, netuid, block) def is_hotkey_registered_any( self, hotkey_ss58: str, block: Optional[int] = None, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.is_hotkey_registered_any( - hotkey_ss58=hotkey_ss58, - block=block, - ), - ) + """ + Checks if a neuron's hotkey is registered on any subnet within the Bittensor network. + + Arguments: + hotkey_ss58 (str): The ``SS58`` address of the neuron's hotkey. + block (Optional[int]): The blockchain block number for the query. + + Returns: + bool: ``True`` if the hotkey is registered on any subnet, False otherwise. + + This function is essential for determining the network-wide presence and participation of a neuron. + """ + hotkeys = self.get_netuids_for_hotkey(hotkey_ss58, block) + return len(hotkeys) > 0 def is_hotkey_registered_on_subnet( self, hotkey_ss58: str, netuid: int, block: Optional[int] = None ) -> bool: - return self.get_uid_for_hotkey_on_subnet(hotkey_ss58, netuid, block) is not None + """Checks if the hotkey is registered on a given netuid.""" + return ( + self.get_uid_for_hotkey_on_subnet(hotkey_ss58, netuid, block=block) + is not None + ) def last_drand_round(self) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.last_drand_round(), + """ + Retrieves the last drand round emitted in bittensor. This corresponds when committed weights will be revealed. + + Returns: + int: The latest Drand round emitted in bittensor. + """ + result = self.substrate.query( + module="Drand", storage_function="LastStoredRound" ) + return getattr(result, "value", None) def max_weight_limit( self, netuid: int, block: Optional[int] = None ) -> Optional[float]: - return self.execute_coroutine( - self.async_subtensor.max_weight_limit(netuid=netuid, block=block), + """ + Returns network MaxWeightsLimit hyperparameter. + + Args: + netuid (int): The unique identifier of the subnetwork. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[float]: The value of the MaxWeightsLimit hyperparameter, or ``None`` if the subnetwork does not + exist or the parameter is not found. + """ + call = self.get_hyperparameter( + param_name="MaxWeightsLimit", netuid=netuid, block=block ) + return None if call is None else u16_normalized_float(int(call)) def metagraph( self, netuid: int, lite: bool = True, block: Optional[int] = None @@ -787,72 +1635,284 @@ def metagraph( def min_allowed_weights( self, netuid: int, block: Optional[int] = None ) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.min_allowed_weights(netuid=netuid, block=block), + """ + Returns network MinAllowedWeights hyperparameter. + + Args: + netuid (int): The unique identifier of the subnetwork. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[int]: The value of the MinAllowedWeights hyperparameter, or ``None`` if the subnetwork does not + exist or the parameter is not found. + """ + call = self.get_hyperparameter( + param_name="MinAllowedWeights", netuid=netuid, block=block ) + return None if call is None else int(call) def neuron_for_uid( self, uid: int, netuid: int, block: Optional[int] = None ) -> "NeuronInfo": - return self.execute_coroutine( - self.async_subtensor.neuron_for_uid(uid=uid, netuid=netuid, block=block), + """ + Retrieves detailed information about a specific neuron identified by its unique identifier (UID) within a + specified subnet (netuid) of the Bittensor network. This function provides a comprehensive view of a + neuron's attributes, including its stake, rank, and operational status. + + Arguments: + uid (int): The unique identifier of the neuron. + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Detailed information about the neuron if found, a null neuron otherwise + + This function is crucial for analyzing individual neurons' contributions and status within a specific subnet, + offering insights into their roles in the network's consensus and validation mechanisms. + """ + if uid is None: + return NeuronInfo.get_null_neuron() + + block_hash = self.determine_block_hash(block) + + params = [netuid, uid, block_hash] if block_hash else [netuid, uid] + json_body = self.substrate.rpc_request( + method="neuronInfo_getNeuron", # custom rpc method + params=params, ) + if not (result := json_body.get("result", None)): + return NeuronInfo.get_null_neuron() + + bytes_result = bytes(result) + return NeuronInfo.from_vec_u8(bytes_result) def neurons(self, netuid: int, block: Optional[int] = None) -> list["NeuronInfo"]: - return self.execute_coroutine( - self.async_subtensor.neurons(netuid=netuid, block=block), + """ + Retrieves a list of all neurons within a specified subnet of the Bittensor network. + This function provides a snapshot of the subnet's neuron population, including each neuron's attributes and + network interactions. + + Arguments: + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + A list of NeuronInfo objects detailing each neuron's characteristics in the subnet. + + Understanding the distribution and status of neurons within a subnet is key to comprehending the network's + decentralized structure and the dynamics of its consensus and governance processes. + """ + hex_bytes_result = self.query_runtime_api( + runtime_api="NeuronInfoRuntimeApi", + method="get_neurons", + params=[netuid], + block=block, ) + if not hex_bytes_result: + return [] + + return NeuronInfo.list_from_vec_u8(hex_to_bytes(hex_bytes_result)) + def neurons_lite( self, netuid: int, block: Optional[int] = None ) -> list["NeuronInfoLite"]: - return self.execute_coroutine( - self.async_subtensor.neurons_lite(netuid=netuid, block=block), + """ + Retrieves a list of neurons in a 'lite' format from a specific subnet of the Bittensor network. + This function provides a streamlined view of the neurons, focusing on key attributes such as stake and network + participation. + + Arguments: + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + A list of simplified neuron information for the subnet. + + This function offers a quick overview of the neuron population within a subnet, facilitating efficient analysis + of the network's decentralized structure and neuron dynamics. + """ + hex_bytes_result = self.query_runtime_api( + runtime_api="NeuronInfoRuntimeApi", + method="get_neurons_lite", + params=[ + netuid + ], # TODO check to see if this can accept more than one at a time + block=block, ) + if not hex_bytes_result: + return [] + + return NeuronInfoLite.list_from_vec_u8(hex_to_bytes(hex_bytes_result)) + def query_identity(self, key: str, block: Optional[int] = None) -> Optional[str]: - return self.execute_coroutine( - self.async_subtensor.query_identity(key=key, block=block), + """ + Queries the identity of a neuron on the Bittensor blockchain using the given key. This function retrieves + detailed identity information about a specific neuron, which is a crucial aspect of the network's + decentralized identity and governance system. + + Arguments: + key (str): The key used to query the neuron's identity, typically the neuron's SS58 address. + block (Optional[int]): The blockchain block number for the query. + + Returns: + An object containing the identity information of the neuron if found, ``None`` otherwise. + + The identity information can include various attributes such as the neuron's stake, rank, and other + network-specific details, providing insights into the neuron's role and status within the Bittensor network. + + Note: + See the `Bittensor CLI documentation `_ for supported identity + parameters. + """ + identity_info = self.substrate.query( + module="Registry", + storage_function="IdentityOf", + params=[key], + block_hash=self.determine_block_hash(block), ) + try: + return _decode_hex_identity_dict(identity_info["info"]) + except TypeError: + return {} def recycle(self, netuid: int, block: Optional[int] = None) -> Optional["Balance"]: - return self.execute_coroutine( - self.async_subtensor.recycle(netuid=netuid, block=block), - ) + """ + Retrieves the 'Burn' hyperparameter for a specified subnet. The 'Burn' parameter represents the amount of Tao + that is effectively recycled within the Bittensor network. + + Args: + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[Balance]: The value of the 'Burn' hyperparameter if the subnet exists, None otherwise. + + Understanding the 'Burn' rate is essential for analyzing the network registration usage, particularly how it is + correlated with user activity and the overall cost of participation in a given subnet. + """ + call = self.get_hyperparameter(param_name="Burn", netuid=netuid, block=block) + return None if call is None else Balance.from_rao(int(call)) def subnet_exists(self, netuid: int, block: Optional[int] = None) -> bool: - return self.execute_coroutine( - self.async_subtensor.subnet_exists(netuid=netuid, block=block), + """ + Checks if a subnet with the specified unique identifier (netuid) exists within the Bittensor network. + + Arguments: + netuid (int): The unique identifier of the subnet. + block (Optional[int]): The blockchain block number for the query. + + Returns: + `True` if the subnet exists, `False` otherwise. + + This function is critical for verifying the presence of specific subnets in the network, + enabling a deeper understanding of the network's structure and composition. + """ + result = self.substrate.query( + module="SubtensorModule", + storage_function="NetworksAdded", + params=[netuid], + block_hash=self.determine_block_hash(block), ) + return getattr(result, "value", False) def subnetwork_n(self, netuid: int, block: Optional[int] = None) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.subnetwork_n(netuid=netuid, block=block), + """ + Returns network SubnetworkN hyperparameter. + + Args: + netuid (int): The unique identifier of the subnetwork. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[int]: The value of the SubnetworkN hyperparameter, or ``None`` if the subnetwork does not exist or + the parameter is not found. + """ + call = self.get_hyperparameter( + param_name="SubnetworkN", netuid=netuid, block=block ) + return None if call is None else int(call) def tempo(self, netuid: int, block: Optional[int] = None) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.tempo(netuid=netuid, block=block), - ) + """ + Returns network Tempo hyperparameter. + + Args: + netuid (int): The unique identifier of the subnetwork. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[int]: The value of the Tempo hyperparameter, or ``None`` if the subnetwork does not exist or the + parameter is not found. + """ + call = self.get_hyperparameter(param_name="Tempo", netuid=netuid, block=block) + return None if call is None else int(call) def tx_rate_limit(self, block: Optional[int] = None) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.tx_rate_limit(block=block), - ) + """ + Retrieves the transaction rate limit for the Bittensor network as of a specific blockchain block. + This rate limit sets the maximum number of transactions that can be processed within a given time frame. + + Args: + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[int]: The transaction rate limit of the network, None if not available. + + The transaction rate limit is an essential parameter for ensuring the stability and scalability of the Bittensor + network. It helps in managing network load and preventing congestion, thereby maintaining efficient and + timely transaction processing. + """ + result = self.query_subtensor("TxRateLimit", block=block) + return getattr(result, "value", None) def weights( self, netuid: int, block: Optional[int] = None ) -> list[tuple[int, list[tuple[int, int]]]]: - return self.execute_coroutine( - self.async_subtensor.weights(netuid=netuid, block=block), + """ + Retrieves the weight distribution set by neurons within a specific subnet of the Bittensor network. + This function maps each neuron's UID to the weights it assigns to other neurons, reflecting the network's trust + and value assignment mechanisms. + + Arguments: + netuid (int): The network UID of the subnet to query. + block (Optional[int]): Block number for synchronization, or ``None`` for the latest block. + + Returns: + A list of tuples mapping each neuron's UID to its assigned weights. + + The weight distribution is a key factor in the network's consensus algorithm and the ranking of neurons, + influencing their influence and reward allocation within the subnet. + """ + w_map_encoded = self.substrate.query_map( + module="SubtensorModule", + storage_function="Weights", + params=[netuid], + block_hash=self.determine_block_hash(block), ) + w_map = [(uid, w.value or []) for uid, w in w_map_encoded] + + return w_map def weights_rate_limit( self, netuid: int, block: Optional[int] = None ) -> Optional[int]: - return self.execute_coroutine( - self.async_subtensor.weights_rate_limit(netuid=netuid, block=block), + """ + Returns network WeightsSetRateLimit hyperparameter. + + Arguments: + netuid (int): The unique identifier of the subnetwork. + block (Optional[int]): The blockchain block number for the query. + + Returns: + Optional[int]: The value of the WeightsSetRateLimit hyperparameter, or ``None`` if the subnetwork does not + exist or the parameter is not found. + """ + call = self.get_hyperparameter( + param_name="WeightsSetRateLimit", netuid=netuid, block=block ) + return None if call is None else int(call) # Extrinsics ======================================================================================================= @@ -864,14 +1924,32 @@ def add_stake( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.add_stake( - wallet=wallet, - hotkey_ss58=hotkey_ss58, - amount=amount, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ), + """ + Adds the specified amount of stake to a neuron identified by the hotkey ``SS58`` address. + Staking is a fundamental process in the Bittensor network that enables neurons to participate actively and earn + incentives. + + Args: + wallet (bittensor_wallet.Wallet): The wallet to be used for staking. + hotkey_ss58 (Optional[str]): The ``SS58`` address of the hotkey associated with the neuron. + amount (Union[Balance, float]): The amount of TAO to stake. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + bool: ``True`` if the staking is successful, False otherwise. + + This function enables neurons to increase their stake in the network, enhancing their influence and potential + rewards in line with Bittensor's consensus and reward mechanisms. + """ + # TODO add this extrinsic + return add_stake_extrinsic( + subtensor=self, + wallet=wallet, + hotkey_ss58=hotkey_ss58, + amount=amount, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) def add_stake_multiple( @@ -882,14 +1960,31 @@ def add_stake_multiple( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.add_stake_multiple( - wallet=wallet, - hotkey_ss58s=hotkey_ss58s, - amounts=amounts, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ), + """ + Adds stakes to multiple neurons identified by their hotkey SS58 addresses. + This bulk operation allows for efficient staking across different neurons from a single wallet. + + Args: + wallet (bittensor_wallet.Wallet): The wallet used for staking. + hotkey_ss58s (list[str]): List of ``SS58`` addresses of hotkeys to stake to. + amounts (list[Union[Balance, float]]): Corresponding amounts of TAO to stake for each hotkey. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + bool: ``True`` if the staking is successful for all specified neurons, False otherwise. + + This function is essential for managing stakes across multiple neurons, reflecting the dynamic and collaborative + nature of the Bittensor network. + """ + # TODO add this extrinsic + return add_stake_multiple_extrinsic( + subtensor=self, + wallet=wallet, + hotkey_ss58s=hotkey_ss58s, + amounts=amounts, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) def burned_register( @@ -899,13 +1994,28 @@ def burned_register( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.burned_register( - wallet=wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ), + """ + Registers a neuron on the Bittensor network by recycling TAO. This method of registration involves recycling + TAO tokens, allowing them to be re-mined by performing work on the network. + + Args: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron to be registered. + netuid (int): The unique identifier of the subnet. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. Defaults to + `False`. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + Defaults to `True`. + + Returns: + bool: ``True`` if the registration is successful, False otherwise. + """ + # TODO add this extrinsic + return burned_register_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) def commit_weights( @@ -920,20 +2030,69 @@ def commit_weights( wait_for_finalization: bool = False, max_retries: int = 5, ) -> tuple[bool, str]: - return self.execute_coroutine( - self.async_subtensor.commit_weights( - wallet=wallet, - netuid=netuid, - salt=salt, - uids=uids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - max_retries=max_retries, - ), + """ + Commits a hash of the neuron's weights to the Bittensor blockchain using the provided wallet. + This action serves as a commitment or snapshot of the neuron's current weight distribution. + + Arguments: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuid (int): The unique identifier of the subnet. + salt (list[int]): list of randomly generated integers as salt to generated weighted hash. + uids (np.ndarray): NumPy array of neuron UIDs for which weights are being committed. + weights (np.ndarray): NumPy array of weight values corresponding to each UID. + version_key (int): Version key for compatibility with the network. Default is ``int representation of + Bittensor version.``. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is + ``False``. + max_retries (int): The number of maximum attempts to commit weights. Default is ``5``. + + Returns: + tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + + This function allows neurons to create a tamper-proof record of their weight distribution at a specific point + in time, enhancing transparency and accountability within the Bittensor network. + """ + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to commit weights!" + + logging.info( + f"Committing weights with params: netuid={netuid}, uids={uids}, weights={weights}, " + f"version_key={version_key}" ) + # Generate the hash of the weights + commit_hash = generate_weight_hash( + address=wallet.hotkey.ss58_address, + netuid=netuid, + uids=list(uids), + values=list(weights), + salt=salt, + version_key=version_key, + ) + + while retries < max_retries and success is False: + try: + # TODO add this extrinsic + success, message = commit_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + commit_hash=commit_hash, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + if success: + break + except Exception as e: + logging.error(f"Error committing weights: {e}") + finally: + retries += 1 + + return success, message + def register( self, wallet: "Wallet", @@ -949,21 +2108,49 @@ def register( update_interval: Optional[int] = None, log_verbose: bool = False, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.register( - wallet=wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - max_allowed_attempts=max_allowed_attempts, - output_in_place=output_in_place, - cuda=cuda, - dev_id=dev_id, - tpb=tpb, - num_processes=num_processes, - update_interval=update_interval, - log_verbose=log_verbose, - ), + """ + Registers a neuron on the Bittensor network using the provided wallet. + + Registration is a critical step for a neuron to become an active participant in the network, enabling it to + stake, set weights, and receive incentives. + + Args: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron to be registered. + netuid (int): The unique identifier of the subnet. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Defaults to `False`. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Defaults to + `True`. + max_allowed_attempts (int): Maximum number of attempts to register the wallet. + output_in_place (bool): If true, prints the progress of the proof of work to the console in-place. Meaning + the progress is printed on the same lines. Defaults to `True`. + cuda (bool): If ``true``, the wallet should be registered using CUDA device(s). Defaults to `False`. + dev_id (Union[List[int], int]): The CUDA device id to use, or a list of device ids. Defaults to `0` (zero). + tpb (int): The number of threads per block (CUDA). Default to `256`. + num_processes (Optional[int]): The number of processes to use to register. Default to `None`. + update_interval (Optional[int]): The number of nonces to solve between updates. Default to `None`. + log_verbose (bool): If ``true``, the registration process will log more information. Default to `False`. + + Returns: + bool: ``True`` if the registration is successful, False otherwise. + + This function facilitates the entry of new neurons into the network, supporting the decentralized + growth and scalability of the Bittensor ecosystem. + """ + # TODO add this extrinsic + return register_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + max_allowed_attempts=max_allowed_attempts, + tpb=tpb, + update_interval=update_interval, + num_processes=num_processes, + cuda=cuda, + dev_id=dev_id, + output_in_place=output_in_place, + log_verbose=log_verbose, ) def reveal_weights( @@ -978,32 +2165,111 @@ def reveal_weights( wait_for_finalization: bool = False, max_retries: int = 5, ) -> tuple[bool, str]: - return self.execute_coroutine( - self.async_subtensor.reveal_weights( - wallet=wallet, - netuid=netuid, - uids=uids, - weights=weights, - salt=salt, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - max_retries=max_retries, - ), - ) + """ + Reveals the weights for a specific subnet on the Bittensor blockchain using the provided wallet. + This action serves as a revelation of the neuron's previously committed weight distribution. + + Args: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron revealing the weights. + netuid (int): The unique identifier of the subnet. + uids (np.ndarray): NumPy array of neuron UIDs for which weights are being revealed. + weights (np.ndarray): NumPy array of weight values corresponding to each UID. + salt (np.ndarray): NumPy array of salt values corresponding to the hash function. + version_key (int): Version key for compatibility with the network. Default is ``int representation of + Bittensor version``. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is + ``False``. + max_retries (int): The number of maximum attempts to reveal weights. Default is ``5``. + + Returns: + tuple[bool, str]: ``True`` if the weight revelation is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + + This function allows neurons to reveal their previously committed weight distribution, ensuring transparency + and accountability within the Bittensor network. + """ + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to reveal weights!" + + while retries < max_retries and success is False: + try: + # TODO add this extrinsic + success, message = reveal_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + uids=list(uids), + weights=list(weights), + salt=list(salt), + version_key=version_key, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + if success: + break + except Exception as e: + logging.error(f"Error revealing weights: {e}") + finally: + retries += 1 + + return success, message def root_register( self, wallet: "Wallet", + netuid: int = 0, wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.root_register( - wallet=wallet, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + """ + Register neuron by recycling some TAO. + + Arguments: + wallet (bittensor_wallet.Wallet): Bittensor wallet instance. + netuid (int): Subnet uniq id. Root subnet uid is 0. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is + ``False``. + + Returns: + `True` if registration was successful, otherwise `False`. + """ + logging.info( + f"Registering on netuid [blue]0[/blue] on network: [blue]{self.network}[/blue]" + ) + + # Check current recycle amount + logging.info("Fetching recycle amount & balance.") + block = self.get_current_block() + + try: + recycle_call = self.get_hyperparameter( + param_name="Burn", netuid=netuid, block=block ) + balance = (self.get_balance(wallet.coldkeypub.ss58_address, block=block),) + except TypeError as e: + logging.error(f"Unable to retrieve current recycle. {e}") + return False + + current_recycle = Balance.from_rao(int(recycle_call)) + + # Check balance is sufficient + if balance < current_recycle: + logging.error( + f"[red]Insufficient balance {balance} to register neuron. " + f"Current recycle is {current_recycle} TAO[/red]." + ) + return False + + # TODO add this extrinsic + return root_register_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) def root_set_weights( @@ -1015,15 +2281,35 @@ def root_set_weights( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.root_set_weights( - wallet=wallet, - netuids=netuids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ), + """ + Set weights for root network. + + Arguments: + wallet (bittensor_wallet.Wallet): bittensor wallet instance. + netuids (list[int]): The list of subnet uids. + weights (list[float]): The list of weights to be set. + version_key (int, optional): Version key for compatibility with the network. Default is ``0``. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. Defaults to + ``False``. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + Defaults to ``False``. + + Returns: + `True` if the setting of weights is successful, `False` otherwise. + """ + netuids_ = np.array(netuids, dtype=np.int64) + weights_ = np.array(weights, dtype=np.float32) + logging.info(f"Setting weights in network: [blue]{self.network}[/blue]") + # Run the set weights operation. + # TODO add this extrinsic + return set_root_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuids=netuids_, + weights=weights_, + version_key=version_key, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, ) def set_weights( @@ -1037,18 +2323,94 @@ def set_weights( wait_for_finalization: bool = False, max_retries: int = 5, ) -> tuple[bool, str]: - return self.execute_coroutine( - self.async_subtensor.set_weights( - wallet=wallet, - netuid=netuid, - uids=uids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - max_retries=max_retries, + """ + Sets the inter-neuronal weights for the specified neuron. This process involves specifying the influence or + trust a neuron places on other neurons in the network, which is a fundamental aspect of Bittensor's + decentralized learning architecture. + + Arguments: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. + netuid (int): The unique identifier of the subnet. + uids (Union[NDArray[np.int64], torch.LongTensor, list]): The list of neuron UIDs that the weights are being + set for. + weights (Union[NDArray[np.float32], torch.FloatTensor, list]): The corresponding weights to be set for each + UID. + version_key (int): Version key for compatibility with the network. Default is int representation of + Bittensor version. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is + ``False``. + max_retries (int): The number of maximum attempts to set weights. Default is ``5``. + + Returns: + tuple[bool, str]: ``True`` if the setting of weights is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + + This function is crucial in shaping the network's collective intelligence, where each neuron's learning and + contribution are influenced by the weights it sets towards others【81†source】. + """ + + def _blocks_weight_limit() -> bool: + bslu = self.blocks_since_last_update(netuid, uid) + wrl = self.weights_rate_limit(netuid) + return bslu > wrl + + retries = 0 + success = False + if ( + uid := self.get_uid_for_hotkey_on_subnet(wallet.hotkey.ss58_address, netuid) + ) is None: + return ( + False, + f"Hotkey {wallet.hotkey.ss58_address} not registered in subnet {netuid}", ) - ) + + if self.commit_reveal_enabled(netuid=netuid) is True: + # go with `commit reveal v3` extrinsic + message = "No attempt made. Perhaps it is too soon to commit weights!" + while retries < max_retries and success is False and _blocks_weight_limit(): + logging.info( + f"Committing weights for subnet #{netuid}. Attempt {retries + 1} of {max_retries}." + ) + # TODO add this extrinsic + success, message = commit_reveal_v3_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + uids=uids, + weights=weights, + version_key=version_key, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + retries += 1 + return success, message + else: + # go with classic `set weights extrinsic` + message = "No attempt made. Perhaps it is too soon to set weights!" + while retries < max_retries and success is False and _blocks_weight_limit(): + try: + logging.info( + f"Setting weights for subnet #[blue]{netuid}[/blue]. " + f"Attempt [blue]{retries + 1} of {max_retries}[/blue]." + ) + # TODO add this extrinsic + success, message = set_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + uids=uids, + weights=weights, + version_key=version_key, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + except Exception as e: + logging.error(f"Error setting weights: {e}") + finally: + retries += 1 + + return success, message def serve_axon( self, @@ -1058,14 +2420,33 @@ def serve_axon( wait_for_finalization: bool = True, certificate: Optional["Certificate"] = None, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.serve_axon( - netuid=netuid, - axon=axon, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - certificate=certificate, - ), + """ + Registers an ``Axon`` serving endpoint on the Bittensor network for a specific neuron. This function is used to + set up the Axon, a key component of a neuron that handles incoming queries and data processing tasks. + + Args: + netuid (int): The unique identifier of the subnetwork. + axon (bittensor.core.axon.Axon): The Axon instance to be registered for serving. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is + ``True``. + certificate (bittensor.utils.Certificate): Certificate to use for TLS. If ``None``, no TLS will be used. + Defaults to ``None``. + + Returns: + bool: ``True`` if the Axon serve registration is successful, False otherwise. + + By registering an Axon, the neuron becomes an active part of the network's distributed computing infrastructure, + contributing to the collective intelligence of Bittensor. + """ + # TODO add extrinsic + return serve_axon_extrinsic( + subtensor=self, + netuid=netuid, + axon=axon, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + certificate=certificate, ) def transfer( @@ -1078,16 +2459,35 @@ def transfer( transfer_all: bool = False, keep_alive: bool = True, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.transfer( - wallet=wallet, - destination=dest, - amount=amount, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - transfer_all=transfer_all, - keep_alive=keep_alive, - ), + """ + Transfer token of amount to destination. + + Arguments: + wallet (bittensor_wallet.Wallet): Source wallet for the transfer. + dest (str): Destination address for the transfer. + amount (float): Amount of tokens to transfer. + transfer_all (bool): Flag to transfer all tokens. Default is ``False``. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``True``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is + ``False``. + keep_alive (bool): Flag to keep the connection alive. Default is ``True``. + + Returns: + `True` if the transferring was successful, otherwise `False`. + """ + if isinstance(amount, float): + amount = Balance.from_tao(amount) + + # TODO add extrinsic, ensure `destination` is changed to `dest` to keep backwards compatibility + return transfer_extrinsic( + subtensor=self, + wallet=wallet, + destination=dest, + amount=amount, + transfer_all=transfer_all, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + keep_alive=keep_alive, ) def unstake( @@ -1098,14 +2498,32 @@ def unstake( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.unstake( - wallet=wallet, - hotkey_ss58=hotkey_ss58, - amount=amount, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ), + """ + Removes a specified amount of stake from a single hotkey account. This function is critical for adjusting + individual neuron stakes within the Bittensor network. + + Args: + wallet (bittensor_wallet.wallet): The wallet associated with the neuron from which the stake is being + removed. + hotkey_ss58 (Optional[str]): The ``SS58`` address of the hotkey account to unstake from. + amount (Union[Balance, float]): The amount of TAO to unstake. If not specified, unstakes all. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + bool: ``True`` if the unstaking process is successful, False otherwise. + + This function supports flexible stake management, allowing neurons to adjust their network participation and + potential reward accruals. + """ + # TODO add extrinsic + return unstake_extrinsic( + subtensor=self, + wallet=wallet, + hotkey_ss58=hotkey_ss58, + amount=amount, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) def unstake_multiple( @@ -1116,12 +2534,31 @@ def unstake_multiple( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return self.execute_coroutine( - self.async_subtensor.unstake_multiple( - wallet=wallet, - hotkey_ss58s=hotkey_ss58s, - amounts=amounts, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ), + """ + Performs batch unstaking from multiple hotkey accounts, allowing a neuron to reduce its staked amounts + efficiently. This function is useful for managing the distribution of stakes across multiple neurons. + + Args: + wallet (bittensor_wallet.Wallet): The wallet linked to the coldkey from which the stakes are being + withdrawn. + hotkey_ss58s (List[str]): A list of hotkey ``SS58`` addresses to unstake from. + amounts (List[Union[Balance, float]]): The amounts of TAO to unstake from each hotkey. If not provided, + unstakes all available stakes. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + bool: ``True`` if the batch unstaking is successful, False otherwise. + + This function allows for strategic reallocation or withdrawal of stakes, aligning with the dynamic stake + management aspect of the Bittensor network. + """ + # TODO add extrinsic + return unstake_multiple_extrinsic( + subtensor=self, + wallet=wallet, + hotkey_ss58s=hotkey_ss58s, + amounts=amounts, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 9de7ba75bd..f1e5ae0034 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -44,6 +44,33 @@ UnlockStatus = namedtuple("UnlockStatus", ["success", "message"]) +def _decode_hex_identity_dict(info_dictionary: dict[str, Any]) -> dict[str, Any]: + # TODO why does this exist alongside `decode_hex_identity_dict`? + """Decodes a dictionary of hexadecimal identities.""" + for k, v in info_dictionary.items(): + if isinstance(v, dict): + item = next(iter(v.values())) + else: + item = v + if isinstance(item, tuple) and item: + if len(item) > 1: + try: + info_dictionary[k] = ( + bytes(item).hex(sep=" ", bytes_per_sep=2).upper() + ) + except UnicodeDecodeError: + logging.error(f"Could not decode: {k}: {item}.") + else: + try: + info_dictionary[k] = bytes(item[0]).decode("utf-8") + except UnicodeDecodeError: + logging.error(f"Could not decode: {k}: {item}.") + else: + info_dictionary[k] = item + + return info_dictionary + + def ss58_to_vec_u8(ss58_address: str) -> list[int]: ss58_bytes: bytes = ss58_address_to_bytes(ss58_address) encoded_address: list[int] = [int(byte) for byte in ss58_bytes] diff --git a/requirements/prod.txt b/requirements/prod.txt index e3fef4c9f2..20af79b989 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -15,6 +15,7 @@ packaging python-statemachine~=2.1 pycryptodome>=3.18.0,<4.0.0 pyyaml +ujson retry requests rich diff --git a/tests/unit_tests/test_async_subtensor.py b/tests/unit_tests/test_async_subtensor.py index c055cd865b..fdbc5b19bb 100644 --- a/tests/unit_tests/test_async_subtensor.py +++ b/tests/unit_tests/test_async_subtensor.py @@ -2495,7 +2495,7 @@ async def test_transfer_success(subtensor, mocker): # Call result = await subtensor.transfer( wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=fake_amount, transfer_all=fake_transfer_all, ) From 72b0f7d6366989b887d004ef7eacd991982b96fe Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 21 Jan 2025 17:58:08 +0200 Subject: [PATCH 09/52] Test stuff --- bittensor/core/dendrite.py | 10 ++++++++- bittensor/core/subtensor.py | 3 +-- bittensor/utils/__init__.py | 4 ---- bittensor/utils/mock/subtensor_mock.py | 5 +---- tests/unit_tests/extrinsics/test_serving.py | 7 ------ tests/unit_tests/test_async_subtensor.py | 2 +- tests/unit_tests/test_metagraph.py | 4 ---- tests/unit_tests/test_subtensor.py | 24 ++++++--------------- 8 files changed, 18 insertions(+), 41 deletions(-) diff --git a/bittensor/core/dendrite.py b/bittensor/core/dendrite.py index 9e56ab4585..6188593a46 100644 --- a/bittensor/core/dendrite.py +++ b/bittensor/core/dendrite.py @@ -31,7 +31,7 @@ from bittensor.core.settings import version_as_int from bittensor.core.stream import StreamingSynapse from bittensor.core.synapse import Synapse, TerminalInfo -from bittensor.utils import networking, event_loop_is_running +from bittensor.utils import networking from bittensor.utils.btlogging import logging from bittensor.utils.registration import torch, use_torch @@ -48,6 +48,14 @@ DENDRITE_DEFAULT_ERROR = ("422", "Failed to parse response") +def event_loop_is_running(): + try: + asyncio.get_running_loop() + return True + except RuntimeError: + return False + + class DendriteMixin: """ The Dendrite class represents the abstracted implementation of a network client module. diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index e4a7278380..5ac86372f1 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -995,7 +995,7 @@ def get_netuids_for_hotkey( ) netuids = [] if result.records: - async for record in result: + for record in result: if record[1].value: netuids.append(record[0]) return netuids @@ -1453,7 +1453,6 @@ def filter_netuids_by_registered_hotkeys( Returns: The filtered list of netuids. """ - # TODO add non-async async stuff here self._get_block_hash(block) # just used to cache the block hash netuids_with_registered_hotkeys = [ item diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index f1e5ae0034..07c3125879 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -6,9 +6,7 @@ import scalecodec from async_substrate_interface.utils import ( - event_loop_is_running, hex_to_bytes, - get_event_loop, ) from bittensor_wallet import Keypair from bittensor_wallet.errors import KeyFileError, PasswordError @@ -31,9 +29,7 @@ check_version = check_version VersionCheckError = VersionCheckError ss58_decode = ss58_decode -event_loop_is_running = event_loop_is_running hex_to_bytes = hex_to_bytes -get_event_loop = get_event_loop RAOPERTAO = 1e9 diff --git a/bittensor/utils/mock/subtensor_mock.py b/bittensor/utils/mock/subtensor_mock.py index ea39e596b6..28769fabe2 100644 --- a/bittensor/utils/mock/subtensor_mock.py +++ b/bittensor/utils/mock/subtensor_mock.py @@ -19,7 +19,7 @@ from bittensor.core.errors import ChainQueryError from bittensor.core.subtensor import Subtensor from bittensor.core.types import AxonServeCallParams, PrometheusServeCallParams -from bittensor.utils import RAOPERTAO, u16_normalized_float, get_event_loop +from bittensor.utils import RAOPERTAO, u16_normalized_float from bittensor.utils.balance import Balance # Mock Testing Constant @@ -250,9 +250,6 @@ def setup(self) -> None: self.network = "mock" self.chain_endpoint = "ws://mock_endpoint.bt" self.substrate = MagicMock(autospec=SubstrateInterface) - self.async_subtensor = AsyncMock(autospec=AsyncSubtensor) - self.async_subtensor.block = ReusableCoroutine(_async_block) - self.event_loop = get_event_loop() def __init__(self, *args, **kwargs) -> None: mock_substrate_interface = MagicMock(autospec=SubstrateInterface) diff --git a/tests/unit_tests/extrinsics/test_serving.py b/tests/unit_tests/extrinsics/test_serving.py index 9b3e2393ea..2793844254 100644 --- a/tests/unit_tests/extrinsics/test_serving.py +++ b/tests/unit_tests/extrinsics/test_serving.py @@ -10,7 +10,6 @@ def test_do_serve_axon(mocker): wait_for_inclusion = True wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(serving, "execute_coroutine") mocked_do_serve_axon = mocker.Mock() serving.async_do_serve_axon = mocked_do_serve_axon @@ -24,11 +23,6 @@ def test_do_serve_axon(mocker): ) # Asserts - - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_do_serve_axon.return_value, - event_loop=fake_subtensor.event_loop, - ) mocked_do_serve_axon.assert_called_once_with( subtensor=fake_subtensor.async_subtensor, wallet=fake_wallet, @@ -36,7 +30,6 @@ def test_do_serve_axon(mocker): wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - assert result == mocked_execute_coroutine.return_value def test_serve_axon_extrinsic(mocker): diff --git a/tests/unit_tests/test_async_subtensor.py b/tests/unit_tests/test_async_subtensor.py index fdbc5b19bb..99ca1c126f 100644 --- a/tests/unit_tests/test_async_subtensor.py +++ b/tests/unit_tests/test_async_subtensor.py @@ -3,7 +3,7 @@ from bittensor.core import async_subtensor from bittensor.core.chain_data import proposal_vote_data -from bittensor.core.subtensor import AsyncSubtensor +from bittensor.core.async_subtensor import AsyncSubtensor @pytest.fixture(autouse=True) diff --git a/tests/unit_tests/test_metagraph.py b/tests/unit_tests/test_metagraph.py index 0a95bbbbc7..bbfd08c04f 100644 --- a/tests/unit_tests/test_metagraph.py +++ b/tests/unit_tests/test_metagraph.py @@ -9,7 +9,6 @@ from bittensor.core import settings from bittensor.core.metagraph import Metagraph from bittensor.core.subtensor import Subtensor -from bittensor.utils import execute_coroutine @pytest.fixture @@ -123,9 +122,6 @@ def mock_subtensor(mocker): get_current_block=mocker.AsyncMock(return_value=601) ) subtensor.event_loop = asyncio.new_event_loop() - subtensor.execute_coroutine = partial( - execute_coroutine, event_loop=subtensor.event_loop - ) return subtensor diff --git a/tests/unit_tests/test_subtensor.py b/tests/unit_tests/test_subtensor.py index bdf6f9720e..b43bd71b85 100644 --- a/tests/unit_tests/test_subtensor.py +++ b/tests/unit_tests/test_subtensor.py @@ -1,4 +1,5 @@ from bittensor.core.subtensor import Subtensor +from bittensor.core.async_subtensor import AsyncSubtensor # TODO: It's probably worth adding a test for each corresponding method to check the correctness of the call with arguments @@ -7,29 +8,16 @@ def test_methods_comparable(mocker): """Verifies that methods in sync and async Subtensors are comparable.""" # Preps - mocker.patch( - "async_substrate_interface.substrate_interface.AsyncSubstrateInterface" - ) - subtensor = Subtensor() + subtensor = Subtensor(_mock=True) + async_subtensor = AsyncSubtensor(_mock=True) - # methods which lives in sync subtensor only - excluded_subtensor_methods = ["async_subtensor", "event_loop", "execute_coroutine"] # methods which lives in async subtensor only - excluded_async_subtensor_methods = [ - "determine_block_hash", - "encode_params", - "get_hyperparameter", - "sign_and_send_extrinsic", - ] - subtensor_methods = [ - m - for m in dir(subtensor) - if not m.startswith("_") and m not in excluded_subtensor_methods - ] + excluded_async_subtensor_methods = ["sign_and_send_extrinsic", "initialize"] + subtensor_methods = [m for m in dir(subtensor) if not m.startswith("_")] async_subtensor_methods = [ m - for m in dir(subtensor.async_subtensor) + for m in dir(async_subtensor) if not m.startswith("_") and m not in excluded_async_subtensor_methods ] From 7489e68d2e8e9840098da291b85edc09e13be6c5 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 21 Jan 2025 19:54:03 +0200 Subject: [PATCH 10/52] Serving extrinsic --- .../core/extrinsics/asyncex/registration.py | 2 +- bittensor/core/extrinsics/asyncex/serving.py | 66 ++-- .../core/extrinsics/asyncex/unstaking.py | 4 +- bittensor/core/extrinsics/serving.py | 311 +++++++++++++++--- bittensor/core/subtensor.py | 25 +- 5 files changed, 330 insertions(+), 78 deletions(-) diff --git a/bittensor/core/extrinsics/asyncex/registration.py b/bittensor/core/extrinsics/asyncex/registration.py index 371b52e834..79520c77c4 100644 --- a/bittensor/core/extrinsics/asyncex/registration.py +++ b/bittensor/core/extrinsics/asyncex/registration.py @@ -106,7 +106,7 @@ async def burned_register_extrinsic( ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or uncluded in the block. If we did not wait for finalization / inclusion, the response is ``true``. + success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. """ if not await subtensor.subnet_exists(netuid): logging.error( diff --git a/bittensor/core/extrinsics/asyncex/serving.py b/bittensor/core/extrinsics/asyncex/serving.py index 1290e53205..189fcee6aa 100644 --- a/bittensor/core/extrinsics/asyncex/serving.py +++ b/bittensor/core/extrinsics/asyncex/serving.py @@ -44,15 +44,29 @@ async def do_serve_axon( """ if call_params["certificate"] is None: - del call_params["certificate"] + call_params_ = { + "version": call_params["version"], + "ip": call_params["ip"], + "port": call_params["port"], + "ip_type": call_params["ip_type"], + "netuid": call_params["netuid"], + } call_function = "serve_axon" else: + call_params_ = { + "version": call_params["version"], + "ip": call_params["ip"], + "port": call_params["port"], + "ip_type": call_params["ip_type"], + "netuid": call_params["netuid"], + "certificate": call_params["certificate"], + } call_function = "serve_axon_tls" call = await subtensor.substrate.compose_call( call_module="SubtensorModule", call_function=call_function, - call_params=call_params, + call_params=call_params_, ) extrinsic = await subtensor.substrate.create_signed_extrinsic( call=call, keypair=wallet.hotkey @@ -95,16 +109,16 @@ async def serve_extrinsic( netuid (int): The network uid to serve on. placeholder1 (int): A placeholder for future use. placeholder2 (int): A placeholder for future use. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or - returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning - ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. certificate (bittensor.utils.Certificate): Certificate to use for TLS. If ``None``, no TLS will be used. Defaults to ``None``. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or uncluded in the block. If we did not wait for - finalization / inclusion, the response is ``true``. + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. """ # Decrypt hotkey if not (unlock := unlock_key(wallet, "hotkey")).success: @@ -187,16 +201,16 @@ async def serve_axon_extrinsic( subtensor (bittensor.core.async_subtensor.AsyncSubtensor): Subtensor instance object. netuid (int): The ``netuid`` being served on. axon (bittensor.core.axon.Axon): Axon to serve. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or - returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning - ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. certificate (bittensor.utils.Certificate): Certificate to use for TLS. If ``None``, no TLS will be used. Defaults to ``None``. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or uncluded in the block. If we did not wait for - finalization / inclusion, the response is ``true``. + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. """ if not (unlock := unlock_key(axon.wallet, "hotkey")).success: logging.error(unlock.message) @@ -282,21 +296,21 @@ async def publish_metadata( }, ) - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, keypair=wallet.hotkey - ) - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - return True + extrinsic = await substrate.create_signed_extrinsic( + call=call, keypair=wallet.hotkey + ) + response = await substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True - if await response.is_success: - return True - raise MetadataError(format_error_message(await response.error_message)) + if await response.is_success: + return True + raise MetadataError(format_error_message(await response.error_message)) async def get_metadata( diff --git a/bittensor/core/extrinsics/asyncex/unstaking.py b/bittensor/core/extrinsics/asyncex/unstaking.py index f9e8fb58a3..e83bcc188e 100644 --- a/bittensor/core/extrinsics/asyncex/unstaking.py +++ b/bittensor/core/extrinsics/asyncex/unstaking.py @@ -103,7 +103,7 @@ async def __do_remove_stake_single( ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or uncluded in the block. If we did not wait for + success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. Raises: @@ -150,7 +150,7 @@ async def unstake_extrinsic( ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or uncluded in the block. If we did not wait for + success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. """ # Decrypt keys, diff --git a/bittensor/core/extrinsics/serving.py b/bittensor/core/extrinsics/serving.py index 04c124bfe0..df73b21acc 100644 --- a/bittensor/core/extrinsics/serving.py +++ b/bittensor/core/extrinsics/serving.py @@ -1,18 +1,20 @@ from typing import Optional, TYPE_CHECKING -from bittensor.core.extrinsics.asyncex.serving import ( - do_serve_axon as async_do_serve_axon, - serve_axon_extrinsic as async_serve_axon_extrinsic, - publish_metadata as async_publish_metadata, - get_metadata as async_get_metadata, +from bittensor.core.errors import MetadataError +from bittensor.core.settings import version_as_int +from bittensor.utils import ( + format_error_message, + networking as net, + unlock_key, + Certificate, ) +from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.axon import Axon from bittensor.core.subtensor import Subtensor from bittensor.core.types import AxonServeCallParams - from bittensor.utils import Certificate def do_serve_axon( @@ -22,16 +24,166 @@ def do_serve_axon( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> tuple[bool, Optional[dict]]: - return subtensor.execute_coroutine( - coroutine=async_do_serve_axon( - subtensor=subtensor.async_subtensor, - wallet=wallet, - call_params=call_params, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + """ + Internal method to submit a serve axon transaction to the Bittensor blockchain. This method creates and submits a + transaction, enabling a neuron's ``Axon`` to serve requests on the network. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance object. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron. + call_params (bittensor.core.types.AxonServeCallParams): Parameters required for the serve axon call. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + + This function is crucial for initializing and announcing a neuron's ``Axon`` service on the network, enhancing the + decentralized computation capabilities of Bittensor. + """ + if call_params["certificate"] is None: + call_params_ = { + "version": call_params["version"], + "ip": call_params["ip"], + "port": call_params["port"], + "ip_type": call_params["ip_type"], + "netuid": call_params["netuid"], + } + call_function = "serve_axon" + else: + call_params_ = { + "version": call_params["version"], + "ip": call_params["ip"], + "port": call_params["port"], + "ip_type": call_params["ip_type"], + "netuid": call_params["netuid"], + "certificate": call_params["certificate"], + } + call_function = "serve_axon_tls" + + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function=call_function, + call_params=call_params_, + ) + extrinsic = subtensor.substrate.create_signed_extrinsic( + call=call, keypair=wallet.hotkey + ) + response = subtensor.substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + if wait_for_inclusion or wait_for_finalization: + if response.is_success: + return True, None + + return False, response.error_message + + return True, None + + +def serve_extrinsic( + subtensor: "Subtensor", + wallet: "Wallet", + ip: str, + port: int, + protocol: int, + netuid: int, + placeholder1: int = 0, + placeholder2: int = 0, + wait_for_inclusion: bool = False, + wait_for_finalization=True, + certificate: Optional[Certificate] = None, +) -> bool: + """Subscribes a Bittensor endpoint to the subtensor chain. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance object. + wallet (bittensor_wallet.Wallet): Bittensor wallet object. + ip (str): Endpoint host port i.e., ``192.122.31.4``. + port (int): Endpoint port number i.e., ``9221``. + protocol (int): An ``int`` representation of the protocol. + netuid (int): The network uid to serve on. + placeholder1 (int): A placeholder for future use. + placeholder2 (int): A placeholder for future use. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. + certificate (bittensor.utils.Certificate): Certificate to use for TLS. If ``None``, no TLS will be used. + Defaults to ``None``. + + Returns: + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. + """ + # Decrypt hotkey + if not (unlock := unlock_key(wallet, "hotkey")).success: + logging.error(unlock.message) + return False + + params: "AxonServeCallParams" = { + "version": version_as_int, + "ip": net.ip_to_int(ip), + "port": port, + "ip_type": net.ip_version(ip), + "netuid": netuid, + "hotkey": wallet.hotkey.ss58_address, + "coldkey": wallet.coldkeypub.ss58_address, + "protocol": protocol, + "placeholder1": placeholder1, + "placeholder2": placeholder2, + "certificate": certificate, + } + logging.debug("Checking axon ...") + neuron = subtensor.get_neuron_for_pubkey_and_subnet( + wallet.hotkey.ss58_address, netuid=netuid + ) + neuron_up_to_date = not neuron.is_null and params == { + "version": neuron.axon_info.version, + "ip": net.ip_to_int(neuron.axon_info.ip), + "port": neuron.axon_info.port, + "ip_type": neuron.axon_info.ip_type, + "netuid": neuron.netuid, + "hotkey": neuron.hotkey, + "coldkey": neuron.coldkey, + "protocol": neuron.axon_info.protocol, + "placeholder1": neuron.axon_info.placeholder1, + "placeholder2": neuron.axon_info.placeholder2, + } + output = params.copy() + output["coldkey"] = wallet.coldkeypub.ss58_address + output["hotkey"] = wallet.hotkey.ss58_address + if neuron_up_to_date: + logging.debug( + f"Axon already served on: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) " ) + return True + + logging.debug( + f"Serving axon with: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) -> {subtensor.network}:{netuid}" + ) + success, error_message = do_serve_axon( + subtensor=subtensor, + wallet=wallet, + call_params=params, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, ) + if wait_for_inclusion or wait_for_finalization: + if success is True: + logging.debug( + f"Axon served with: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) on {subtensor.network}:{netuid} " + ) + return True + else: + logging.error(f"Failed: {format_error_message(error_message)}") + return False + else: + return True + def serve_axon_extrinsic( subtensor: "Subtensor", @@ -41,16 +193,55 @@ def serve_axon_extrinsic( wait_for_finalization: bool = True, certificate: Optional["Certificate"] = None, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_serve_axon_extrinsic( - subtensor=subtensor.async_subtensor, - netuid=netuid, - axon=axon, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - certificate=certificate, - ) + """Serves the axon to the network. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance object. + netuid (int): The ``netuid`` being served on. + axon (bittensor.core.axon.Axon): Axon to serve. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. + certificate (bittensor.utils.Certificate): Certificate to use for TLS. If ``None``, no TLS will be used. + Defaults to ``None``. + + Returns: + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``true``. + """ + if not (unlock := unlock_key(axon.wallet, "hotkey")).success: + logging.error(unlock.message) + return False + external_port = axon.external_port + + # ---- Get external ip ---- + if axon.external_ip is None: + try: + external_ip = net.get_external_ip + logging.success( + f":white_heavy_check_mark: [green]Found external ip:[/green] [blue]{external_ip}[/blue]" + ) + except Exception as e: + raise RuntimeError( + f"Unable to attain your external ip. Check your internet connection. error: {e}" + ) from e + else: + external_ip = axon.external_ip + + # ---- Subscribe to chain ---- + serve_success = serve_extrinsic( + subtensor=subtensor, + wallet=axon.wallet, + ip=external_ip, + port=external_port, + protocol=4, + netuid=netuid, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + certificate=certificate, ) + return serve_success def publish_metadata( @@ -62,27 +253,69 @@ def publish_metadata( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_publish_metadata( - subtensor=subtensor.async_subtensor, - wallet=wallet, - netuid=netuid, - data_type=data_type, - data=data, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) + """ + Publishes metadata on the Bittensor network using the specified wallet and network identifier. + + Args: + subtensor (bittensor.subtensor): The subtensor instance representing the Bittensor blockchain connection. + wallet (bittensor.wallet): The wallet object used for authentication in the transaction. + netuid (int): Network UID on which the metadata is to be published. + data_type (str): The data type of the information being submitted. It should be one of the following: + ``'Sha256'``, ``'Blake256'``, ``'Keccak256'``, or ``'Raw0-128'``. This specifies the format or hashing + algorithm used for the data. + data (str): The actual metadata content to be published. This should be formatted or hashed according to the + ``type`` specified. (Note: max ``str`` length is 128 bytes) + wait_for_inclusion (bool, optional): If ``True``, the function will wait for the extrinsic to be included in a + block before returning. Defaults to ``False``. + wait_for_finalization (bool, optional): If ``True``, the function will wait for the extrinsic to be finalized + on the chain before returning. Defaults to ``True``. + + Returns: + bool: ``True`` if the metadata was successfully published (and finalized if specified). ``False`` otherwise. + + Raises: + MetadataError: If there is an error in submitting the extrinsic or if the response from the blockchain indicates + failure. + """ + + if not (unlock := unlock_key(wallet, "hotkey")).success: + logging.error(unlock.message) + return False + + call = subtensor.substrate.compose_call( + call_module="Commitments", + call_function="set_commitment", + call_params={ + "netuid": netuid, + "info": {"fields": [[{f"{data_type}": data}]]}, + }, ) + extrinsic = subtensor.substrate.create_signed_extrinsic( + call=call, keypair=wallet.hotkey + ) + response = subtensor.substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True + + if response.is_success: + return True + raise MetadataError(format_error_message(response.error_message)) + def get_metadata( subtensor: "Subtensor", netuid: int, hotkey: str, block: Optional[int] = None ) -> str: - return subtensor.execute_coroutine( - coroutine=async_get_metadata( - subtensor=subtensor.async_subtensor, - netuid=netuid, - hotkey=hotkey, - block=block, - ) + """Fetches metadata from the blockchain for a given hotkey and netuid.""" + commit_data = subtensor.substrate.query( + module="Commitments", + storage_function="CommitmentOf", + params=[netuid, hotkey], + block_hash=subtensor.determine_block_hash(block), ) + return getattr(commit_data, "value", None) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 5ac86372f1..98f6187261 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -2,6 +2,7 @@ from functools import lru_cache from typing import TYPE_CHECKING, Any, Iterable, Optional, Union +# TODO clean up this import section import numpy as np import ujson from async_substrate_interface.errors import SubstrateRequestException @@ -20,6 +21,10 @@ WeightCommitInfo, ) from bittensor.core.metagraph import Metagraph +from bittensor.core.extrinsics.serving import ( + publish_metadata, + get_metadata, +) from bittensor.core.settings import ( version_as_int, SS58_FORMAT, @@ -37,19 +42,19 @@ ) from bittensor.utils.btlogging import logging from bittensor.utils.weight_utils import generate_weight_hash +from bittensor.core.async_subtensor import ProposalVoteData +from bittensor.core.axon import Axon +from bittensor.core.config import Config +from bittensor.core.chain_data.delegate_info import DelegateInfo +from bittensor.core.chain_data.neuron_info import NeuronInfo +from bittensor.core.chain_data.neuron_info_lite import NeuronInfoLite +from bittensor.core.chain_data.stake_info import StakeInfo +from bittensor.core.chain_data.subnet_hyperparameters import SubnetHyperparameters +from bittensor.core.chain_data.subnet_info import SubnetInfo +from bittensor.utils.balance import Balance if TYPE_CHECKING: from bittensor_wallet import Wallet - from bittensor.core.async_subtensor import ProposalVoteData - from bittensor.core.axon import Axon - from bittensor.core.config import Config - from bittensor.core.chain_data.delegate_info import DelegateInfo - from bittensor.core.chain_data.neuron_info import NeuronInfo - from bittensor.core.chain_data.neuron_info_lite import NeuronInfoLite - from bittensor.core.chain_data.stake_info import StakeInfo - from bittensor.core.chain_data.subnet_hyperparameters import SubnetHyperparameters - from bittensor.core.chain_data.subnet_info import SubnetInfo - from bittensor.utils.balance import Balance from bittensor.utils import Certificate from async_substrate_interface.sync_substrate import QueryMapResult from bittensor.utils.delegates_details import DelegatesDetails From b79ffad4d5b22840d4188eb285833671f4799f6b Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 21 Jan 2025 22:09:00 +0200 Subject: [PATCH 11/52] Staking extrinsics --- bittensor/core/extrinsics/asyncex/staking.py | 17 +- bittensor/core/extrinsics/staking.py | 386 +++++++++++++++++-- bittensor/core/subtensor.py | 62 ++- 3 files changed, 429 insertions(+), 36 deletions(-) diff --git a/bittensor/core/extrinsics/asyncex/staking.py b/bittensor/core/extrinsics/asyncex/staking.py index 2cba0cb92c..0396bf76a6 100644 --- a/bittensor/core/extrinsics/asyncex/staking.py +++ b/bittensor/core/extrinsics/asyncex/staking.py @@ -162,7 +162,8 @@ async def add_stake_extrinsic( logging.success(":white_heavy_check_mark: [green]Finalized[/green]") logging.info( - f":satellite: [magenta]Checking Balance on:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + f":satellite: [magenta]Checking Balance on:[/magenta] [blue]{subtensor.network}[/blue] " + "[magenta]...[/magenta]" ) new_block_hash = await subtensor.substrate.get_chain_head() new_balance, new_stake = await asyncio.gather( @@ -270,6 +271,10 @@ async def add_stake_multiple_extrinsic( total_staking_rao = sum( [amount.rao if amount is not None else 0 for amount in new_amounts] ) + if old_balance is None: + old_balance = await subtensor.get_balance( + wallet.coldkeypub.ss58_address, block_hash=block_hash + ) if total_staking_rao == 0: # Staking all to the first wallet. if old_balance.rao > 1000: @@ -305,7 +310,8 @@ async def add_stake_multiple_extrinsic( # Check enough to stake if staking_balance > old_balance: logging.error( - f":cross_mark: [red]Not enough balance[/red]: [green]{old_balance}[/green] to stake: [blue]{staking_balance}[/blue] from wallet: [white]{wallet.name}[/white]" + f":cross_mark: [red]Not enough balance[/red]: [green]{old_balance}[/green] to stake: " + f"[blue]{staking_balance}[/blue] from wallet: [white]{wallet.name}[/white]" ) continue @@ -328,9 +334,7 @@ async def add_stake_multiple_extrinsic( if idx < len(hotkey_ss58s) - 1: # Wait for tx rate limit. tx_query = await subtensor.substrate.query( - module="SubtensorModule", - storage_function="TxRateLimit", - block_hash=block_hash, + module="SubtensorModule", storage_function="TxRateLimit" ) tx_rate_limit_blocks: int = tx_query if tx_rate_limit_blocks > 0: @@ -391,7 +395,8 @@ async def add_stake_multiple_extrinsic( if successful_stakes != 0: logging.info( - f":satellite: [magenta]Checking Balance on:[/magenta] ([blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + f":satellite: [magenta]Checking Balance on:[/magenta] ([blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" ) new_balance = await subtensor.get_balance(wallet.coldkeypub.ss58_address) logging.info( diff --git a/bittensor/core/extrinsics/staking.py b/bittensor/core/extrinsics/staking.py index ea127327ed..9e85613bb4 100644 --- a/bittensor/core/extrinsics/staking.py +++ b/bittensor/core/extrinsics/staking.py @@ -1,14 +1,14 @@ -from typing import Union, Optional, TYPE_CHECKING +import time +from typing import Union, Optional, TYPE_CHECKING, Sequence -from bittensor.core.extrinsics.asyncex.staking import ( - add_stake_extrinsic as async_add_stake_extrinsic, - add_stake_multiple_extrinsic as async_add_stake_multiple_extrinsic, -) +from bittensor.core.errors import StakeError, NotRegisteredError +from bittensor.utils import unlock_key +from bittensor.utils.balance import Balance +from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor - from bittensor.utils.balance import Balance def add_stake_extrinsic( @@ -19,16 +19,167 @@ def add_stake_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_add_stake_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - hotkey_ss58=hotkey_ss58, - amount=amount, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) + """ + Adds the specified amount of stake to passed hotkey `uid`. + + Arguments: + subtensor: the Subtensor object to use + wallet: Bittensor wallet object. + hotkey_ss58: The `ss58` address of the hotkey account to stake to defaults to the wallet's hotkey. + amount: Amount to stake as Bittensor balance, `None` if staking all. + wait_for_inclusion: If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization: If set, waits for the extrinsic to be finalized on the chain before returning `True`, + or returns `False` if the extrinsic fails to be finalized within the timeout. + + Returns: + success: Flag is `True` if extrinsic was finalized or included in the block. If we did not wait for + finalization/inclusion, the response is `True`. + """ + + def _check_threshold_amount( + balance: "Balance", + block_hash: str, + min_req_stake: Optional["Balance"] = None, + ) -> tuple[bool, "Balance"]: + """Checks if the new stake balance will be above the minimum required stake threshold.""" + if not min_req_stake: + min_req_stake_ = subtensor.substrate.query( + module="SubtensorModule", + storage_function="NominatorMinRequiredStake", + block_hash=block_hash, + ) + min_req_stake = Balance.from_rao(min_req_stake_) + if min_req_stake > balance: + return False, min_req_stake + return True, min_req_stake + + # Decrypt keys, + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + # Default to wallet's own hotkey if the value is not passed. + if hotkey_ss58 is None: + hotkey_ss58 = wallet.hotkey.ss58_address + + # Flag to indicate if we are using the wallet's own hotkey. + own_hotkey: bool + + logging.info( + f":satellite: [magenta]Syncing with chain:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + ) + old_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address) + block = subtensor.get_current_block() + + # Get hotkey owner + hotkey_owner = subtensor.get_hotkey_owner(hotkey_ss58=hotkey_ss58, block=block) + own_hotkey = wallet.coldkeypub.ss58_address == hotkey_owner + if not own_hotkey: + # This is not the wallet's own hotkey, so we are delegating. + if not subtensor.is_hotkey_delegate(hotkey_ss58, block=block): + logging.debug(f"Hotkey {hotkey_ss58} is not a delegate on the chain.") + return False + + # Get current stake and existential deposit + old_stake = subtensor.get_stake_for_coldkey_and_hotkey( + coldkey_ss58=wallet.coldkeypub.ss58_address, + hotkey_ss58=hotkey_ss58, + block=block, ) + existential_deposit = subtensor.get_existential_deposit(block=block) + + # Convert to bittensor.Balance + if amount is None: + # Stake it all. + staking_balance = Balance.from_tao(old_balance.tao) + else: + staking_balance = Balance.from_tao(amount.tao) + + # Leave existential balance to keep key alive. + if staking_balance > old_balance - existential_deposit: + # If we are staking all, we need to leave at least the existential deposit. + staking_balance = old_balance - existential_deposit + else: + staking_balance = staking_balance + + # Check enough to stake. + if staking_balance > old_balance: + logging.error(":cross_mark: [red]Not enough stake:[/red]") + logging.error(f"\t\tbalance:{old_balance}") + logging.error(f"\t\tamount: {staking_balance}") + logging.error(f"\t\twallet: {wallet.name}") + return False + + # If nominating, we need to check if the new stake balance will be above the minimum required stake threshold. + if not own_hotkey: + new_stake_balance = old_stake + staking_balance + is_above_threshold, threshold = _check_threshold_amount( + new_stake_balance, block_hash=subtensor.get_block_hash(block) + ) + if not is_above_threshold: + logging.error( + f":cross_mark: [red]New stake balance of {new_stake_balance} is below the minimum required " + f"nomination stake threshold {threshold}.[/red]" + ) + return False + + try: + logging.info( + f":satellite: [magenta]Staking to:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + ) + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="add_stake", + call_params={"hotkey": hotkey_ss58, "amount_staked": staking_balance.rao}, + ) + staking_response, err_msg = subtensor.sign_and_send_extrinsic( + call, wallet, wait_for_inclusion, wait_for_finalization + ) + if staking_response is True: # If we successfully staked. + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True + + logging.success(":white_heavy_check_mark: [green]Finalized[/green]") + + logging.info( + f":satellite: [magenta]Checking Balance on:[/magenta] [blue]{subtensor.network}[/blue] " + "[magenta]...[/magenta]" + ) + new_block = subtensor.get_current_block() + new_balance = subtensor.get_balance( + wallet.coldkeypub.ss58_address, block=new_block + ) + new_stake = subtensor.get_stake_for_coldkey_and_hotkey( + coldkey_ss58=wallet.coldkeypub.ss58_address, + hotkey_ss58=hotkey_ss58, + block=new_block, + ) + logging.info("Balance:") + logging.info( + f"[blue]{old_balance}[/blue] :arrow_right: {new_balance}[/green]" + ) + logging.info("Stake:") + logging.info( + f"[blue]{old_stake}[/blue] :arrow_right: [green]{new_stake}[/green]" + ) + return True + else: + logging.error(":cross_mark: [red]Failed[/red]: Error unknown.") + return False + + # TODO I don't think these are used. Maybe should just catch SubstrateRequestException? + except NotRegisteredError: + logging.error( + ":cross_mark: [red]Hotkey: {} is not registered.[/red]".format( + wallet.hotkey_str + ) + ) + return False + except StakeError as e: + logging.error(f":cross_mark: [red]Stake Error: {e}[/red]") + return False def add_stake_multiple_extrinsic( @@ -39,13 +190,200 @@ def add_stake_multiple_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_add_stake_multiple_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - hotkey_ss58s=hotkey_ss58s, - amounts=amounts, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) + """Adds stake to each ``hotkey_ss58`` in the list, using each amount, from a common coldkey. + + Arguments: + subtensor: The initialized SubtensorInterface object. + wallet: Bittensor wallet object for the coldkey. + hotkey_ss58s: List of hotkeys to stake to. + amounts: List of amounts to stake. If `None`, stake all to the first hotkey. + wait_for_inclusion: If set, waits for the extrinsic to enter a block before returning `True`, or returns `False` + if the extrinsic fails to enter the block within the timeout. + wait_for_finalization: If set, waits for the extrinsic to be finalized on the chain before returning `True`, or + returns `False` if the extrinsic fails to be finalized within the timeout. + + Returns: + success: `True` if extrinsic was finalized or included in the block. `True` if any wallet was staked. If we did + not wait for finalization/inclusion, the response is `True`. + """ + + def get_old_stakes(block_hash: str) -> dict[str, Balance]: + calls = [ + ( + subtensor.substrate.create_storage_key( + "SubtensorModule", + "Stake", + [hotkey_ss58, wallet.coldkeypub.ss58_address], + block_hash=block_hash, + ) + ) + for hotkey_ss58 in hotkey_ss58s + ] + batch_call = subtensor.substrate.query_multi(calls, block_hash=block_hash) + results = {} + for item in batch_call: + results.update({item[0].params[0]: Balance.from_rao(item[1] or 0)}) + return results + + if not isinstance(hotkey_ss58s, list) or not all( + isinstance(hotkey_ss58, str) for hotkey_ss58 in hotkey_ss58s + ): + raise TypeError("hotkey_ss58s must be a list of str") + + if len(hotkey_ss58s) == 0: + return True + + if amounts is not None and len(amounts) != len(hotkey_ss58s): + raise ValueError("amounts must be a list of the same length as hotkey_ss58s") + + new_amounts: Sequence[Optional[Balance]] + if amounts is None: + new_amounts = [None] * len(hotkey_ss58s) + else: + new_amounts = [Balance.from_tao(amount) for amount in amounts] + if sum(amount.tao for amount in new_amounts) == 0: + # Staking 0 tao + return True + + # Decrypt keys, + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + logging.info( + f":satellite: [magenta]Syncing with chain:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + ) + block = subtensor.get_current_block() + old_stakes: dict[str, Balance] = get_old_stakes(subtensor.get_block_hash(block)) + + # Remove existential balance to keep key alive. + # Keys must maintain a balance of at least 1000 rao to stay alive. + total_staking_rao = sum( + [amount.rao if amount is not None else 0 for amount in new_amounts] ) + old_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address, block=block) + if total_staking_rao == 0: + # Staking all to the first wallet. + if old_balance.rao > 1000: + old_balance -= Balance.from_rao(1000) + + elif total_staking_rao < 1000: + # Staking less than 1000 rao to the wallets. + pass + else: + # Staking more than 1000 rao to the wallets. + # Reduce the amount to stake to each wallet to keep the balance above 1000 rao. + percent_reduction = 1 - (1000 / total_staking_rao) + new_amounts = [ + Balance.from_tao(amount.tao * percent_reduction) for amount in new_amounts + ] + + successful_stakes = 0 + for idx, (hotkey_ss58, amount) in enumerate(zip(hotkey_ss58s, new_amounts)): + staking_all = False + # Convert to bittensor.Balance + if amount is None: + # Stake it all. + staking_balance = Balance.from_tao(old_balance.tao) + staking_all = True + else: + # Amounts are cast to balance earlier in the function + assert isinstance(amount, Balance) + staking_balance = amount + + # Check enough to stake + if staking_balance > old_balance: + logging.error( + f":cross_mark: [red]Not enough balance[/red]: [green]{old_balance}[/green] to stake: " + f"[blue]{staking_balance}[/blue] from wallet: [white]{wallet.name}[/white]" + ) + continue + + try: + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="add_stake", + call_params={ + "hotkey": hotkey_ss58, + "amount_staked": staking_balance.rao, + }, + ) + staking_response, err_msg = subtensor.sign_and_send_extrinsic( + call, wallet, wait_for_inclusion, wait_for_finalization + ) + + if staking_response is True: # If we successfully staked. + # We only wait here if we expect finalization. + + if idx < len(hotkey_ss58s) - 1: + # Wait for tx rate limit. + tx_query = subtensor.substrate.query( + module="SubtensorModule", storage_function="TxRateLimit" + ) + tx_rate_limit_blocks: int = tx_query + if tx_rate_limit_blocks > 0: + logging.error( + f":hourglass: [yellow]Waiting for tx rate limit: [white]{tx_rate_limit_blocks}[/white] " + f"blocks[/yellow]" + ) + # 12 seconds per block + time.sleep(tx_rate_limit_blocks * 12) + + if not wait_for_finalization and not wait_for_inclusion: + old_balance -= staking_balance + successful_stakes += 1 + if staking_all: + # If staked all, no need to continue + break + + continue + + logging.success(":white_heavy_check_mark: [green]Finalized[/green]") + + new_block = subtensor.get_current_block() + new_stake = subtensor.get_stake_for_coldkey_and_hotkey( + coldkey_ss58=wallet.coldkeypub.ss58_address, + hotkey_ss58=hotkey_ss58, + block=new_block, + ) + new_balance = subtensor.get_balance( + wallet.coldkeypub.ss58_address, block=new_block + ) + logging.info( + "Stake ({}): [blue]{}[/blue] :arrow_right: [green]{}[/green]".format( + hotkey_ss58, old_stakes[hotkey_ss58], new_stake + ) + ) + old_balance = new_balance + successful_stakes += 1 + if staking_all: + # If staked all, no need to continue + break + + else: + logging.error(":cross_mark: [red]Failed[/red]: Error unknown.") + continue + + except NotRegisteredError: + logging.error( + ":cross_mark: [red]Hotkey: {} is not registered.[/red]".format( + hotkey_ss58 + ) + ) + continue + except StakeError as e: + logging.error(":cross_mark: [red]Stake Error: {}[/red]".format(e)) + continue + + if successful_stakes != 0: + logging.info( + f":satellite: [magenta]Checking Balance on:[/magenta] ([blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" + ) + new_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address) + logging.info( + f"Balance: [blue]{old_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + ) + return True + + return False diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 98f6187261..b039f0a0ac 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -20,10 +20,15 @@ decode_account_id, WeightCommitInfo, ) +from bittensor.core.extrinsics.staking import ( + add_stake_extrinsic, + add_stake_multiple_extrinsic, +) from bittensor.core.metagraph import Metagraph from bittensor.core.extrinsics.serving import ( publish_metadata, get_metadata, + serve_axon_extrinsic, ) from bittensor.core.settings import ( version_as_int, @@ -58,7 +63,7 @@ from bittensor.utils import Certificate from async_substrate_interface.sync_substrate import QueryMapResult from bittensor.utils.delegates_details import DelegatesDetails - from scalecodec.types import ScaleType + from scalecodec.types import ScaleType, GenericCall class Subtensor(SubtensorMixin): @@ -402,7 +407,6 @@ def commit(self, wallet, netuid: int, data: str) -> bool: netuid (int): The unique identifier of the subnetwork. data (str): The data to be committed to the network. """ - # TODO add return publish_metadata( subtensor=self, wallet=wallet, @@ -701,7 +705,7 @@ def get_commitment(self, netuid: int, uid: int, block: Optional[int] = None) -> ) return "" - metadata = get_metadata(self, netuid, hotkey, block) # TODO add + metadata = get_metadata(self, netuid, hotkey, block) try: commitment = metadata["info"]["fields"][0] # type: ignore hex_data = commitment[list(commitment.keys())[0]][2:] # type: ignore @@ -1918,6 +1922,55 @@ def weights_rate_limit( ) return None if call is None else int(call) + # Extrinsics helper ================================================================================================ + + def sign_and_send_extrinsic( + self, + call: "GenericCall", + wallet: "Wallet", + wait_for_inclusion: bool = True, + wait_for_finalization: bool = False, + sign_with: str = "coldkey", + ) -> tuple[bool, str]: + """ + Helper method to sign and submit an extrinsic call to chain. + + Arguments: + call (scalecodec.types.GenericCall): a prepared Call object + wallet (bittensor_wallet.Wallet): the wallet whose coldkey will be used to sign the extrinsic + wait_for_inclusion (bool): whether to wait until the extrinsic call is included on the chain + wait_for_finalization (bool): whether to wait until the extrinsic call is finalized on the chain + sign_with: the wallet's keypair to use for the signing. Options are "coldkey", "hotkey", "coldkeypub" + + Returns: + (success, error message) + """ + if sign_with not in ("coldkey", "hotkey", "coldkeypub"): + raise AttributeError( + f"'sign_with' must be either 'coldkey', 'hotkey' or 'coldkeypub', not '{sign_with}'" + ) + + extrinsic = self.substrate.create_signed_extrinsic( + call=call, keypair=getattr(wallet, sign_with) + ) + try: + response = self.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, "" + + if response.is_success: + return True, "" + + return False, format_error_message(response.error_message) + + except SubstrateRequestException as e: + return False, format_error_message(e) + # Extrinsics ======================================================================================================= def add_stake( @@ -1946,7 +1999,6 @@ def add_stake( This function enables neurons to increase their stake in the network, enhancing their influence and potential rewards in line with Bittensor's consensus and reward mechanisms. """ - # TODO add this extrinsic return add_stake_extrinsic( subtensor=self, wallet=wallet, @@ -1981,7 +2033,6 @@ def add_stake_multiple( This function is essential for managing stakes across multiple neurons, reflecting the dynamic and collaborative nature of the Bittensor network. """ - # TODO add this extrinsic return add_stake_multiple_extrinsic( subtensor=self, wallet=wallet, @@ -2443,7 +2494,6 @@ def serve_axon( By registering an Axon, the neuron becomes an active part of the network's distributed computing infrastructure, contributing to the collective intelligence of Bittensor. """ - # TODO add extrinsic return serve_axon_extrinsic( subtensor=self, netuid=netuid, From b2dea07b6c9cb6d06a9022c1819ca290ab86e420 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 21 Jan 2025 22:57:15 +0200 Subject: [PATCH 12/52] Registration extrinsics --- bittensor/core/errors.py | 8 + .../core/extrinsics/asyncex/registration.py | 125 +++--- bittensor/core/extrinsics/registration.py | 358 ++++++++++++++++-- bittensor/core/subtensor.py | 6 +- bittensor/utils/registration/__init__.py | 11 + bittensor/utils/registration/async_pow.py | 7 +- bittensor/utils/registration/pow.py | 2 +- 7 files changed, 404 insertions(+), 113 deletions(-) diff --git a/bittensor/core/errors.py b/bittensor/core/errors.py index 7abe28e8b3..9f856d15e6 100644 --- a/bittensor/core/errors.py +++ b/bittensor/core/errors.py @@ -17,6 +17,14 @@ ExtrinsicNotFound = ExtrinsicNotFound +class MaxSuccessException(Exception): + """Raised when the POW Solver has reached the max number of successful solutions.""" + + +class MaxAttemptsException(Exception): + """Raised when the POW Solver has reached the max number of attempts.""" + + class ChainError(SubstrateRequestException): """Base error for any chain related errors.""" diff --git a/bittensor/core/extrinsics/asyncex/registration.py b/bittensor/core/extrinsics/asyncex/registration.py index 79520c77c4..d4b5c6a271 100644 --- a/bittensor/core/extrinsics/asyncex/registration.py +++ b/bittensor/core/extrinsics/asyncex/registration.py @@ -10,28 +10,14 @@ import asyncio from typing import Optional, Union, TYPE_CHECKING -from bittensor.utils import format_error_message from bittensor.utils import unlock_key from bittensor.utils.btlogging import logging -from bittensor.utils.registration import log_no_torch_error, create_pow_async +from bittensor.utils.registration import log_no_torch_error, create_pow_async, torch if TYPE_CHECKING: - import torch from bittensor_wallet import Wallet from bittensor.core.async_subtensor import AsyncSubtensor from bittensor.utils.registration.pow import POWSolution -else: - from bittensor.utils.registration.pow import LazyLoadedTorch - - torch = LazyLoadedTorch() - - -class MaxSuccessException(Exception): - """Raised when the POW Solver has reached the max number of successful solutions.""" - - -class MaxAttemptsException(Exception): - """Raised when the POW Solver has reached the max number of attempts.""" async def _do_burned_register( @@ -40,21 +26,22 @@ async def _do_burned_register( wallet: "Wallet", wait_for_inclusion: bool = False, wait_for_finalization: bool = True, -) -> tuple[bool, Optional[str]]: +) -> tuple[bool, str]: """ Performs a burned register extrinsic call to the Subtensor chain. This method sends a registration transaction to the Subtensor blockchain using the burned register mechanism. Args: - subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): Subtensor instance. netuid (int): The network unique identifier to register on. wallet (bittensor_wallet.Wallet): The wallet to be registered. wait_for_inclusion (bool): Whether to wait for the transaction to be included in a block. Default is False. wait_for_finalization (bool): Whether to wait for the transaction to be finalized. Default is True. Returns: - Tuple[bool, Optional[str]]: A tuple containing a boolean indicating success or failure, and an optional error message. + Tuple[bool, Optional[str]]: A tuple containing a boolean indicating success or failure, and an optional error + message. """ # create extrinsic call @@ -66,26 +53,13 @@ async def _do_burned_register( "hotkey": wallet.hotkey.ss58_address, }, ) - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, keypair=wallet.coldkey - ) - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, + return await subtensor.sign_and_send_extrinsic( + call=call, + wallet=wallet, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - return True, None - - # process if registration successful, try again if pow is still valid - if not await response.is_success: - return False, format_error_message(await response.error_message) - # Successful registration - - return True, None - async def burned_register_extrinsic( subtensor: "AsyncSubtensor", @@ -97,18 +71,20 @@ async def burned_register_extrinsic( """Registers the wallet to chain by recycling TAO. Args: - subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): Subtensor instance. wallet (bittensor.wallet): Bittensor wallet object. netuid (int): The ``netuid`` of the subnet to register on. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or - returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning - ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. """ - if not await subtensor.subnet_exists(netuid): + block_hash = await subtensor.substrate.get_chain_head() + if not await subtensor.subnet_exists(netuid, block_hash=block_hash): logging.error( f":cross_mark: [red]Failed error:[/red] subnet [blue]{netuid}[/blue] does not exist." ) @@ -121,11 +97,17 @@ async def burned_register_extrinsic( logging.info( f":satellite: [magenta]Checking Account on subnet[/magenta] [blue]{netuid}[/blue][magenta] ...[/magenta]" ) - neuron = await subtensor.get_neuron_for_pubkey_and_subnet( - wallet.hotkey.ss58_address, netuid=netuid - ) - old_balance = await subtensor.get_balance(wallet.coldkeypub.ss58_address) + # We could do this as_completed because we don't actually need old_balance and recycle + # if neuron is null, but the complexity isn't worth it considering the small performance + # gains we'd hypothetically receive in this situation + neuron, old_balance, recycle_amount = await asyncio.gather( + subtensor.get_neuron_for_pubkey_and_subnet( + wallet.hotkey.ss58_address, netuid=netuid, block_hash=block_hash + ), + subtensor.get_balance(wallet.coldkeypub.ss58_address, block_hash=block_hash), + subtensor.recycle(netuid=netuid, block_hash=block_hash), + ) if not neuron.is_null: logging.info(":white_heavy_check_mark: [green]Already Registered[/green]") @@ -135,9 +117,7 @@ async def burned_register_extrinsic( logging.info(f"\t\tcoldkey: [blue]{neuron.coldkey}[/blue]") return True - logging.info(":satellite: [magenta]Recycling TAO for Registration...[/magenta]") - - recycle_amount = await subtensor.recycle(netuid=netuid) + logging.debug(":satellite: [magenta]Recycling TAO for Registration...[/magenta]") logging.info(f"Recycling {recycle_amount} to register on subnet:{netuid}") success, err_msg = await _do_burned_register( @@ -195,7 +175,8 @@ async def _do_pow_register( Returns: success (bool): ``True`` if the extrinsic was included in a block. - error (Optional[str]): ``None`` on success or not waiting for inclusion/finalization, otherwise the error message. + error (Optional[str]): ``None`` on success or not waiting for inclusion/finalization, otherwise the error + message. """ # create extrinsic call call = await subtensor.substrate.compose_call( @@ -210,26 +191,13 @@ async def _do_pow_register( "coldkey": wallet.coldkeypub.ss58_address, }, ) - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, keypair=wallet.hotkey - ) - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, + return await subtensor.sign_and_send_extrinsic( + call=call, + wallet=wallet, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - return True, None - - # process if registration successful, try again if pow is still valid - if not await response.is_success: - return False, format_error_message(error_message=await response.error_message) - # Successful registration - else: - return True, None - async def register_extrinsic( subtensor: "AsyncSubtensor", @@ -249,11 +217,14 @@ async def register_extrinsic( """Registers the wallet to the chain. Args: - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): initialized AsyncSubtensor object to use for chain interactions + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): initialized AsyncSubtensor object to use for chain + interactions wallet (bittensor_wallet.Wallet): Bittensor wallet object. netuid (int): The ``netuid`` of the subnet to register on. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns `False` if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. max_allowed_attempts (int): Maximum number of attempts to register the wallet. output_in_place (bool): Whether the POW solving should be outputted to the console as it goes along. cuda (bool): If `True`, the wallet should be registered using CUDA device(s). @@ -264,11 +235,12 @@ async def register_extrinsic( log_verbose: If `True`, the registration process will log more information. Returns: - `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the response is `True`. + `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the + response is `True`. """ - + block_hash = await subtensor.substrate.get_chain_head() logging.debug("[magenta]Checking subnet status... [/magenta]") - if not await subtensor.subnet_exists(netuid): + if not await subtensor.subnet_exists(netuid, block_hash=block_hash): logging.error( f":cross_mark: [red]Failed error:[/red] subnet [blue]{netuid}[/blue] does not exist." ) @@ -278,8 +250,7 @@ async def register_extrinsic( f":satellite: [magenta]Checking Account on subnet[/magenta] [blue]{netuid}[/blue] [magenta]...[/magenta]" ) neuron = await subtensor.get_neuron_for_pubkey_and_subnet( - hotkey_ss58=wallet.hotkey.ss58_address, - netuid=netuid, + hotkey_ss58=wallet.hotkey.ss58_address, netuid=netuid, block_hash=block_hash ) if not neuron.is_null: @@ -291,7 +262,8 @@ async def register_extrinsic( return True logging.debug( - f"Registration hotkey: {wallet.hotkey.ss58_address}, Public coldkey: {wallet.coldkey.ss58_address} in the network: {subtensor.network}." + f"Registration hotkey: {wallet.hotkey.ss58_address}, Public coldkey: " + f"{wallet.coldkey.ss58_address} in the network: {subtensor.network}." ) if not torch: @@ -367,7 +339,8 @@ async def register_extrinsic( if "HotKeyAlreadyRegisteredInSubNet" in err_msg: logging.info( - f":white_heavy_check_mark: [green]Already Registered on subnet:[/green] [blue]{netuid}[/blue]." + f":white_heavy_check_mark: [green]Already Registered on subnet:[/green] " + f"[blue]{netuid}[/blue]." ) return True logging.error(f":cross_mark: [red]Failed[/red]: {err_msg}") @@ -394,13 +367,13 @@ async def register_extrinsic( # Exited loop because pow is no longer valid. logging.error("[red]POW is stale.[/red]") # Try again. - # continue if attempts < max_allowed_attempts: # Failed registration, retry pow attempts += 1 logging.error( - f":satellite: [magenta]Failed registration, retrying pow ...[/magenta] [blue]({attempts}/{max_allowed_attempts})[/blue]" + f":satellite: [magenta]Failed registration, retrying pow ...[/magenta] " + f"[blue]({attempts}/{max_allowed_attempts})[/blue]" ) else: # Failed to register after max attempts. diff --git a/bittensor/core/extrinsics/registration.py b/bittensor/core/extrinsics/registration.py index fc43f94795..eef316f387 100644 --- a/bittensor/core/extrinsics/registration.py +++ b/bittensor/core/extrinsics/registration.py @@ -6,17 +6,58 @@ - burned_register_extrinsic: Registers the wallet to chain by recycling TAO. """ -from typing import Union, Optional, TYPE_CHECKING +import time +from typing import Optional, Union, TYPE_CHECKING -from bittensor.core.extrinsics.asyncex.registration import ( - burned_register_extrinsic as async_burned_register_extrinsic, - register_extrinsic as async_register_extrinsic, -) +from bittensor.utils import unlock_key +from bittensor.utils.btlogging import logging +from bittensor.utils.registration import log_no_torch_error, torch -# For annotation and lazy import purposes if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor + from bittensor.utils.registration.pow import POWSolution, create_pow + + +def _do_burned_register( + subtensor: "Subtensor", + netuid: int, + wallet: "Wallet", + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, +) -> tuple[bool, str]: + """ + Performs a burned register extrinsic call to the Subtensor chain. + + This method sends a registration transaction to the Subtensor blockchain using the burned register mechanism. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + netuid (int): The network unique identifier to register on. + wallet (bittensor_wallet.Wallet): The wallet to be registered. + wait_for_inclusion (bool): Whether to wait for the transaction to be included in a block. Default is False. + wait_for_finalization (bool): Whether to wait for the transaction to be finalized. Default is True. + + Returns: + Tuple[bool, Optional[str]]: A tuple containing a boolean indicating success or failure, and an optional error + message. + """ + + # create extrinsic call + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="burned_register", + call_params={ + "netuid": netuid, + "hotkey": wallet.hotkey.ss58_address, + }, + ) + return subtensor.sign_and_send_extrinsic( + call=call, + wallet=wallet, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) def burned_register_extrinsic( @@ -26,14 +67,127 @@ def burned_register_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_burned_register_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + """Registers the wallet to chain by recycling TAO. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + wallet (bittensor.wallet): Bittensor wallet object. + netuid (int): The ``netuid`` of the subnet to register on. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. + + Returns: + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. + """ + block = subtensor.get_current_block() + if not subtensor.subnet_exists(netuid, block=block): + logging.error( + f":cross_mark: [red]Failed error:[/red] subnet [blue]{netuid}[/blue] does not exist." ) + return False + + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + logging.info( + f":satellite: [magenta]Checking Account on subnet[/magenta] [blue]{netuid}[/blue][magenta] ...[/magenta]" + ) + neuron = subtensor.get_neuron_for_pubkey_and_subnet( + wallet.hotkey.ss58_address, netuid=netuid, block=block + ) + + old_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address, block=block) + + if not neuron.is_null: + logging.info(":white_heavy_check_mark: [green]Already Registered[/green]") + logging.info(f"\t\tuid: [blue]{neuron.uid}[/blue]") + logging.info(f"\t\tnetuid: [blue]{neuron.netuid}[/blue]") + logging.info(f"\t\thotkey: [blue]{neuron.hotkey}[/blue]") + logging.info(f"\t\tcoldkey: [blue]{neuron.coldkey}[/blue]") + return True + + recycle_amount = subtensor.recycle(netuid=netuid, block=block) + logging.debug(":satellite: [magenta]Recycling TAO for Registration...[/magenta]") + logging.info(f"Recycling {recycle_amount} to register on subnet:{netuid}") + + success, err_msg = _do_burned_register( + subtensor=subtensor, + netuid=netuid, + wallet=wallet, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not success: + logging.error(f":cross_mark: [red]Failed error:[/red] {err_msg}") + time.sleep(0.5) + return False + # Successful registration, final check for neuron and pubkey + else: + logging.info(":satellite: [magenta]Checking Balance...[/magenta]") + block = subtensor.get_current_block() + new_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address, block=block) + + logging.info( + f"Balance: [blue]{old_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + ) + is_registered = subtensor.is_hotkey_registered( + netuid=netuid, hotkey_ss58=wallet.hotkey.ss58_address, block=block + ) + if is_registered: + logging.info(":white_heavy_check_mark: [green]Registered[/green]") + return True + else: + # neuron not found, try again + logging.error(":cross_mark: [red]Unknown error. Neuron not found.[/red]") + return False + + +def _do_pow_register( + subtensor: "Subtensor", + netuid: int, + wallet: "Wallet", + pow_result: "POWSolution", + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, +) -> tuple[bool, Optional[str]]: + """Sends a (POW) register extrinsic to the chain. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): The subtensor to send the extrinsic to. + netuid (int): The subnet to register on. + wallet (bittensor.wallet): The wallet to register. + pow_result (POWSolution): The PoW result to register. + wait_for_inclusion (bool): If ``True``, waits for the extrinsic to be included in a block. Default to `False`. + wait_for_finalization (bool): If ``True``, waits for the extrinsic to be finalized. Default to `True`. + + Returns: + success (bool): ``True`` if the extrinsic was included in a block. + error (Optional[str]): ``None`` on success or not waiting for inclusion/finalization, otherwise the error + message. + """ + # create extrinsic call + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="register", + call_params={ + "netuid": netuid, + "block_number": pow_result.block_number, + "nonce": pow_result.nonce, + "work": [int(byte_) for byte_ in pow_result.seal], + "hotkey": wallet.hotkey.ss58_address, + "coldkey": wallet.coldkeypub.ss58_address, + }, + ) + return subtensor.sign_and_send_extrinsic( + call=call, + wallet=wallet, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) @@ -52,20 +206,168 @@ def register_extrinsic( update_interval: Optional[int] = None, log_verbose: bool = False, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_register_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - max_allowed_attempts=max_allowed_attempts, - output_in_place=output_in_place, - cuda=cuda, - dev_id=dev_id, - tpb=tpb, - num_processes=num_processes, - update_interval=update_interval, - log_verbose=log_verbose, + """Registers the wallet to the chain. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor object to use for chain interactions + wallet (bittensor_wallet.Wallet): Bittensor wallet object. + netuid (int): The ``netuid`` of the subnet to register on. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + max_allowed_attempts (int): Maximum number of attempts to register the wallet. + output_in_place (bool): Whether the POW solving should be outputted to the console as it goes along. + cuda (bool): If `True`, the wallet should be registered using CUDA device(s). + dev_id: The CUDA device id to use, or a list of device ids. + tpb: The number of threads per block (CUDA). + num_processes: The number of processes to use to register. + update_interval: The number of nonces to solve between updates. + log_verbose: If `True`, the registration process will log more information. + + Returns: + `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the + response is `True`. + """ + + logging.debug("[magenta]Checking subnet status... [/magenta]") + block = subtensor.get_current_block() + if not subtensor.subnet_exists(netuid, block=block): + logging.error( + f":cross_mark: [red]Failed error:[/red] subnet [blue]{netuid}[/blue] does not exist." ) + return False + + logging.info( + f":satellite: [magenta]Checking Account on subnet[/magenta] [blue]{netuid}[/blue] [magenta]...[/magenta]" + ) + neuron = subtensor.get_neuron_for_pubkey_and_subnet( + hotkey_ss58=wallet.hotkey.ss58_address, netuid=netuid, block=block ) + + if not neuron.is_null: + logging.info(":white_heavy_check_mark: [green]Already Registered[/green]") + logging.info(f"\t\tuid: [blue]{neuron.uid}[/blue]") + logging.info(f"\t\tnetuid: [blue]{neuron.netuid}[/blue]") + logging.info(f"\t\thotkey: [blue]{neuron.hotkey}[/blue]") + logging.info(f"\t\tcoldkey: [blue]{neuron.coldkey}[/blue]") + return True + + logging.debug( + f"Registration hotkey: {wallet.hotkey.ss58_address}, Public coldkey: " + f"{wallet.coldkey.ss58_address} in the network: {subtensor.network}." + ) + + if not torch: + log_no_torch_error() + return False + + # Attempt rolling registration. + attempts = 1 + + while True: + logging.info( + f":satellite: [magenta]Registering...[/magenta] [blue]({attempts}/{max_allowed_attempts})[/blue]" + ) + # Solve latest POW. + if cuda: + if not torch.cuda.is_available(): + return False + + pow_result = create_pow( + subtensor=subtensor, + wallet=wallet, + netuid=netuid, + output_in_place=output_in_place, + cuda=cuda, + dev_id=dev_id, + tpb=tpb, + num_processes=num_processes, + update_interval=update_interval, + log_verbose=log_verbose, + ) + else: + pow_result = create_pow( + subtensor=subtensor, + wallet=wallet, + netuid=netuid, + output_in_place=output_in_place, + cuda=cuda, + num_processes=num_processes, + update_interval=update_interval, + log_verbose=log_verbose, + ) + + # pow failed + if not pow_result: + # might be registered already on this subnet + is_registered = subtensor.is_hotkey_registered( + netuid=netuid, hotkey_ss58=wallet.hotkey.ss58_address + ) + if is_registered: + logging.error( + f":white_heavy_check_mark: [green]Already registered on netuid:[/green] [blue]{netuid}[/blue]" + ) + return True + + # pow successful, proceed to submit pow to chain for registration + else: + logging.info(":satellite: [magenta]Submitting POW...[/magenta]") + # check if pow result is still valid + while not pow_result.is_stale(subtensor=subtensor): + result: tuple[bool, Optional[str]] = _do_pow_register( + subtensor=subtensor, + netuid=netuid, + wallet=wallet, + pow_result=pow_result, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + success, err_msg = result + if not success: + # Look error here + # https://github.com/opentensor/subtensor/blob/development/pallets/subtensor/src/errors.rs + + if "HotKeyAlreadyRegisteredInSubNet" in err_msg: + logging.info( + f":white_heavy_check_mark: [green]Already Registered on subnet:[/green] " + f"[blue]{netuid}[/blue]." + ) + return True + logging.error(f":cross_mark: [red]Failed[/red]: {err_msg}") + time.sleep(0.5) + + # Successful registration, final check for neuron and pubkey + if success: + logging.info(":satellite: Checking Registration status...") + is_registered = subtensor.is_hotkey_registered( + netuid=netuid, hotkey_ss58=wallet.hotkey.ss58_address + ) + if is_registered: + logging.success( + ":white_heavy_check_mark: [green]Registered[/green]" + ) + return True + else: + # neuron not found, try again + logging.error( + ":cross_mark: [red]Unknown error. Neuron not found.[/red]" + ) + continue + else: + # Exited loop because pow is no longer valid. + logging.error("[red]POW is stale.[/red]") + # Try again. + + if attempts < max_allowed_attempts: + # Failed registration, retry pow + attempts += 1 + logging.error( + f":satellite: [magenta]Failed registration, retrying pow ...[/magenta] " + f"[blue]({attempts}/{max_allowed_attempts})[/blue]" + ) + else: + # Failed to register after max attempts. + logging.error("[red]No more attempts.[/red]") + return False diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index b039f0a0ac..47059167d9 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -20,6 +20,10 @@ decode_account_id, WeightCommitInfo, ) +from bittensor.core.extrinsics.registration import ( + burned_register_extrinsic, + register_extrinsic, +) from bittensor.core.extrinsics.staking import ( add_stake_extrinsic, add_stake_multiple_extrinsic, @@ -2064,7 +2068,6 @@ def burned_register( Returns: bool: ``True`` if the registration is successful, False otherwise. """ - # TODO add this extrinsic return burned_register_extrinsic( subtensor=self, wallet=wallet, @@ -2191,7 +2194,6 @@ def register( This function facilitates the entry of new neurons into the network, supporting the decentralized growth and scalability of the Bittensor ecosystem. """ - # TODO add this extrinsic return register_extrinsic( subtensor=self, wallet=wallet, diff --git a/bittensor/utils/registration/__init__.py b/bittensor/utils/registration/__init__.py index 37a913e20a..ea527e4dc3 100644 --- a/bittensor/utils/registration/__init__.py +++ b/bittensor/utils/registration/__init__.py @@ -8,3 +8,14 @@ POWSolution, ) from bittensor.utils.registration.async_pow import create_pow_async + +__all__ = [ + create_pow, + legacy_torch_api_compat, + log_no_torch_error, + torch, + use_torch, + LazyLoadedTorch, + POWSolution, + create_pow_async, +] diff --git a/bittensor/utils/registration/async_pow.py b/bittensor/utils/registration/async_pow.py index e02e8c7bb8..838dca463a 100644 --- a/bittensor/utils/registration/async_pow.py +++ b/bittensor/utils/registration/async_pow.py @@ -6,7 +6,6 @@ from queue import Empty from typing import Callable, Union, Optional, TYPE_CHECKING -from retry import retry from bittensor.core.errors import SubstrateRequestException from bittensor.utils.registration.pow import ( @@ -14,7 +13,7 @@ update_curr_block, terminate_workers_and_wait_for_exit, CUDASolver, - LazyLoadedTorch, + torch, RegistrationStatistics, RegistrationStatisticsLogger, Solver, @@ -25,12 +24,8 @@ from bittensor.core.async_subtensor import AsyncSubtensor from bittensor_wallet import Wallet from bittensor.utils.registration import POWSolution - import torch -else: - torch = LazyLoadedTorch() -@retry(Exception, tries=3, delay=1) async def _get_block_with_retry( subtensor: "AsyncSubtensor", netuid: int ) -> tuple[int, int, str]: diff --git a/bittensor/utils/registration/pow.py b/bittensor/utils/registration/pow.py index c96295b0cd..b70e5bc748 100644 --- a/bittensor/utils/registration/pow.py +++ b/bittensor/utils/registration/pow.py @@ -1088,7 +1088,7 @@ def create_pow( num_processes: Optional[int] = None, update_interval: Optional[int] = None, log_verbose: bool = False, -) -> Optional[dict[str, Any]]: +) -> Optional["POWSolution"]: """ Creates a proof of work for the given subtensor and wallet. From 428211ae0e4c193bd528fad60186dea4a8275734 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 21 Jan 2025 23:45:50 +0200 Subject: [PATCH 13/52] Weights extrinsics --- bittensor/core/extrinsics/asyncex/weights.py | 144 ++++++------ bittensor/core/extrinsics/commit_weights.py | 225 ++++++++++++++++--- bittensor/core/extrinsics/set_weights.py | 138 +++++++++++- bittensor/core/subtensor.py | 8 +- 4 files changed, 396 insertions(+), 119 deletions(-) diff --git a/bittensor/core/extrinsics/asyncex/weights.py b/bittensor/core/extrinsics/asyncex/weights.py index f2affa3b5d..cb9fe16798 100644 --- a/bittensor/core/extrinsics/asyncex/weights.py +++ b/bittensor/core/extrinsics/asyncex/weights.py @@ -14,6 +14,42 @@ from bittensor_wallet import Wallet from bittensor.core.async_subtensor import AsyncSubtensor from bittensor.utils.registration import torch + from scalecodec.types import GenericCall + + +async def sign_and_send_with_nonce( + subtensor: "AsyncSubtensor", + call: "GenericCall", + wallet: "Wallet", + wait_for_inclusion: bool, + wait_for_finalization: bool, + period: Optional[int] = None, +): + """ + Signs an extrinsic call with the wallet hotkey, adding an optional era for period + """ + next_nonce = await subtensor.substrate.get_account_next_index( + wallet.hotkey.ss58_address + ) + + extrinsic_data = {"call": call, "keypair": wallet.hotkey, "nonce": next_nonce} + if period is not None: + extrinsic_data["era"] = {"period": period} + + extrinsic = await subtensor.substrate.create_signed_extrinsic(**extrinsic_data) + response = await subtensor.substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, None + + if await response.is_success: + return True, None + + return False, format_error_message(await response.error_message) async def _do_commit_weights( @@ -29,7 +65,8 @@ async def _do_commit_weights( This method constructs and submits the transaction, handling retries and blockchain communication. Args: - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain interaction. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain + interaction. wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. netuid (int): The unique identifier of the subnet. commit_hash (str): The hash of the neuron's weights to be committed. @@ -39,7 +76,8 @@ async def _do_commit_weights( Returns: tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. - This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a verifiable record of the neuron's weight distribution at a specific point in time. + This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a + verifiable record of the neuron's weight distribution at a specific point in time. """ call = await subtensor.substrate.compose_call( call_module="SubtensorModule", @@ -49,30 +87,10 @@ async def _do_commit_weights( "commit_hash": commit_hash, }, ) - - next_nonce = await subtensor.substrate.get_account_next_index( - wallet.hotkey.ss58_address + return await sign_and_send_with_nonce( + subtensor, call, wallet, wait_for_inclusion, wait_for_finalization ) - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, - keypair=wallet.hotkey, - nonce=next_nonce, - ) - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) - - if not wait_for_finalization and not wait_for_inclusion: - return True, None - - if await response.is_success: - return True, None - - return False, format_error_message(response.error_message) - async def commit_weights_extrinsic( subtensor: "AsyncSubtensor", @@ -87,7 +105,8 @@ async def commit_weights_extrinsic( This function is a wrapper around the `do_commit_weights` method. Args: - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain interaction. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain + interaction. wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. netuid (int): The unique identifier of the subnet. commit_hash (str): The hash of the neuron's weights to be committed. @@ -98,7 +117,8 @@ async def commit_weights_extrinsic( tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string value describing the success or potential error. - This function provides a user-friendly interface for committing weights to the Bittensor blockchain, ensuring proper error handling and user interaction when required. + This function provides a user-friendly interface for committing weights to the Bittensor blockchain, ensuring proper + error handling and user interaction when required. """ success, error_message = await _do_commit_weights( @@ -135,7 +155,8 @@ async def _do_reveal_weights( This method constructs and submits the transaction, handling retries and blockchain communication. Args: - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain interaction. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain + interaction. wallet (bittensor_wallet.Wallet): The wallet associated with the neuron revealing the weights. netuid (int): The unique identifier of the subnet. uids (list[int]): List of neuron UIDs for which weights are being revealed. @@ -163,27 +184,9 @@ async def _do_reveal_weights( "version_key": version_key, }, ) - next_nonce = await subtensor.substrate.get_account_next_index( - wallet.hotkey.ss58_address + return await sign_and_send_with_nonce( + subtensor, call, wallet, wait_for_inclusion, wait_for_finalization ) - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, - keypair=wallet.hotkey, - nonce=next_nonce, - ) - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) - - if not wait_for_finalization and not wait_for_inclusion: - return True, None - - if await response.is_success: - return True, None - - return False, await response.error_message async def reveal_weights_extrinsic( @@ -202,7 +205,8 @@ async def reveal_weights_extrinsic( This function is a wrapper around the `_do_reveal_weights` method. Args: - subtensor (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain + interaction. wallet (bittensor_wallet.Wallet): The wallet associated with the neuron revealing the weights. netuid (int): The unique identifier of the subnet. uids (list[int]): List of neuron UIDs for which weights are being revealed. @@ -213,9 +217,11 @@ async def reveal_weights_extrinsic( wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Returns: - tuple[bool, str]: ``True`` if the weight revelation is successful, False otherwise. And `msg`, a string value describing the success or potential error. + tuple[bool, str]: ``True`` if the weight revelation is successful, False otherwise. And `msg`, a string value + describing the success or potential error. - This function provides a user-friendly interface for revealing weights on the Bittensor blockchain, ensuring proper error handling and user interaction when required. + This function provides a user-friendly interface for revealing weights on the Bittensor blockchain, ensuring proper + error handling and user interaction when required. """ success, error_message = await _do_reveal_weights( @@ -284,31 +290,9 @@ async def _do_set_weights( "version_key": version_key, }, ) - - next_nonce = await subtensor.substrate.get_account_next_index( - wallet.hotkey.ss58_address - ) - - # Period dictates how long the extrinsic will stay as part of waiting pool - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, - keypair=wallet.hotkey, - era={"period": period}, - nonce=next_nonce, + return await sign_and_send_with_nonce( + subtensor, call, wallet, wait_for_inclusion, wait_for_finalization, period ) - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - return True, "Not waiting for finalization or inclusion." - - if await response.is_success: - return True, "Successfully set weights." - - return False, format_error_message(response.error_message) async def set_weights_extrinsic( @@ -328,13 +312,17 @@ async def set_weights_extrinsic( wallet (bittensor_wallet.Wallet): Bittensor wallet object. netuid (int): The ``netuid`` of the subnet to set weights for. uids (Union[NDArray[np.int64], torch.LongTensor, list]): The ``uint64`` uids of destination neurons. - weights (Union[NDArray[np.float32], torch.FloatTensor, list]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. + weights (Union[NDArray[np.float32], torch.FloatTensor, list]): The weights to set. These must be ``float`` s and + correspond to the passed ``uid`` s. version_key (int): The version key of the validator. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. """ # First convert types. if isinstance(uids, list): diff --git a/bittensor/core/extrinsics/commit_weights.py b/bittensor/core/extrinsics/commit_weights.py index 289918839b..fc09aa4031 100644 --- a/bittensor/core/extrinsics/commit_weights.py +++ b/bittensor/core/extrinsics/commit_weights.py @@ -1,15 +1,83 @@ """Module sync commit weights and reveal weights extrinsic.""" -from typing import TYPE_CHECKING +from typing import Union, TYPE_CHECKING, Optional -from bittensor.core.extrinsics.asyncex.weights import ( - reveal_weights_extrinsic as async_reveal_weights_extrinsic, - commit_weights_extrinsic as async_commit_weights_extrinsic, -) +import numpy as np +from numpy.typing import NDArray + +import bittensor.utils.weight_utils as weight_utils +from bittensor.core.settings import version_as_int +from bittensor.utils import format_error_message +from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor + from bittensor.utils.registration import torch + + +def sign_and_send_with_nonce( + subtensor: "Subtensor", call, wallet, wait_for_inclusion, wait_for_finalization +): + next_nonce = subtensor.substrate.get_account_next_index(wallet.hotkey.ss58_address) + extrinsic = subtensor.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + nonce=next_nonce, + ) + response = subtensor.substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, None + + if response.is_success: + return True, None + + return False, format_error_message(response.error_message) + + +def _do_commit_weights( + subtensor: "Subtensor", + wallet: "Wallet", + netuid: int, + commit_hash: str, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, Optional[str]]: + """ + Internal method to send a transaction to the Bittensor blockchain, committing the hash of a neuron's weights. + This method constructs and submits the transaction, handling retries and blockchain communication. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuid (int): The unique identifier of the subnet. + commit_hash (str): The hash of the neuron's weights to be committed. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + + This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a + verifiable record of the neuron's weight distribution at a specific point in time. + """ + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="commit_weights", + call_params={ + "netuid": netuid, + "commit_hash": commit_hash, + }, + ) + + return sign_and_send_with_nonce( + subtensor, call, wallet, wait_for_inclusion, wait_for_finalization + ) def commit_weights_extrinsic( @@ -20,15 +88,90 @@ def commit_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return subtensor.execute_coroutine( - coroutine=async_commit_weights_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - netuid=netuid, - commit_hash=commit_hash, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) + """ + Commits a hash of the neuron's weights to the Bittensor blockchain using the provided wallet. + This function is a wrapper around the `do_commit_weights` method. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuid (int): The unique identifier of the subnet. + commit_hash (str): The hash of the neuron's weights to be committed. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + + This function provides a user-friendly interface for committing weights to the Bittensor blockchain, ensuring proper + error handling and user interaction when required. + """ + + success, error_message = _do_commit_weights( + subtensor=subtensor, + wallet=wallet, + netuid=netuid, + commit_hash=commit_hash, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if success: + success_message = "Successfully committed weights." + logging.info(success_message) + return True, success_message + + logging.error(f"Failed to commit weights: {error_message}") + return False, error_message + + +def _do_reveal_weights( + subtensor: "AsyncSubtensor", + wallet: "Wallet", + netuid: int, + uids: list[int], + values: list[int], + salt: list[int], + version_key: int, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, Optional[dict]]: + """ + Internal method to send a transaction to the Bittensor blockchain, revealing the weights for a specific subnet. + This method constructs and submits the transaction, handling retries and blockchain communication. + + Args: + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron revealing the weights. + netuid (int): The unique identifier of the subnet. + uids (list[int]): List of neuron UIDs for which weights are being revealed. + values (list[int]): List of weight values corresponding to each UID. + salt (list[int]): List of salt values corresponding to the hash function. + version_key (int): Version key for compatibility with the network. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + + This method ensures that the weight revelation is securely recorded on the Bittensor blockchain, providing + transparency and accountability for the neuron's weight distribution. + """ + + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="reveal_weights", + call_params={ + "netuid": netuid, + "uids": uids, + "values": values, + "salt": salt, + "version_key": version_key, + }, + ) + return sign_and_send_with_nonce( + subtensor, call, wallet, wait_for_inclusion, wait_for_finalization ) @@ -43,16 +186,46 @@ def reveal_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return subtensor.execute_coroutine( - coroutine=async_reveal_weights_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - netuid=netuid, - uids=uids, - weights=weights, - salt=salt, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) + """ + Reveals the weights for a specific subnet on the Bittensor blockchain using the provided wallet. + This function is a wrapper around the `_do_reveal_weights` method. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron revealing the weights. + netuid (int): The unique identifier of the subnet. + uids (list[int]): List of neuron UIDs for which weights are being revealed. + weights (list[int]): List of weight values corresponding to each UID. + salt (list[int]): List of salt values corresponding to the hash function. + version_key (int): Version key for compatibility with the network. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, str]: ``True`` if the weight revelation is successful, False otherwise. And `msg`, a string value + describing the success or potential error. + + This function provides a user-friendly interface for revealing weights on the Bittensor blockchain, ensuring proper + error handling and user interaction when required. + """ + + success, error_message = _do_reveal_weights( + subtensor=subtensor, + wallet=wallet, + netuid=netuid, + uids=uids, + values=weights, + salt=salt, + version_key=version_key, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, ) + + if success: + success_message = "Successfully revealed weights." + logging.info(success_message) + return True, success_message + + error_message = format_error_message(error_message) + logging.error(f"Failed to reveal weights: {error_message}") + return False, error_message diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index a908e59c09..5e86c9110e 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -1,18 +1,85 @@ """Module sync setting weights extrinsic.""" -from typing import Union, TYPE_CHECKING +from typing import Union, TYPE_CHECKING, Optional import numpy as np from numpy.typing import NDArray -from bittensor.core.extrinsics.asyncex.weights import ( - set_weights_extrinsic as async_set_weights_extrinsic, -) -from bittensor.utils.registration import torch +from bittensor.core.settings import version_as_int +from bittensor.utils import format_error_message, weight_utils +from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor.core.subtensor import Subtensor from bittensor_wallet import Wallet + from bittensor.utils.registration import torch + + +def _do_set_weights( + subtensor: "Subtensor", + wallet: "Wallet", + netuid: int, + uids: list[int], + vals: list[int], + version_key: int = version_as_int, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + period: int = 5, +) -> tuple[bool, Optional[str]]: # (success, error_message) + """ + Internal method to send a transaction to the Bittensor blockchain, setting weights + for specified neurons. This method constructs and submits the transaction, handling + retries and blockchain communication. + + Args: + subtensor (subtensor.core.subtensor.Subtensor): Subtensor instance. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. + uids (List[int]): List of neuron UIDs for which weights are being set. + vals (List[int]): List of weight values corresponding to each UID. + netuid (int): Unique identifier for the network. + version_key (int, optional): Version key for compatibility with the network. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + period (int, optional): The period in seconds to wait for extrinsic inclusion or finalization. Defaults to 5. + + Returns: + Tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + + This method is vital for the dynamic weighting mechanism in Bittensor, where neurons adjust their + trust in other neurons based on observed performance and contributions. + """ + + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="set_weights", + call_params={ + "dests": uids, + "weights": vals, + "netuid": netuid, + "version_key": version_key, + }, + ) + next_nonce = subtensor.substrate.get_account_next_index(wallet.hotkey.ss58_address) + # Period dictates how long the extrinsic will stay as part of waiting pool + extrinsic = subtensor.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + era={"period": period}, + nonce=next_nonce, + ) + response = subtensor.substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, "Not waiting for finalization or inclusion." + + if response.is_success: + return True, "Successfully set weights." + + return False, format_error_message(response.error_message) def set_weights_extrinsic( @@ -25,15 +92,62 @@ def set_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return subtensor.execute_coroutine( - coroutine=async_set_weights_extrinsic( - subtensor=subtensor.async_subtensor, + """Sets the given weights and values on chain for wallet hotkey account. + + Args: + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): Bittensor subtensor object. + wallet (bittensor_wallet.Wallet): Bittensor wallet object. + netuid (int): The ``netuid`` of the subnet to set weights for. + uids (Union[NDArray[np.int64], torch.LongTensor, list]): The ``uint64`` uids of destination neurons. + weights (Union[NDArray[np.float32], torch.FloatTensor, list]): The weights to set. These must be ``float`` s + and correspond to the passed ``uid`` s. + version_key (int): The version key of the validator. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. + + Returns: + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. + """ + # First convert types. + if isinstance(uids, list): + uids = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights = np.array(weights, dtype=np.float32) + + # Reformat and normalize. + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids, weights + ) + + logging.info( + ":satellite: [magenta]Setting weights on [/magenta][blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + ) + try: + success, error_message = _do_set_weights( + subtensor=subtensor, wallet=wallet, netuid=netuid, - uids=uids, - weights=weights, + uids=weight_uids, + vals=weight_vals, version_key=version_key, - wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, ) - ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, "Not waiting for finalization or inclusion." + + if success is True: + message = "Successfully set weights and Finalized." + logging.success(f":white_heavy_check_mark: [green]{message}[/green]") + return True, message + + logging.error(f"[red]Failed[/red] set weights. Error: {error_message}") + return False, error_message + + except Exception as error: + logging.error(f":cross_mark: [red]Failed[/red] set weights. Error: {error}") + return False, str(error) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 47059167d9..2b8e270dfe 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -20,10 +20,15 @@ decode_account_id, WeightCommitInfo, ) +from bittensor.core.extrinsics.commit_weights import ( + commit_weights_extrinsic, + reveal_weights_extrinsic, +) from bittensor.core.extrinsics.registration import ( burned_register_extrinsic, register_extrinsic, ) +from bittensor.core.extrinsics.set_weights import set_weights_extrinsic from bittensor.core.extrinsics.staking import ( add_stake_extrinsic, add_stake_multiple_extrinsic, @@ -2133,7 +2138,6 @@ def commit_weights( while retries < max_retries and success is False: try: - # TODO add this extrinsic success, message = commit_weights_extrinsic( subtensor=self, wallet=wallet, @@ -2252,7 +2256,6 @@ def reveal_weights( while retries < max_retries and success is False: try: - # TODO add this extrinsic success, message = reveal_weights_extrinsic( subtensor=self, wallet=wallet, @@ -2451,7 +2454,6 @@ def _blocks_weight_limit() -> bool: f"Setting weights for subnet #[blue]{netuid}[/blue]. " f"Attempt [blue]{retries + 1} of {max_retries}[/blue]." ) - # TODO add this extrinsic success, message = set_weights_extrinsic( subtensor=self, wallet=wallet, From 333ba83caa4a066969c29fe56fcd6e526d5f2eb3 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 21 Jan 2025 23:46:14 +0200 Subject: [PATCH 14/52] Sets nest_asyncio as default off. --- bittensor/core/settings.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/bittensor/core/settings.py b/bittensor/core/settings.py index 04d94436ef..68854802ce 100644 --- a/bittensor/core/settings.py +++ b/bittensor/core/settings.py @@ -329,15 +329,7 @@ def __apply_nest_asyncio(): If not set, warn the user that the default will change in the future. """ nest_asyncio_env = os.getenv("NEST_ASYNCIO") - if nest_asyncio_env == "1" or nest_asyncio_env is None: - if nest_asyncio_env is None: - warnings.warn( - """NEST_ASYNCIO implicitly set to '1'. In the future, the default value will be '0'. - If you use `nest_asyncio`, make sure to add it explicitly to your project dependencies, - as it will be removed from `bittensor` package dependencies in the future. - To silence this warning, explicitly set the environment variable, e.g. `export NEST_ASYNCIO=0`.""", - DeprecationWarning, - ) + if nest_asyncio_env == "1": # Install and apply nest asyncio to allow the async functions to run in a .ipynb import nest_asyncio From 1a4deddab966fa0266499e276be889ac50a9132d Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 21 Jan 2025 23:48:26 +0200 Subject: [PATCH 15/52] Import order --- bittensor/core/subtensor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 2b8e270dfe..c84ddc1b63 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -2,17 +2,16 @@ from functools import lru_cache from typing import TYPE_CHECKING, Any, Iterable, Optional, Union -# TODO clean up this import section -import numpy as np -import ujson from async_substrate_interface.errors import SubstrateRequestException from async_substrate_interface.sync_substrate import SubstrateInterface from async_substrate_interface.utils import hex_to_bytes +import numpy as np from numpy.typing import NDArray import requests import scalecodec from scalecodec.base import RuntimeConfiguration from scalecodec.type_registry import load_type_registry_preset +import ujson from bittensor.core import SubtensorMixin from bittensor.core.chain_data import ( From 8c796b5d610128393f714252f6e412111737c3ef Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 16:42:00 +0200 Subject: [PATCH 16/52] Root extrinsics --- bittensor/core/async_subtensor.py | 4 +- bittensor/core/extrinsics/asyncex/root.py | 42 ++-- bittensor/core/extrinsics/root.py | 266 ++++++++++++++++++++-- bittensor/core/subtensor.py | 7 +- 4 files changed, 276 insertions(+), 43 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index a7bdeeb076..442735992a 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -2922,7 +2922,6 @@ async def reveal_weights( async def root_register( self, wallet: "Wallet", - netuid: int = 0, block_hash: Optional[str] = None, wait_for_inclusion: bool = True, wait_for_finalization: bool = True, @@ -2932,7 +2931,6 @@ async def root_register( Arguments: wallet (bittensor_wallet.Wallet): Bittensor wallet instance. - netuid (int): Subnet uniq id. Root subnet uid is 0. block_hash (Optional[str]): The hash of the blockchain block for the query. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is @@ -2941,6 +2939,7 @@ async def root_register( Returns: `True` if registration was successful, otherwise `False`. """ + netuid = 0 logging.info( f"Registering on netuid [blue]0[/blue] on network: [blue]{self.network}[/blue]" ) @@ -2973,7 +2972,6 @@ async def root_register( return await root_register_extrinsic( subtensor=self, wallet=wallet, - netuid=netuid, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) diff --git a/bittensor/core/extrinsics/asyncex/root.py b/bittensor/core/extrinsics/asyncex/root.py index 9a77051039..61f9455988 100644 --- a/bittensor/core/extrinsics/asyncex/root.py +++ b/bittensor/core/extrinsics/asyncex/root.py @@ -1,9 +1,8 @@ import asyncio -import time from typing import Union, TYPE_CHECKING -import numpy as np from bittensor_wallet import Wallet +import numpy as np from numpy.typing import NDArray from bittensor.core.errors import SubstrateRequestException @@ -45,7 +44,6 @@ async def _get_limits(subtensor: "AsyncSubtensor") -> tuple[int, float]: async def root_register_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", - netuid: int, wait_for_inclusion: bool = True, wait_for_finalization: bool = True, ) -> bool: @@ -54,14 +52,16 @@ async def root_register_extrinsic( Arguments: subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The AsyncSubtensor object wallet (bittensor_wallet.Wallet): Bittensor wallet object. - netuid (int): Subnet uid. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns `False` if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. Returns: - `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the response is `True`. + `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, + the response is `True`. """ - + netuid = 0 if not (unlock := unlock_key(wallet)).success: logging.error(unlock.message) return False @@ -93,7 +93,7 @@ async def root_register_extrinsic( if not success: logging.error(f":cross_mark: [red]Failed error:[/red] {err_msg}") - time.sleep(0.5) + await asyncio.sleep(0.5) return False # Successful registration, final check for neuron and pubkey @@ -132,18 +132,22 @@ async def _do_set_root_weights( It waits for inclusion or finalization of the extrinsic based on the provided parameters. Arguments: - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The AsyncSubtensor object used to interact with the blockchain. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The AsyncSubtensor object used to interact with the + blockchain. wallet (bittensor_wallet.Wallet): The wallet containing the hotkey and coldkey for the transaction. netuids (Union[NDArray[np.int64], list[int]]): List of UIDs to set weights for. weights (Union[NDArray[np.float32], list[float]]): Corresponding weights to set for each UID. netuid (int): The netuid of the subnet to set weights for. Defaults to 0. version_key (int, optional): The version key of the validator. Defaults to 0. - wait_for_inclusion (bool, optional): If True, waits for the extrinsic to be included in a block. Defaults to False. - wait_for_finalization (bool, optional): If True, waits for the extrinsic to be finalized on the chain. Defaults to False. + wait_for_inclusion (bool, optional): If True, waits for the extrinsic to be included in a block. Defaults to + False. + wait_for_finalization (bool, optional): If True, waits for the extrinsic to be finalized on the chain. Defaults + to False. period (int, optional): The period in seconds to wait for extrinsic inclusion or finalization. Defaults to 5. Returns: - tuple: Returns a tuple containing a boolean indicating success and a message describing the result of the operation. + tuple: Returns a tuple containing a boolean indicating success and a message describing the result of the + operation. """ call = await subtensor.substrate.compose_call( call_module="SubtensorModule", @@ -198,13 +202,17 @@ async def set_root_weights_extrinsic( subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The AsyncSubtensor object wallet (bittensor_wallet.Wallet): Bittensor wallet object. netuids (Union[NDArray[np.int64], list[int]]): The `netuid` of the subnet to set weights for. - weights (Union[NDArray[np.float32], list[float]]): Weights to set. These must be `float` s and must correspond to the passed `netuid` s. + weights (Union[NDArray[np.float32], list[float]]): Weights to set. These must be `float` s and must correspond + to the passed `netuid` s. version_key (int): The version key of the validator. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns `False` if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ` + True`, or returns `False` if the extrinsic fails to be finalized within the timeout. Returns: - `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the response is `True`. + `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the + response is `True`. """ my_uid = await subtensor.substrate.query( "SubtensorModule", "Uids", [0, wallet.hotkey.ss58_address] diff --git a/bittensor/core/extrinsics/root.py b/bittensor/core/extrinsics/root.py index 8631e652c0..bf49e8023a 100644 --- a/bittensor/core/extrinsics/root.py +++ b/bittensor/core/extrinsics/root.py @@ -1,34 +1,192 @@ +import time from typing import Union, TYPE_CHECKING import numpy as np from numpy.typing import NDArray -from bittensor.core.extrinsics.asyncex.root import ( - root_register_extrinsic as async_root_register_extrinsic, - set_root_weights_extrinsic as async_set_root_weights_extrinsic, +from bittensor.core.errors import SubstrateRequestException +from bittensor.utils import ( + u16_normalized_float, + format_error_message, + unlock_key, + torch, +) +from bittensor.utils.btlogging import logging +from bittensor.utils.weight_utils import ( + normalize_max_weight, + convert_weights_and_uids_for_emit, ) -from bittensor.utils.registration import torch if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor +def _get_limits(subtensor: "Subtensor") -> tuple[int, float]: + """ + Retrieves the minimum allowed weights and maximum weight limit for the given subnet. + + These values are fetched asynchronously using `asyncio.gather` to run both requests concurrently. + + Args: + subtensor (Subtensor): The AsyncSubtensor object used to interface with the network's substrate node. + + Returns: + tuple[int, float]: A tuple containing: + - `min_allowed_weights` (int): The minimum allowed weights. + - `max_weight_limit` (float): The maximum weight limit, normalized to a float value. + """ + # Get weight restrictions. + maw = subtensor.get_hyperparameter("MinAllowedWeights", netuid=0) + mwl = subtensor.get_hyperparameter("MaxWeightsLimit", netuid=0) + min_allowed_weights = int(maw) + max_weight_limit = u16_normalized_float(int(mwl)) + return min_allowed_weights, max_weight_limit + + def root_register_extrinsic( subtensor: "Subtensor", wallet: "Wallet", wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_root_register_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - netuid=0, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + """Registers the wallet to root network. + + Arguments: + subtensor (bittensor.core.subtensor.Subtensor): The Subtensor object + wallet (bittensor_wallet.Wallet): Bittensor wallet object. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + + Returns: + `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the + response is `True`. + """ + netuid = 0 + + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + logging.debug( + f"Checking if hotkey ([blue]{wallet.hotkey_str}[/blue]) is registered on root." + ) + is_registered = subtensor.is_hotkey_registered( + netuid=netuid, hotkey_ss58=wallet.hotkey.ss58_address + ) + if is_registered: + logging.error( + ":white_heavy_check_mark: [green]Already registered on root network.[/green]" + ) + return True + + logging.info(":satellite: [magenta]Registering to root network...[/magenta]") + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="root_register", + call_params={"hotkey": wallet.hotkey.ss58_address}, + ) + success, err_msg = subtensor.sign_and_send_extrinsic( + call, + wallet=wallet, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not success: + logging.error(f":cross_mark: [red]Failed error:[/red] {err_msg}") + time.sleep(0.5) + return False + + # Successful registration, final check for neuron and pubkey + else: + uid = subtensor.substrate.query( + module="SubtensorModule", + storage_function="Uids", + params=[netuid, wallet.hotkey.ss58_address], ) + if uid is not None: + logging.info( + f":white_heavy_check_mark: [green]Registered with UID[/green] [blue]{uid}[/blue]." + ) + return True + else: + # neuron not found, try again + logging.error(":cross_mark: [red]Unknown error. Neuron not found.[/red]") + return False + + +def _do_set_root_weights( + subtensor: "Subtensor", + wallet: "Wallet", + netuids: Union[NDArray[np.int64], list[int]], + weights: Union[NDArray[np.float32], list[float]], + netuid: int = 0, + version_key: int = 0, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + period: int = 5, +) -> tuple[bool, str]: + """ + Sets the root weights on the Subnet for the given wallet hotkey account. + + This function constructs and submits an extrinsic to set the root weights for the given wallet hotkey account. + It waits for inclusion or finalization of the extrinsic based on the provided parameters. + + Arguments: + subtensor (bittensor.core.subtensor.Subtensor): The Subtensor object used to interact with the + blockchain. + wallet (bittensor_wallet.Wallet): The wallet containing the hotkey and coldkey for the transaction. + netuids (Union[NDArray[np.int64], list[int]]): List of UIDs to set weights for. + weights (Union[NDArray[np.float32], list[float]]): Corresponding weights to set for each UID. + netuid (int): The netuid of the subnet to set weights for. Defaults to 0. + version_key (int, optional): The version key of the validator. Defaults to 0. + wait_for_inclusion (bool, optional): If True, waits for the extrinsic to be included in a block. Defaults to + False. + wait_for_finalization (bool, optional): If True, waits for the extrinsic to be finalized on the chain. Defaults + to False. + period (int, optional): The period in seconds to wait for extrinsic inclusion or finalization. Defaults to 5. + + Returns: + tuple: Returns a tuple containing a boolean indicating success and a message describing the result of the + operation. + """ + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="set_root_weights", + call_params={ + "dests": netuids, + "weights": weights, + "netuid": netuid, + "version_key": version_key, + "hotkey": wallet.hotkey.ss58_address, + }, + ) + + next_nonce = subtensor.substrate.get_account_next_index(wallet.hotkey.ss58_address) + + # Period dictates how long the extrinsic will stay as part of waiting pool + extrinsic = subtensor.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.coldkey, + era={"period": period}, + nonce=next_nonce, ) + response = subtensor.substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, "Not waiting for finalization or inclusion." + + if response.is_success: + return True, "Successfully set weights." + + return False, format_error_message(response.error_message) def set_root_weights_extrinsic( @@ -40,14 +198,88 @@ def set_root_weights_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_set_root_weights_extrinsic( - subtensor=subtensor.async_subtensor, + """Sets the given weights and values on chain for wallet hotkey account. + + Arguments: + subtensor (bittensor.core.subtensor.Subtensor): The Subtensor object + wallet (bittensor_wallet.Wallet): Bittensor wallet object. + netuids (Union[NDArray[np.int64], list[int]]): The `netuid` of the subnet to set weights for. + weights (Union[NDArray[np.float32], list[float]]): Weights to set. These must be `float` s and must correspond + to the passed `netuid` s. + version_key (int): The version key of the validator. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + + Returns: + `True` if extrinsic was finalized or included in the block. If we did not wait for finalization/inclusion, the + response is `True`. + """ + my_uid = subtensor.substrate.query( + "SubtensorModule", "Uids", [0, wallet.hotkey.ss58_address] + ) + + if my_uid is None: + logging.error("Your hotkey is not registered to the root network.") + return False + + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + # First convert types. + if isinstance(netuids, list): + netuids = np.array(netuids, dtype=np.int64) + if isinstance(weights, list): + weights = np.array(weights, dtype=np.float32) + + logging.debug("Fetching weight limits") + min_allowed_weights, max_weight_limit = _get_limits(subtensor) + + # Get non zero values. + non_zero_weight_idx = np.argwhere(weights > 0).squeeze(axis=1) + non_zero_weights = weights[non_zero_weight_idx] + if non_zero_weights.size < min_allowed_weights: + raise ValueError( + "The minimum number of weights required to set weights is {}, got {}".format( + min_allowed_weights, non_zero_weights.size + ) + ) + + # Normalize the weights to max value. + logging.info("Normalizing weights") + formatted_weights = normalize_max_weight(x=weights, limit=max_weight_limit) + logging.info( + f"Raw weights -> Normalized weights: [blue]{weights}[/blue] -> [green]{formatted_weights}[/green]" + ) + + try: + logging.info(":satellite: [magenta]Setting root weights...[magenta]") + weight_uids, weight_vals = convert_weights_and_uids_for_emit(netuids, weights) + + success, error_message = _do_set_root_weights( + subtensor=subtensor, wallet=wallet, - netuids=netuids, - weights=weights, + netuids=weight_uids, + weights=weight_vals, version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - ) + + if not wait_for_finalization and not wait_for_inclusion: + return True + + if success is True: + logging.info(":white_heavy_check_mark: [green]Finalized[/green]") + return True + else: + fmt_err = error_message + logging.error(f":cross_mark: [red]Failed error:[/red] {fmt_err}") + return False + + except SubstrateRequestException as e: + fmt_err = format_error_message(e) + logging.error(f":cross_mark: [red]Failed error:[/red] {fmt_err}") + return False diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index c84ddc1b63..6d3233dced 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -27,6 +27,7 @@ burned_register_extrinsic, register_extrinsic, ) +from bittensor.core.extrinsics.root import root_register_extrinsic, set_root_weights_extrinsic from bittensor.core.extrinsics.set_weights import set_weights_extrinsic from bittensor.core.extrinsics.staking import ( add_stake_extrinsic, @@ -2278,7 +2279,6 @@ def reveal_weights( def root_register( self, wallet: "Wallet", - netuid: int = 0, wait_for_inclusion: bool = False, wait_for_finalization: bool = True, ) -> bool: @@ -2287,7 +2287,6 @@ def root_register( Arguments: wallet (bittensor_wallet.Wallet): Bittensor wallet instance. - netuid (int): Subnet uniq id. Root subnet uid is 0. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. @@ -2322,11 +2321,9 @@ def root_register( ) return False - # TODO add this extrinsic return root_register_extrinsic( subtensor=self, wallet=wallet, - netuid=netuid, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) @@ -2359,8 +2356,6 @@ def root_set_weights( netuids_ = np.array(netuids, dtype=np.int64) weights_ = np.array(weights, dtype=np.float32) logging.info(f"Setting weights in network: [blue]{self.network}[/blue]") - # Run the set weights operation. - # TODO add this extrinsic return set_root_weights_extrinsic( subtensor=self, wallet=wallet, From fa84405d7004ec00a5a7b819097f4c0f8bee872a Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 16:49:11 +0200 Subject: [PATCH 17/52] Commit reveal extrinsic --- .../core/extrinsics/asyncex/commit_reveal.py | 29 +---- bittensor/core/extrinsics/commit_reveal.py | 122 ++++++++++++++++-- bittensor/core/subtensor.py | 7 +- 3 files changed, 124 insertions(+), 34 deletions(-) diff --git a/bittensor/core/extrinsics/asyncex/commit_reveal.py b/bittensor/core/extrinsics/asyncex/commit_reveal.py index e26e1fb9cc..d5812a15d5 100644 --- a/bittensor/core/extrinsics/asyncex/commit_reveal.py +++ b/bittensor/core/extrinsics/asyncex/commit_reveal.py @@ -7,7 +7,6 @@ from numpy.typing import NDArray from bittensor.core.settings import version_as_int -from bittensor.utils import format_error_message from bittensor.utils.btlogging import logging from bittensor.utils.weight_utils import convert_weights_and_uids_for_emit @@ -27,10 +26,11 @@ async def _do_commit_reveal_v3( wait_for_finalization: bool = False, ) -> tuple[bool, Optional[str]]: """ - Executes the commit-reveal phase 3 for a given netuid and commit, and optionally waits for extrinsic inclusion or finalization. + Executes the commit-reveal phase 3 for a given netuid and commit, and optionally waits for extrinsic inclusion or + finalization. Arguments: - subtensor: An instance of the Subtensor class. + subtensor: An instance of the AsyncSubtensor class. wallet: Wallet An instance of the Wallet class containing the user's keypair. netuid: int The network unique identifier. commit bytes The commit data in bytes format. @@ -39,7 +39,8 @@ async def _do_commit_reveal_v3( wait_for_finalization: bool, optional Flag indicating whether to wait for the extrinsic to be finalized. Returns: - A tuple where the first element is a boolean indicating success or failure, and the second element is an optional string containing error message if any. + A tuple where the first element is a boolean indicating success or failure, and the second element is an + optional string containing error message if any. """ logging.info( f"Committing weights hash [blue]{commit.hex()}[/blue] for subnet #[blue]{netuid}[/blue] with " @@ -55,26 +56,10 @@ async def _do_commit_reveal_v3( "reveal_round": reveal_round, }, ) - - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, - keypair=wallet.hotkey, - ) - - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + return await subtensor.sign_and_send_extrinsic( + call, wallet, wait_for_inclusion, wait_for_finalization, sign_with="hotkey" ) - if not wait_for_finalization and not wait_for_inclusion: - return True, "Not waiting for finalization or inclusion." - - if await response.is_success: - return True, None - - return False, format_error_message(await response.error_message) - async def commit_reveal_v3_extrinsic( subtensor: "AsyncSubtensor", diff --git a/bittensor/core/extrinsics/commit_reveal.py b/bittensor/core/extrinsics/commit_reveal.py index 739e6dbff2..9f466a5e29 100644 --- a/bittensor/core/extrinsics/commit_reveal.py +++ b/bittensor/core/extrinsics/commit_reveal.py @@ -1,14 +1,14 @@ """This module provides sync functionality for commit reveal in the Bittensor network.""" -from typing import Union, TYPE_CHECKING +from typing import Union, TYPE_CHECKING, Optional +from bittensor_commit_reveal import get_encrypted_commit import numpy as np from numpy.typing import NDArray -from bittensor.core.extrinsics.asyncex.commit_reveal import ( - commit_reveal_v3_extrinsic as async_commit_reveal_v3_extrinsic, -) from bittensor.core.settings import version_as_int +from bittensor.utils.btlogging import logging +from bittensor.utils.weight_utils import convert_weights_and_uids_for_emit if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -16,6 +16,51 @@ from bittensor.utils.registration import torch +def _do_commit_reveal_v3( + subtensor: "Subtensor", + wallet: "Wallet", + netuid: int, + commit: bytes, + reveal_round: int, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, Optional[str]]: + """ + Executes the commit-reveal phase 3 for a given netuid and commit, and optionally waits for extrinsic inclusion or + finalization. + + Arguments: + subtensor: An instance of the Subtensor class. + wallet: Wallet An instance of the Wallet class containing the user's keypair. + netuid: int The network unique identifier. + commit bytes The commit data in bytes format. + reveal_round: int The round number for the reveal phase. + wait_for_inclusion: bool, optional Flag indicating whether to wait for the extrinsic to be included in a block. + wait_for_finalization: bool, optional Flag indicating whether to wait for the extrinsic to be finalized. + + Returns: + A tuple where the first element is a boolean indicating success or failure, and the second element is an + optional string containing error message if any. + """ + logging.info( + f"Committing weights hash [blue]{commit.hex()}[/blue] for subnet #[blue]{netuid}[/blue] with " + f"reveal round [blue]{reveal_round}[/blue]..." + ) + + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="commit_crv3_weights", + call_params={ + "netuid": netuid, + "commit": commit, + "reveal_round": reveal_round, + }, + ) + return subtensor.sign_and_send_extrinsic( + call, wallet, wait_for_inclusion, wait_for_finalization, sign_with="hotkey" + ) + + def commit_reveal_v3_extrinsic( subtensor: "Subtensor", wallet: "Wallet", @@ -26,15 +71,72 @@ def commit_reveal_v3_extrinsic( wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: - return subtensor.execute_coroutine( - coroutine=async_commit_reveal_v3_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - netuid=netuid, + """ + Commits and reveals weights for given subtensor and wallet with provided uids and weights. + + Arguments: + subtensor: The Subtensor instance. + wallet: The wallet to use for committing and revealing. + netuid: The id of the network. + uids: The uids to commit. + weights: The weights associated with the uids. + version_key: The version key to use for committing and revealing. Default is version_as_int. + wait_for_inclusion: Whether to wait for the inclusion of the transaction. Default is False. + wait_for_finalization: Whether to wait for the finalization of the transaction. Default is False. + + Returns: + tuple[bool, str]: A tuple where the first element is a boolean indicating success or failure, and the second + element is a message associated with the result + """ + try: + # Convert uids and weights + if isinstance(uids, list): + uids = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights = np.array(weights, dtype=np.float32) + + # Reformat and normalize. + uids, weights = convert_weights_and_uids_for_emit(uids, weights) + + current_block = subtensor.get_current_block() + subnet_hyperparameters = subtensor.get_subnet_hyperparameters( + netuid, block=current_block + ) + tempo = subnet_hyperparameters.tempo + subnet_reveal_period_epochs = ( + subnet_hyperparameters.commit_reveal_weights_interval + ) + + # Encrypt `commit_hash` with t-lock and `get reveal_round` + commit_for_reveal, reveal_round = get_encrypted_commit( uids=uids, weights=weights, version_key=version_key, + tempo=tempo, + current_block=current_block, + netuid=netuid, + subnet_reveal_period_epochs=subnet_reveal_period_epochs, + ) + + success, message = _do_commit_reveal_v3( + subtensor=subtensor, + wallet=wallet, + netuid=netuid, + commit=commit_for_reveal, + reveal_round=reveal_round, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - ) + + if success is not True: + logging.error(message) + return False, message + + logging.success( + f"[green]Finalized![/green] Weights commited with reveal round [blue]{reveal_round}[/blue]." + ) + return True, f"reveal_round:{reveal_round}" + + except Exception as e: + logging.error(f":cross_mark: [red]Failed. Error:[/red] {e}") + return False, str(e) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 6d3233dced..266177712e 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -19,6 +19,7 @@ decode_account_id, WeightCommitInfo, ) +from bittensor.core.extrinsics.commit_reveal import commit_reveal_v3_extrinsic from bittensor.core.extrinsics.commit_weights import ( commit_weights_extrinsic, reveal_weights_extrinsic, @@ -27,7 +28,10 @@ burned_register_extrinsic, register_extrinsic, ) -from bittensor.core.extrinsics.root import root_register_extrinsic, set_root_weights_extrinsic +from bittensor.core.extrinsics.root import ( + root_register_extrinsic, + set_root_weights_extrinsic, +) from bittensor.core.extrinsics.set_weights import set_weights_extrinsic from bittensor.core.extrinsics.staking import ( add_stake_extrinsic, @@ -2426,7 +2430,6 @@ def _blocks_weight_limit() -> bool: logging.info( f"Committing weights for subnet #{netuid}. Attempt {retries + 1} of {max_retries}." ) - # TODO add this extrinsic success, message = commit_reveal_v3_extrinsic( subtensor=self, wallet=wallet, From 4be459236a364b89a28f6c49896900c47009c003 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 17:03:10 +0200 Subject: [PATCH 18/52] Transfer extrinsic --- bittensor/core/async_subtensor.py | 2 +- bittensor/core/extrinsics/asyncex/transfer.py | 22 ++- bittensor/core/extrinsics/transfer.py | 165 ++++++++++++++++-- bittensor/core/subtensor.py | 4 +- 4 files changed, 169 insertions(+), 24 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 442735992a..32e2a258b8 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -3193,7 +3193,7 @@ async def transfer( return await transfer_extrinsic( subtensor=self, wallet=wallet, - destination=dest, + dest=dest, amount=amount, transfer_all=transfer_all, wait_for_inclusion=wait_for_inclusion, diff --git a/bittensor/core/extrinsics/asyncex/transfer.py b/bittensor/core/extrinsics/asyncex/transfer.py index f1d3bc65e2..c86a29976e 100644 --- a/bittensor/core/extrinsics/asyncex/transfer.py +++ b/bittensor/core/extrinsics/asyncex/transfer.py @@ -32,8 +32,10 @@ async def _do_transfer( wallet (bittensor_wallet.Wallet): Bittensor wallet object to make transfer from. destination (str): Destination public key address (ss58_address or ed25519) of recipient. amount (bittensor.utils.balance.Balance): Amount to stake as Bittensor balance. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns `False` if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. Returns: success, block hash, formatted error message @@ -66,7 +68,7 @@ async def _do_transfer( async def transfer_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", - destination: str, + dest: str, amount: "Balance", transfer_all: bool = False, wait_for_inclusion: bool = True, @@ -78,16 +80,20 @@ async def transfer_extrinsic( Args: subtensor (bittensor.core.async_subtensor.AsyncSubtensor): initialized AsyncSubtensor object used for transfer wallet (bittensor_wallet.Wallet): Bittensor wallet object to make transfer from. - destination (str): Destination public key address (ss58_address or ed25519) of recipient. + dest (str): Destination public key address (ss58_address or ed25519) of recipient. amount (bittensor.utils.balance.Balance): Amount to stake as Bittensor balance. transfer_all (bool): Whether to transfer all funds from this wallet to the destination address. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns `False` if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. keep_alive (bool): If set, keeps the account alive by keeping the balance above the existential deposit. Returns: - success (bool): Flag is `True` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is `True`, regardless of its inclusion. + success (bool): Flag is `True` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is `True`, regardless of its inclusion. """ + destination = dest # Validate destination address. if not is_valid_bittensor_address_or_public_key(destination): logging.error( @@ -153,7 +159,7 @@ async def transfer_extrinsic( explorer_urls = get_explorer_url_for_network( subtensor.network, block_hash, NETWORK_EXPLORER_MAP ) - if explorer_urls != {} and explorer_urls: + if explorer_urls: logging.info( f"[green]Opentensor Explorer Link: {explorer_urls.get('opentensor')}[/green]" ) diff --git a/bittensor/core/extrinsics/transfer.py b/bittensor/core/extrinsics/transfer.py index c4a0b73072..5257fc347d 100644 --- a/bittensor/core/extrinsics/transfer.py +++ b/bittensor/core/extrinsics/transfer.py @@ -1,13 +1,67 @@ from typing import Union, TYPE_CHECKING -from bittensor.core.extrinsics.asyncex.transfer import ( - transfer_extrinsic as async_transfer_extrinsic, +from bittensor.core.settings import NETWORK_EXPLORER_MAP +from bittensor.utils.balance import Balance +from bittensor.utils import ( + is_valid_bittensor_address_or_public_key, + unlock_key, + get_explorer_url_for_network, + format_error_message, ) +from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor - from bittensor.utils.balance import Balance + + +def _do_transfer( + subtensor: "Subtensor", + wallet: "Wallet", + destination: str, + amount: "Balance", + wait_for_inclusion: bool = True, + wait_for_finalization: bool = False, +) -> tuple[bool, str, str]: + """ + Makes transfer from wallet to destination public key address. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): the Subtensor object used for transfer + wallet (bittensor_wallet.Wallet): Bittensor wallet object to make transfer from. + destination (str): Destination public key address (ss58_address or ed25519) of recipient. + amount (bittensor.utils.balance.Balance): Amount to stake as Bittensor balance. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + + Returns: + success, block hash, formatted error message + """ + call = subtensor.substrate.compose_call( + call_module="Balances", + call_function="transfer_allow_death", + call_params={"dest": destination, "value": amount.rao}, + ) + extrinsic = subtensor.substrate.create_signed_extrinsic( + call=call, keypair=wallet.coldkey + ) + response = subtensor.substrate.submit_extrinsic( + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, "", "Success, extrinsic submitted without waiting." + + # Otherwise continue with finalization. + if response.is_success: + block_hash_ = response.block_hash + return True, block_hash_, "Success with response." + + return False, "", format_error_message(response.error_message) def transfer_extrinsic( @@ -20,15 +74,100 @@ def transfer_extrinsic( wait_for_finalization: bool = False, keep_alive: bool = True, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_transfer_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - destination=dest, - amount=amount, - transfer_all=transfer_all, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - keep_alive=keep_alive, + """Transfers funds from this wallet to the destination public key address. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): the Subtensor object used for transfer + wallet (bittensor_wallet.Wallet): Bittensor wallet object to make transfer from. + dest (str): Destination public key address (ss58_address or ed25519) of recipient. + amount (bittensor.utils.balance.Balance): Amount to stake as Bittensor balance. + transfer_all (bool): Whether to transfer all funds from this wallet to the destination address. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning `True`, or returns + `False` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + `True`, or returns `False` if the extrinsic fails to be finalized within the timeout. + keep_alive (bool): If set, keeps the account alive by keeping the balance above the existential deposit. + + Returns: + success (bool): Flag is `True` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is `True`, regardless of its inclusion. + """ + destination = dest + # Validate destination address. + if not is_valid_bittensor_address_or_public_key(destination): + logging.error( + f":cross_mark: [red]Invalid destination SS58 address[/red]: {destination}" ) + return False + logging.info(f"Initiating transfer on network: {subtensor.network}") + # Unlock wallet coldkey. + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + # Check balance. + logging.info( + f":satellite: [magenta]Checking balance and fees on chain [/magenta] [blue]{subtensor.network}[/blue]" + ) + # check existential deposit and fee + logging.debug("Fetching existential and fee") + block = subtensor.get_current_block() + account_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address, block=block) + if not keep_alive: + # Check if the transfer should keep_alive the account + existential_deposit = Balance(0) + else: + existential_deposit = subtensor.get_existential_deposit(block=block) + + fee = subtensor.get_transfer_fee(wallet=wallet, dest=destination, value=amount.rao) + + # Check if we have enough balance. + if transfer_all is True: + amount = account_balance - fee - existential_deposit + if amount < Balance(0): + logging.error("Not enough balance to transfer") + return False + + if account_balance < (amount + fee + existential_deposit): + logging.error(":cross_mark: [red]Not enough balance[/red]") + logging.error(f"\t\tBalance:\t[blue]{account_balance}[/blue]") + logging.error(f"\t\tAmount:\t[blue]{amount}[/blue]") + logging.error(f"\t\tFor fee:\t[blue]{fee}[/blue]") + return False + + logging.info(":satellite: [magenta]Transferring... Date: Wed, 22 Jan 2025 17:20:55 +0200 Subject: [PATCH 19/52] Unstaking extrinsic --- .../core/extrinsics/asyncex/unstaking.py | 79 ++-- bittensor/core/extrinsics/unstaking.py | 421 +++++++++++++++++- bittensor/core/subtensor.py | 6 +- 3 files changed, 444 insertions(+), 62 deletions(-) diff --git a/bittensor/core/extrinsics/asyncex/unstaking.py b/bittensor/core/extrinsics/asyncex/unstaking.py index e83bcc188e..fd57578bab 100644 --- a/bittensor/core/extrinsics/asyncex/unstaking.py +++ b/bittensor/core/extrinsics/asyncex/unstaking.py @@ -2,7 +2,7 @@ from typing import Union, Optional, TYPE_CHECKING from bittensor.core.errors import StakeError, NotRegisteredError -from bittensor.utils import format_error_message, unlock_key +from bittensor.utils import unlock_key from bittensor.utils.balance import Balance from bittensor.utils.btlogging import logging @@ -18,17 +18,19 @@ async def _check_threshold_amount( Checks if the remaining stake balance is above the minimum required stake threshold. Args: - subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): Subtensor instance. stake_balance (bittensor.utils.balance.Balance): the balance to check for threshold limits. Returns: - success (bool): ``true`` if the unstaking is above the threshold or 0, or ``false`` if the unstaking is below the threshold, but not 0. + success (bool): `True` if the unstaking is above the threshold or 0, or `False` if the unstaking is below the + threshold, but not 0. """ min_req_stake: Balance = await subtensor.get_minimum_required_stake() if min_req_stake > stake_balance > 0: logging.warning( - f":cross_mark: [yellow]Remaining stake balance of {stake_balance} less than minimum of {min_req_stake} TAO[/yellow]" + f":cross_mark: [yellow]Remaining stake balance of {stake_balance} less than minimum of " + f"{min_req_stake} TAO[/yellow]" ) return False else: @@ -49,8 +51,8 @@ async def _do_unstake( wallet (bittensor_wallet.Wallet): Wallet object that can sign the extrinsic. hotkey_ss58 (str): Hotkey ``ss58`` address to unstake from. amount (bittensor.utils.balance.Balance): Amount to unstake. - wait_for_inclusion (bool): If ``true``, waits for inclusion before returning. - wait_for_finalization (bool): If ``true``, waits for finalization before returning. + wait_for_inclusion (bool): If ``True``, waits for inclusion before returning. + wait_for_finalization (bool): If ``True``, waits for finalization before returning. Returns: success (bool): ``True`` if the extrinsic was successful. @@ -64,22 +66,13 @@ async def _do_unstake( call_function="remove_stake", call_params={"hotkey": hotkey_ss58, "amount_unstaked": amount.rao}, ) - extrinsic = await subtensor.substrate.create_signed_extrinsic( - call=call, keypair=wallet.coldkey - ) - response = await subtensor.substrate.submit_extrinsic( - extrinsic=extrinsic, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + success, err_msg = await subtensor.sign_and_send_extrinsic( + call, wallet, wait_for_inclusion, wait_for_finalization ) - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - return True - - if await response.is_success: - return True - - raise StakeError(format_error_message(await response.error_message)) + if success: + return success + else: + raise StakeError(err_msg) async def __do_remove_stake_single( @@ -97,14 +90,14 @@ async def __do_remove_stake_single( wallet (bittensor_wallet.Wallet): Bittensor wallet object. hotkey_ss58 (str): Hotkey address to unstake from. amount (bittensor.utils.balance.Balance): Amount to unstake as Bittensor balance object. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or - returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning - ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for - finalization / inclusion, the response is ``true``. + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. Raises: bittensor.core.errors.StakeError: If the extrinsic fails to be finalized or included in the block. @@ -139,19 +132,19 @@ async def unstake_extrinsic( """Removes stake into the wallet coldkey from the specified hotkey ``uid``. Args: - subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): AsyncSubtensor instance. wallet (bittensor_wallet.Wallet): Bittensor wallet object. hotkey_ss58 (Optional[str]): The ``ss58`` address of the hotkey to unstake from. By default, the wallet hotkey is used. amount (Union[Balance, float]): Amount to stake as Bittensor balance, or ``float`` interpreted as Tao. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or - returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning - ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for - finalization / inclusion, the response is ``true``. + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. """ # Decrypt keys, if not (unlock := unlock_key(wallet)).success: @@ -205,7 +198,8 @@ async def unstake_extrinsic( try: logging.info( - f":satellite: [magenta]Unstaking from chain:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + f":satellite: [magenta]Unstaking from chain:[/magenta] [blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" ) staking_response: bool = await __do_remove_stake_single( subtensor=subtensor, @@ -224,7 +218,8 @@ async def unstake_extrinsic( logging.success(":white_heavy_check_mark: [green]Finalized[/green]") logging.info( - f":satellite: [magenta]Checking Balance on:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + f":satellite: [magenta]Checking Balance on:[/magenta] [blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" ) block_hash = await subtensor.substrate.get_chain_head() new_balance, new_stake = await asyncio.gather( @@ -237,7 +232,7 @@ async def unstake_extrinsic( block_hash=block_hash, ), ) - logging.info(f"Balance:") + logging.info("Balance:") logging.info( f"\t\t[blue]{old_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" ) @@ -256,7 +251,7 @@ async def unstake_extrinsic( ) return False except StakeError as e: - logging.error(":cross_mark: [red]Stake Error: {}[/red]".format(e)) + logging.error(f":cross_mark: [red]Stake Error: {e}[/red]") return False @@ -275,14 +270,14 @@ async def unstake_multiple_extrinsic( wallet (bittensor_wallet.Wallet): The wallet with the coldkey to unstake to. hotkey_ss58s (List[str]): List of hotkeys to unstake from. amounts (List[Union[Balance, float]]): List of amounts to unstake. If ``None``, unstake all. - wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or - returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning - ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. Returns: - success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. Flag is ``true`` if any - wallet was unstaked. If we did not wait for finalization / inclusion, the response is ``true``. + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. Flag is ``True`` if any + wallet was unstaked. If we did not wait for finalization / inclusion, the response is ``True``. """ if not isinstance(hotkey_ss58s, list) or not all( isinstance(hotkey_ss58, str) for hotkey_ss58 in hotkey_ss58s @@ -320,8 +315,6 @@ async def unstake_multiple_extrinsic( logging.error(unlock.message) return False - old_stakes = [] - own_hotkeys = [] logging.info( f":satellite: [magenta]Syncing with chain:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" ) diff --git a/bittensor/core/extrinsics/unstaking.py b/bittensor/core/extrinsics/unstaking.py index c647daef3f..1c48eeb7ad 100644 --- a/bittensor/core/extrinsics/unstaking.py +++ b/bittensor/core/extrinsics/unstaking.py @@ -1,16 +1,124 @@ +import time from typing import Union, Optional, TYPE_CHECKING -from bittensor.core.extrinsics.asyncex.unstaking import ( - unstake_extrinsic as async_unstake_extrinsic, - unstake_multiple_extrinsic as async_unstake_multiple_extrinsic, -) +from bittensor.core.errors import StakeError, NotRegisteredError +from bittensor.utils import unlock_key from bittensor.utils.balance import Balance +from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor +def _check_threshold_amount(subtensor: "Subtensor", stake_balance: "Balance") -> bool: + """ + Checks if the remaining stake balance is above the minimum required stake threshold. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + stake_balance (bittensor.utils.balance.Balance): the balance to check for threshold limits. + + Returns: + success (bool): `True` if the unstaking is above the threshold or 0, or `False` if the unstaking is below the + threshold, but not 0. + """ + min_req_stake: Balance = subtensor.get_minimum_required_stake() + + if min_req_stake > stake_balance > 0: + logging.warning( + f":cross_mark: [yellow]Remaining stake balance of {stake_balance} less than minimum of " + f"{min_req_stake} TAO[/yellow]" + ) + return False + else: + return True + + +def _do_unstake( + subtensor: "Subtensor", + wallet: "Wallet", + hotkey_ss58: str, + amount: "Balance", + wait_for_inclusion: bool = True, + wait_for_finalization: bool = False, +) -> bool: + """Sends an unstake extrinsic to the chain. + + Args: + wallet (bittensor_wallet.Wallet): Wallet object that can sign the extrinsic. + hotkey_ss58 (str): Hotkey ``ss58`` address to unstake from. + amount (bittensor.utils.balance.Balance): Amount to unstake. + wait_for_inclusion (bool): If ``True``, waits for inclusion before returning. + wait_for_finalization (bool): If ``True``, waits for finalization before returning. + + Returns: + success (bool): ``True`` if the extrinsic was successful. + + Raises: + StakeError: If the extrinsic failed. + """ + + call = subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="remove_stake", + call_params={"hotkey": hotkey_ss58, "amount_unstaked": amount.rao}, + ) + success, err_msg = subtensor.sign_and_send_extrinsic( + call, wallet, wait_for_inclusion, wait_for_finalization + ) + if success: + return success + else: + raise StakeError(err_msg) + + +def __do_remove_stake_single( + subtensor: "Subtensor", + wallet: "Wallet", + hotkey_ss58: str, + amount: "Balance", + wait_for_inclusion: bool = True, + wait_for_finalization: bool = False, +) -> bool: + """ + Executes an unstake call to the chain using the wallet and the amount specified. + + Args: + wallet (bittensor_wallet.Wallet): Bittensor wallet object. + hotkey_ss58 (str): Hotkey address to unstake from. + amount (bittensor.utils.balance.Balance): Amount to unstake as Bittensor balance object. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. + + Returns: + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. + + Raises: + bittensor.core.errors.StakeError: If the extrinsic fails to be finalized or included in the block. + bittensor.core.errors.NotRegisteredError: If the hotkey is not registered in any subnets. + + """ + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + success = _do_unstake( + subtensor=subtensor, + wallet=wallet, + hotkey_ss58=hotkey_ss58, + amount=amount, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if success: + return True + + def unstake_extrinsic( subtensor: "Subtensor", wallet: "Wallet", @@ -19,16 +127,126 @@ def unstake_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_unstake_extrinsic( - subtensor=subtensor.async_subtensor, + """Removes stake into the wallet coldkey from the specified hotkey ``uid``. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + wallet (bittensor_wallet.Wallet): Bittensor wallet object. + hotkey_ss58 (Optional[str]): The ``ss58`` address of the hotkey to unstake from. By default, the wallet hotkey + is used. + amount (Union[Balance, float]): Amount to stake as Bittensor balance, or ``float`` interpreted as Tao. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. + + Returns: + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. If we did not wait for + finalization / inclusion, the response is ``True``. + """ + # Decrypt keys, + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + if hotkey_ss58 is None: + hotkey_ss58 = wallet.hotkey.ss58_address # Default to wallet's own hotkey. + + logging.info( + f":satellite: [magenta]Syncing with chain:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + ) + block = subtensor.get_current_block() + old_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address, block=block) + old_stake = subtensor.get_stake_for_coldkey_and_hotkey( + coldkey_ss58=wallet.coldkeypub.ss58_address, + hotkey_ss58=hotkey_ss58, + block=block, + ) + hotkey_owner = subtensor.get_hotkey_owner(hotkey_ss58, block=block) + own_hotkey: bool = wallet.coldkeypub.ss58_address == hotkey_owner + + # Convert to bittensor.Balance + if amount is None: + # Unstake it all. + unstaking_balance = old_stake + elif not isinstance(amount, Balance): + unstaking_balance = Balance.from_tao(amount) + else: + unstaking_balance = amount + + # Check enough to unstake. + stake_on_uid = old_stake + if unstaking_balance > stake_on_uid: + logging.error( + f":cross_mark: [red]Not enough stake[/red]: [green]{stake_on_uid}[/green] to unstake: " + f"[blue]{unstaking_balance}[/blue] from hotkey: [yellow]{wallet.hotkey_str}[/yellow]" + ) + return False + + # If nomination stake, check threshold. + if not own_hotkey and not _check_threshold_amount( + subtensor=subtensor, stake_balance=(stake_on_uid - unstaking_balance) + ): + logging.warning( + ":warning: [yellow]This action will unstake the entire staked balance![/yellow]" + ) + unstaking_balance = stake_on_uid + + try: + logging.info( + f":satellite: [magenta]Unstaking from chain:[/magenta] [blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" + ) + staking_response: bool = __do_remove_stake_single( + subtensor=subtensor, wallet=wallet, hotkey_ss58=hotkey_ss58, - amount=amount, + amount=unstaking_balance, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - ) + + if staking_response is True: # If we successfully unstaked. + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True + + logging.success(":white_heavy_check_mark: [green]Finalized[/green]") + + logging.info( + f":satellite: [magenta]Checking Balance on:[/magenta] [blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" + ) + block = subtensor.get_current_block() + new_balance = subtensor.get_balance( + wallet.coldkeypub.ss58_address, block=block + ) + new_stake = subtensor.get_stake_for_coldkey_and_hotkey( + coldkey_ss58=wallet.coldkeypub.ss58_address, + hotkey_ss58=hotkey_ss58, + block=block, + ) + logging.info("Balance:") + logging.info( + f"\t\t[blue]{old_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + ) + logging.info("Stake:") + logging.info( + f"\t\t[blue]{old_stake}[/blue] :arrow_right: [green]{new_stake}[/green]" + ) + return True + else: + logging.error(":cross_mark: [red]Failed[/red]: Unknown Error.") + return False + + except NotRegisteredError: + logging.error( + f":cross_mark: [red]Hotkey: {wallet.hotkey_str} is not registered.[/red]" + ) + return False + except StakeError as e: + logging.error(f":cross_mark: [red]Stake Error: {e}[/red]") + return False def unstake_multiple_extrinsic( @@ -39,13 +257,182 @@ def unstake_multiple_extrinsic( wait_for_inclusion: bool = True, wait_for_finalization: bool = False, ) -> bool: - return subtensor.execute_coroutine( - coroutine=async_unstake_multiple_extrinsic( - subtensor=subtensor.async_subtensor, - wallet=wallet, - hotkey_ss58s=hotkey_ss58s, - amounts=amounts, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + """Removes stake from each ``hotkey_ss58`` in the list, using each amount, to a common coldkey. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): Subtensor instance. + wallet (bittensor_wallet.Wallet): The wallet with the coldkey to unstake to. + hotkey_ss58s (List[str]): List of hotkeys to unstake from. + amounts (List[Union[Balance, float]]): List of amounts to unstake. If ``None``, unstake all. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``True``, or + returns ``False`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning + ``True``, or returns ``False`` if the extrinsic fails to be finalized within the timeout. + + Returns: + success (bool): Flag is ``True`` if extrinsic was finalized or included in the block. Flag is ``True`` if any + wallet was unstaked. If we did not wait for finalization / inclusion, the response is ``True``. + """ + if not isinstance(hotkey_ss58s, list) or not all( + isinstance(hotkey_ss58, str) for hotkey_ss58 in hotkey_ss58s + ): + raise TypeError("hotkey_ss58s must be a list of str") + + if len(hotkey_ss58s) == 0: + return True + + if amounts is not None and len(amounts) != len(hotkey_ss58s): + raise ValueError("amounts must be a list of the same length as hotkey_ss58s") + + if amounts is not None and not all( + isinstance(amount, (Balance, float)) for amount in amounts + ): + raise TypeError( + "amounts must be a [list of bittensor.Balance or float] or None" ) + + if amounts is None: + amounts = [None] * len(hotkey_ss58s) + else: + # Convert to Balance + amounts = [ + Balance.from_tao(amount) if isinstance(amount, float) else amount + for amount in amounts + ] + + if sum(amount.tao for amount in amounts) == 0: + # Staking 0 tao + return True + + # Unlock coldkey. + if not (unlock := unlock_key(wallet)).success: + logging.error(unlock.message) + return False + + logging.info( + f":satellite: [magenta]Syncing with chain:[/magenta] [blue]{subtensor.network}[/blue] [magenta]...[/magenta]" ) + block = subtensor.get_current_block() + old_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address, block=block) + # these calls can probably be sped up with query_multi, but honestly if you're looking for + # concurrency speed, use AsyncSubtensor + old_stakes = [ + subtensor.get_stake_for_coldkey_and_hotkey( + coldkey_ss58=wallet.coldkeypub.ss58_address, + hotkey_ss58=hotkey_ss58, + block=block, + ) + for hotkey_ss58 in hotkey_ss58s + ] + hotkeys_ = [ + subtensor.get_hotkey_owner(hotkey_ss58, block=block) + for hotkey_ss58 in hotkey_ss58s + ] + + own_hotkeys = [ + (wallet.coldkeypub.ss58_address == hotkey_owner) for hotkey_owner in hotkeys_ + ] + + successful_unstakes = 0 + for idx, (hotkey_ss58, amount, old_stake, own_hotkey) in enumerate( + zip(hotkey_ss58s, amounts, old_stakes, own_hotkeys) + ): + # Covert to bittensor.Balance + if amount is None: + # Unstake it all. + unstaking_balance = old_stake + else: + unstaking_balance = ( + amount if isinstance(amount, Balance) else Balance.from_tao(amount) + ) + + # Check enough to unstake. + stake_on_uid = old_stake + if unstaking_balance > stake_on_uid: + logging.error( + f":cross_mark: [red]Not enough stake[/red]: [green]{stake_on_uid}[/green] to unstake: " + f"[blue]{unstaking_balance}[/blue] from hotkey: [blue]{wallet.hotkey_str}[/blue]." + ) + continue + + # If nomination stake, check threshold. + if not own_hotkey and not _check_threshold_amount( + subtensor=subtensor, stake_balance=(stake_on_uid - unstaking_balance) + ): + logging.warning( + ":warning: [yellow]This action will unstake the entire staked balance![/yellow]" + ) + unstaking_balance = stake_on_uid + + try: + logging.info( + f":satellite: [magenta]Unstaking from chain:[/magenta] [blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" + ) + staking_response: bool = __do_remove_stake_single( + subtensor=subtensor, + wallet=wallet, + hotkey_ss58=hotkey_ss58, + amount=unstaking_balance, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if staking_response is True: # If we successfully unstaked. + # We only wait here if we expect finalization. + + if idx < len(hotkey_ss58s) - 1: + # Wait for tx rate limit. + tx_rate_limit_blocks = subtensor.tx_rate_limit() + if tx_rate_limit_blocks > 0: + logging.info( + f":hourglass: [yellow]Waiting for tx rate limit: " + f"[white]{tx_rate_limit_blocks}[/white] blocks[/yellow]" + ) + time.sleep(tx_rate_limit_blocks * 12) # 12 seconds per block + + if not wait_for_finalization and not wait_for_inclusion: + successful_unstakes += 1 + continue + + logging.info(":white_heavy_check_mark: [green]Finalized[/green]") + + logging.info( + f":satellite: [magenta]Checking Balance on:[/magenta] [blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]..." + ) + block = subtensor.get_current_block() + new_stake = subtensor.get_stake_for_coldkey_and_hotkey( + coldkey_ss58=wallet.coldkeypub.ss58_address, + hotkey_ss58=hotkey_ss58, + block=block, + ) + logging.info( + f"Stake ({hotkey_ss58}): [blue]{stake_on_uid}[/blue] :arrow_right: [green]{new_stake}[/green]" + ) + successful_unstakes += 1 + else: + logging.error(":cross_mark: [red]Failed: Unknown Error.[/red]") + continue + + except NotRegisteredError: + logging.error( + f":cross_mark: [red]Hotkey[/red] [blue]{hotkey_ss58}[/blue] [red]is not registered.[/red]" + ) + continue + except StakeError as e: + logging.error(":cross_mark: [red]Stake Error: {}[/red]".format(e)) + continue + + if successful_unstakes != 0: + logging.info( + f":satellite: [magenta]Checking Balance on:[/magenta] ([blue]{subtensor.network}[/blue] " + f"[magenta]...[/magenta]" + ) + new_balance = subtensor.get_balance(wallet.coldkeypub.ss58_address) + logging.info( + f"Balance: [blue]{old_balance}[/blue] :arrow_right: [green]{new_balance}[/green]" + ) + return True + + return False diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 8dbab64665..8894f4eab5 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -38,6 +38,10 @@ add_stake_multiple_extrinsic, ) from bittensor.core.extrinsics.transfer import transfer_extrinsic +from bittensor.core.extrinsics.unstaking import ( + unstake_extrinsic, + unstake_multiple_extrinsic, +) from bittensor.core.metagraph import Metagraph from bittensor.core.extrinsics.serving import ( publish_metadata, @@ -2571,7 +2575,6 @@ def unstake( This function supports flexible stake management, allowing neurons to adjust their network participation and potential reward accruals. """ - # TODO add extrinsic return unstake_extrinsic( subtensor=self, wallet=wallet, @@ -2608,7 +2611,6 @@ def unstake_multiple( This function allows for strategic reallocation or withdrawal of stakes, aligning with the dynamic stake management aspect of the Bittensor network. """ - # TODO add extrinsic return unstake_multiple_extrinsic( subtensor=self, wallet=wallet, From 04eda9a5a2491c153c3e8e491655380e08adc969 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 17:23:38 +0200 Subject: [PATCH 20/52] Cleanup --- bittensor/core/subtensor.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 8894f4eab5..963c52c021 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -85,9 +85,7 @@ class Subtensor(SubtensorMixin): - """ - TODO docstring - """ + """Thin layer for interacting with Substrate Interface. Mostly a collection of frequently-used calls.""" def __init__( self, @@ -97,7 +95,7 @@ def __init__( log_verbose: bool = False, ): """ - Initializes an instance of the AsyncSubtensor class. + Initializes an instance of the Subtensor class. Arguments: network (str): The network name or type to connect to. From 8f1d639d55b65e58d5cf1e86cbd6fc876a52ac16 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 17:28:48 +0200 Subject: [PATCH 21/52] Move SubtensorMixin to types file to avoid import conflicts. --- bittensor/core/__init__.py | 201 ------------------------------ bittensor/core/async_subtensor.py | 2 +- bittensor/core/subtensor.py | 2 +- bittensor/core/types.py | 200 ++++++++++++++++++++++++++++- 4 files changed, 201 insertions(+), 204 deletions(-) diff --git a/bittensor/core/__init__.py b/bittensor/core/__init__.py index e8411b35c2..e69de29bb2 100644 --- a/bittensor/core/__init__.py +++ b/bittensor/core/__init__.py @@ -1,201 +0,0 @@ -from abc import ABC -import argparse -from typing import Optional - -from bittensor.utils import networking -from bittensor.utils.btlogging import logging -from bittensor.core import settings -from bittensor.core.config import Config - - -class SubtensorMixin(ABC): - network: str - chain_endpoint: str - log_verbose: bool - - def __str__(self): - return f"Network: {self.network}, Chain: {self.chain_endpoint}" - - def __repr__(self): - return self.__str__() - - def _check_and_log_network_settings(self): - if self.network == settings.NETWORKS[3]: # local - logging.warning( - ":warning: Verify your local subtensor is running on port [blue]9944[/blue]." - ) - - if ( - self.network == "finney" - or self.chain_endpoint == settings.FINNEY_ENTRYPOINT - ) and self.log_verbose: - logging.info( - f"You are connecting to {self.network} network with endpoint {self.chain_endpoint}." - ) - logging.debug( - "We strongly encourage running a local subtensor node whenever possible. " - "This increases decentralization and resilience of the network." - ) - # TODO: remove or apply this warning as updated default endpoint? - logging.debug( - "In a future release, local subtensor will become the default endpoint. " - "To get ahead of this change, please run a local subtensor node and point to it." - ) - - @staticmethod # TODO can this be a class method? - def config() -> "Config": - """ - Creates and returns a Bittensor configuration object. - - Returns: - config (bittensor.core.config.Config): A Bittensor configuration object configured with arguments added by - the `subtensor.add_args` method. - """ - parser = argparse.ArgumentParser() - SubtensorMixin.add_args(parser) - return Config(parser) - - @staticmethod - def setup_config(network: Optional[str], config: "Config"): - """ - Sets up and returns the configuration for the Subtensor network and endpoint. - - This method determines the appropriate network and chain endpoint based on the provided network string or - configuration object. It evaluates the network and endpoint in the following order of precedence: - 1. Provided network string. - 2. Configured chain endpoint in the `config` object. - 3. Configured network in the `config` object. - 4. Default chain endpoint. - 5. Default network. - - Arguments: - network (Optional[str]): The name of the Subtensor network. If None, the network and endpoint will be - determined from the `config` object. - config (bittensor.core.config.Config): The configuration object containing the network and chain endpoint - settings. - - Returns: - tuple: A tuple containing the formatted WebSocket endpoint URL and the evaluated network name. - """ - if network is None: - candidates = [ - ( - config.is_set("subtensor.chain_endpoint"), - config.subtensor.chain_endpoint, - ), - (config.is_set("subtensor.network"), config.subtensor.network), - ( - config.subtensor.get("chain_endpoint"), - config.subtensor.chain_endpoint, - ), - (config.subtensor.get("network"), config.subtensor.network), - ] - for check, config_network in candidates: - if check: - network = config_network - - evaluated_network, evaluated_endpoint = ( - SubtensorMixin.determine_chain_endpoint_and_network(network) - ) - - return networking.get_formatted_ws_endpoint_url( - evaluated_endpoint - ), evaluated_network - - @classmethod - def help(cls): - """Print help to stdout.""" - parser = argparse.ArgumentParser() - cls.add_args(parser) - print(cls.__new__.__doc__) - parser.print_help() - - @classmethod - def add_args(cls, parser: "argparse.ArgumentParser", prefix: Optional[str] = None): - """ - Adds command-line arguments to the provided ArgumentParser for configuring the Subtensor settings. - - Arguments: - parser (argparse.ArgumentParser): The ArgumentParser object to which the Subtensor arguments will be added. - prefix (Optional[str]): An optional prefix for the argument names. If provided, the prefix is prepended to - each argument name. - - Arguments added: - --subtensor.network: The Subtensor network flag. Possible values are 'finney', 'test', 'archive', and - 'local'. Overrides the chain endpoint if set. - --subtensor.chain_endpoint: The Subtensor chain endpoint flag. If set, it overrides the network flag. - --subtensor._mock: If true, uses a mocked connection to the chain. - - Example: - parser = argparse.ArgumentParser() - Subtensor.add_args(parser) - """ - prefix_str = "" if prefix is None else f"{prefix}." - try: - default_network = settings.DEFAULT_NETWORK - default_chain_endpoint = settings.FINNEY_ENTRYPOINT - - parser.add_argument( - f"--{prefix_str}subtensor.network", - default=default_network, - type=str, - help="""The subtensor network flag. The likely choices are: - -- finney (main network) - -- test (test network) - -- archive (archive network +300 blocks) - -- local (local running network) - If this option is set it overloads subtensor.chain_endpoint with - an entry point node from that network. - """, - ) - parser.add_argument( - f"--{prefix_str}subtensor.chain_endpoint", - default=default_chain_endpoint, - type=str, - help="""The subtensor endpoint flag. If set, overrides the --network flag.""", - ) - parser.add_argument( - f"--{prefix_str}subtensor._mock", - default=False, - type=bool, - help="""If true, uses a mocked connection to the chain.""", - ) - - except argparse.ArgumentError: - # re-parsing arguments. - pass - - @staticmethod - def determine_chain_endpoint_and_network( - network: str, - ) -> tuple[Optional[str], Optional[str]]: - """Determines the chain endpoint and network from the passed network or chain_endpoint. - - Arguments: - network (str): The network flag. The choices are: ``finney`` (main network), ``archive`` (archive network - +300 blocks), ``local`` (local running network), ``test`` (test network). - - Returns: - tuple[Optional[str], Optional[str]]: The network and chain endpoint flag. If passed, overrides the - ``network`` argument. - """ - - if network is None: - return None, None - if network in settings.NETWORKS: - return network, settings.NETWORK_MAP[network] - - substrings_map = { - "entrypoint-finney.opentensor.ai": ("finney", settings.FINNEY_ENTRYPOINT), - "test.finney.opentensor.ai": ("test", settings.FINNEY_TEST_ENTRYPOINT), - "archive.chain.opentensor.ai": ("archive", settings.ARCHIVE_ENTRYPOINT), - "subvortex": ("subvortex", settings.SUBVORTEX_ENTRYPOINT), - "127.0.0.1": ("local", settings.LOCAL_ENTRYPOINT), - "localhost": ("local", settings.LOCAL_ENTRYPOINT), - } - - for substring, result in substrings_map.items(): - if substring in network: - return result - - return "unknown", network diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 32e2a258b8..f275174fc8 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -14,7 +14,7 @@ from scalecodec.base import RuntimeConfiguration from scalecodec.type_registry import load_type_registry_preset -from bittensor.core import SubtensorMixin +from bittensor.core.types import SubtensorMixin from bittensor.core.chain_data import ( DelegateInfo, StakeInfo, diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 963c52c021..e7c25c0b14 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -13,7 +13,7 @@ from scalecodec.type_registry import load_type_registry_preset import ujson -from bittensor.core import SubtensorMixin +from bittensor.core.types import SubtensorMixin from bittensor.core.chain_data import ( custom_rpc_type_registry, decode_account_id, diff --git a/bittensor/core/types.py b/bittensor/core/types.py index 908e384015..601dc471cb 100644 --- a/bittensor/core/types.py +++ b/bittensor/core/types.py @@ -15,8 +15,206 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +from abc import ABC +import argparse from typing import TypedDict, Optional -from bittensor.utils import Certificate +from bittensor.utils import networking, Certificate +from bittensor.utils.btlogging import logging +from bittensor.core import settings +from bittensor.core.config import Config + + +class SubtensorMixin(ABC): + network: str + chain_endpoint: str + log_verbose: bool + + def __str__(self): + return f"Network: {self.network}, Chain: {self.chain_endpoint}" + + def __repr__(self): + return self.__str__() + + def _check_and_log_network_settings(self): + if self.network == settings.NETWORKS[3]: # local + logging.warning( + ":warning: Verify your local subtensor is running on port [blue]9944[/blue]." + ) + + if ( + self.network == "finney" + or self.chain_endpoint == settings.FINNEY_ENTRYPOINT + ) and self.log_verbose: + logging.info( + f"You are connecting to {self.network} network with endpoint {self.chain_endpoint}." + ) + logging.debug( + "We strongly encourage running a local subtensor node whenever possible. " + "This increases decentralization and resilience of the network." + ) + # TODO: remove or apply this warning as updated default endpoint? + logging.debug( + "In a future release, local subtensor will become the default endpoint. " + "To get ahead of this change, please run a local subtensor node and point to it." + ) + + @staticmethod # TODO can this be a class method? + def config() -> "Config": + """ + Creates and returns a Bittensor configuration object. + + Returns: + config (bittensor.core.config.Config): A Bittensor configuration object configured with arguments added by + the `subtensor.add_args` method. + """ + parser = argparse.ArgumentParser() + SubtensorMixin.add_args(parser) + return Config(parser) + + @staticmethod + def setup_config(network: Optional[str], config: "Config"): + """ + Sets up and returns the configuration for the Subtensor network and endpoint. + + This method determines the appropriate network and chain endpoint based on the provided network string or + configuration object. It evaluates the network and endpoint in the following order of precedence: + 1. Provided network string. + 2. Configured chain endpoint in the `config` object. + 3. Configured network in the `config` object. + 4. Default chain endpoint. + 5. Default network. + + Arguments: + network (Optional[str]): The name of the Subtensor network. If None, the network and endpoint will be + determined from the `config` object. + config (bittensor.core.config.Config): The configuration object containing the network and chain endpoint + settings. + + Returns: + tuple: A tuple containing the formatted WebSocket endpoint URL and the evaluated network name. + """ + if network is None: + candidates = [ + ( + config.is_set("subtensor.chain_endpoint"), + config.subtensor.chain_endpoint, + ), + (config.is_set("subtensor.network"), config.subtensor.network), + ( + config.subtensor.get("chain_endpoint"), + config.subtensor.chain_endpoint, + ), + (config.subtensor.get("network"), config.subtensor.network), + ] + for check, config_network in candidates: + if check: + network = config_network + + evaluated_network, evaluated_endpoint = ( + SubtensorMixin.determine_chain_endpoint_and_network(network) + ) + + return networking.get_formatted_ws_endpoint_url( + evaluated_endpoint + ), evaluated_network + + @classmethod + def help(cls): + """Print help to stdout.""" + parser = argparse.ArgumentParser() + cls.add_args(parser) + print(cls.__new__.__doc__) + parser.print_help() + + @classmethod + def add_args(cls, parser: "argparse.ArgumentParser", prefix: Optional[str] = None): + """ + Adds command-line arguments to the provided ArgumentParser for configuring the Subtensor settings. + + Arguments: + parser (argparse.ArgumentParser): The ArgumentParser object to which the Subtensor arguments will be added. + prefix (Optional[str]): An optional prefix for the argument names. If provided, the prefix is prepended to + each argument name. + + Arguments added: + --subtensor.network: The Subtensor network flag. Possible values are 'finney', 'test', 'archive', and + 'local'. Overrides the chain endpoint if set. + --subtensor.chain_endpoint: The Subtensor chain endpoint flag. If set, it overrides the network flag. + --subtensor._mock: If true, uses a mocked connection to the chain. + + Example: + parser = argparse.ArgumentParser() + Subtensor.add_args(parser) + """ + prefix_str = "" if prefix is None else f"{prefix}." + try: + default_network = settings.DEFAULT_NETWORK + default_chain_endpoint = settings.FINNEY_ENTRYPOINT + + parser.add_argument( + f"--{prefix_str}subtensor.network", + default=default_network, + type=str, + help="""The subtensor network flag. The likely choices are: + -- finney (main network) + -- test (test network) + -- archive (archive network +300 blocks) + -- local (local running network) + If this option is set it overloads subtensor.chain_endpoint with + an entry point node from that network. + """, + ) + parser.add_argument( + f"--{prefix_str}subtensor.chain_endpoint", + default=default_chain_endpoint, + type=str, + help="""The subtensor endpoint flag. If set, overrides the --network flag.""", + ) + parser.add_argument( + f"--{prefix_str}subtensor._mock", + default=False, + type=bool, + help="""If true, uses a mocked connection to the chain.""", + ) + + except argparse.ArgumentError: + # re-parsing arguments. + pass + + @staticmethod + def determine_chain_endpoint_and_network( + network: str, + ) -> tuple[Optional[str], Optional[str]]: + """Determines the chain endpoint and network from the passed network or chain_endpoint. + + Arguments: + network (str): The network flag. The choices are: ``finney`` (main network), ``archive`` (archive network + +300 blocks), ``local`` (local running network), ``test`` (test network). + + Returns: + tuple[Optional[str], Optional[str]]: The network and chain endpoint flag. If passed, overrides the + ``network`` argument. + """ + + if network is None: + return None, None + if network in settings.NETWORKS: + return network, settings.NETWORK_MAP[network] + + substrings_map = { + "entrypoint-finney.opentensor.ai": ("finney", settings.FINNEY_ENTRYPOINT), + "test.finney.opentensor.ai": ("test", settings.FINNEY_TEST_ENTRYPOINT), + "archive.chain.opentensor.ai": ("archive", settings.ARCHIVE_ENTRYPOINT), + "subvortex": ("subvortex", settings.SUBVORTEX_ENTRYPOINT), + "127.0.0.1": ("local", settings.LOCAL_ENTRYPOINT), + "localhost": ("local", settings.LOCAL_ENTRYPOINT), + } + + for substring, result in substrings_map.items(): + if substring in network: + return result + + return "unknown", network class AxonServeCallParams(TypedDict): From 4acf20b761c3e5b9bc9bf254abd5b3f793dc6115 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 17:50:28 +0200 Subject: [PATCH 22/52] imports --- bittensor/core/extrinsics/commit_weights.py | 8 +------- bittensor/core/settings.py | 1 - bittensor/core/subtensor.py | 2 +- bittensor/utils/mock/subtensor_mock.py | 7 +++---- bittensor/utils/registration/pow.py | 2 +- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/bittensor/core/extrinsics/commit_weights.py b/bittensor/core/extrinsics/commit_weights.py index fc09aa4031..c92d8b9529 100644 --- a/bittensor/core/extrinsics/commit_weights.py +++ b/bittensor/core/extrinsics/commit_weights.py @@ -1,19 +1,13 @@ """Module sync commit weights and reveal weights extrinsic.""" -from typing import Union, TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional -import numpy as np -from numpy.typing import NDArray - -import bittensor.utils.weight_utils as weight_utils -from bittensor.core.settings import version_as_int from bittensor.utils import format_error_message from bittensor.utils.btlogging import logging if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor - from bittensor.utils.registration import torch def sign_and_send_with_nonce( diff --git a/bittensor/core/settings.py b/bittensor/core/settings.py index 68854802ce..fb1c3c8c63 100644 --- a/bittensor/core/settings.py +++ b/bittensor/core/settings.py @@ -19,7 +19,6 @@ import os import re -import warnings from pathlib import Path from munch import munchify diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index e7c25c0b14..0757590a7e 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -2311,7 +2311,7 @@ def root_register( try: recycle_call = self.get_hyperparameter( - param_name="Burn", netuid=netuid, block=block + param_name="Burn", netuid=0, block=block ) balance = (self.get_balance(wallet.coldkeypub.ss58_address, block=block),) except TypeError as e: diff --git a/bittensor/utils/mock/subtensor_mock.py b/bittensor/utils/mock/subtensor_mock.py index 28769fabe2..a6ff9a49a6 100644 --- a/bittensor/utils/mock/subtensor_mock.py +++ b/bittensor/utils/mock/subtensor_mock.py @@ -3,13 +3,12 @@ from hashlib import sha256 from types import SimpleNamespace from typing import Any, Optional, Union, TypedDict -from unittest.mock import MagicMock, patch, AsyncMock +from unittest.mock import MagicMock, patch from async_substrate_interface import SubstrateInterface from bittensor_wallet import Wallet import bittensor.core.subtensor as subtensor_module -from bittensor.core.async_subtensor import AsyncSubtensor from bittensor.core.chain_data import ( NeuronInfo, NeuronInfoLite, @@ -264,8 +263,8 @@ def __init__(self, *args, **kwargs) -> None: if not hasattr(self, "chain_state") or getattr(self, "chain_state") is None: self.setup() - def get_block_hash(self, block_id: int) -> str: - return "0x" + sha256(str(block_id).encode()).hexdigest()[:64] + def get_block_hash(self, block: Optional[int] = None) -> str: + return "0x" + sha256(str(block).encode()).hexdigest()[:64] def create_subnet(self, netuid: int) -> None: subtensor_state = self.chain_state["SubtensorModule"] diff --git a/bittensor/utils/registration/pow.py b/bittensor/utils/registration/pow.py index b70e5bc748..29eeedb7c7 100644 --- a/bittensor/utils/registration/pow.py +++ b/bittensor/utils/registration/pow.py @@ -13,7 +13,7 @@ from datetime import timedelta from multiprocessing.queues import Queue as QueueType from queue import Empty, Full -from typing import Any, Callable, Optional, Union, TYPE_CHECKING +from typing import Callable, Optional, Union, TYPE_CHECKING import numpy from Crypto.Hash import keccak From 1781e8b6da4b79afece64a806fd469cd71dc4a48 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 21:03:54 +0200 Subject: [PATCH 23/52] Remove ujson --- bittensor/core/subtensor.py | 5 ++--- requirements/prod.txt | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 0757590a7e..f1ae724eba 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -4,14 +4,13 @@ from async_substrate_interface.errors import SubstrateRequestException from async_substrate_interface.sync_substrate import SubstrateInterface -from async_substrate_interface.utils import hex_to_bytes +from async_substrate_interface.utils import hex_to_bytes, json import numpy as np from numpy.typing import NDArray import requests import scalecodec from scalecodec.base import RuntimeConfiguration from scalecodec.type_registry import load_type_registry_preset -import ujson from bittensor.core.types import SubtensorMixin from bittensor.core.chain_data import ( @@ -818,7 +817,7 @@ def get_delegate_identities( } ) if response.ok: - all_delegates: dict[str, Any] = ujson.loads(response.content) + all_delegates: dict[str, Any] = json.loads(response.content) for delegate_hotkey, delegate_details in all_delegates.items(): delegate_info = all_delegates_details.setdefault( diff --git a/requirements/prod.txt b/requirements/prod.txt index 20af79b989..e3fef4c9f2 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -15,7 +15,6 @@ packaging python-statemachine~=2.1 pycryptodome>=3.18.0,<4.0.0 pyyaml -ujson retry requests rich From c812324b8861e001f9f1e41b56d304599737f0d8 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 22 Jan 2025 21:13:40 +0200 Subject: [PATCH 24/52] Apply new json generally. --- bittensor/core/axon.py | 2 +- bittensor/core/chain_data/axon_info.py | 2 +- bittensor/utils/networking.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bittensor/core/axon.py b/bittensor/core/axon.py index 5b851b7e92..cd6f5e6d11 100644 --- a/bittensor/core/axon.py +++ b/bittensor/core/axon.py @@ -21,7 +21,6 @@ import contextlib import copy import inspect -import json import threading import time import traceback @@ -31,6 +30,7 @@ from inspect import signature, Signature, Parameter from typing import Any, Awaitable, Callable, Optional, Tuple +from async_substrate_interface.utils import json import uvicorn from bittensor_wallet import Wallet, Keypair diff --git a/bittensor/core/chain_data/axon_info.py b/bittensor/core/chain_data/axon_info.py index eee9cb82a1..9357301670 100644 --- a/bittensor/core/chain_data/axon_info.py +++ b/bittensor/core/chain_data/axon_info.py @@ -20,10 +20,10 @@ in the bittensor network. """ -import json from dataclasses import asdict, dataclass from typing import Any, Union +from async_substrate_interface.utils import json from bittensor.utils import networking from bittensor.utils.btlogging import logging from bittensor.utils.registration import torch, use_torch diff --git a/bittensor/utils/networking.py b/bittensor/utils/networking.py index c8a943e708..2c081305cc 100644 --- a/bittensor/utils/networking.py +++ b/bittensor/utils/networking.py @@ -1,10 +1,10 @@ """Utils for handling local network with ip and ports.""" -import json import os import urllib from typing import Optional +from async_substrate_interface.utils import json import netaddr import requests From 57cc58ea0026b826c183412f7b90eec324de3024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 11:54:22 +0100 Subject: [PATCH 25/52] test: fix async unittests --- .../extrinsics/asyncex/test_commit_reveal.py | 8 +-- .../extrinsics/asyncex/test_registration.py | 65 ++++++++++++++----- .../extrinsics/asyncex/test_root.py | 17 ++--- .../extrinsics/asyncex/test_transfer.py | 12 ++-- .../extrinsics/asyncex/test_weights.py | 12 ++-- tests/unit_tests/test_async_subtensor.py | 4 +- tests/unit_tests/test_subtensor.py | 2 +- 7 files changed, 75 insertions(+), 45 deletions(-) diff --git a/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py b/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py index 1dd7e6aab9..37ba957731 100644 --- a/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py +++ b/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py @@ -91,11 +91,11 @@ async def test_do_commit_reveal_v3_success(mocker, subtensor): call=mocked_compose_call.return_value, keypair=fake_wallet.hotkey ) mocked_submit_extrinsic.assert_awaited_once_with( - extrinsic=mocked_create_signed_extrinsic.return_value, + mocked_create_signed_extrinsic.return_value, wait_for_inclusion=False, wait_for_finalization=False, ) - assert result == (True, "Not waiting for finalization or inclusion.") + assert result == (True, "") @pytest.mark.asyncio @@ -121,7 +121,7 @@ async def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): ) mocked_format_error_message = mocker.patch.object( - async_commit_reveal, "format_error_message", return_value="Formatted error" + subtensor_module, "format_error_message", return_value="Formatted error", ) # Call @@ -149,7 +149,7 @@ async def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): call=mocked_compose_call.return_value, keypair=fake_wallet.hotkey ) mocked_submit_extrinsic.assert_awaited_once_with( - extrinsic=mocked_create_signed_extrinsic.return_value, + mocked_create_signed_extrinsic.return_value, wait_for_inclusion=True, wait_for_finalization=True, ) diff --git a/tests/unit_tests/extrinsics/asyncex/test_registration.py b/tests/unit_tests/extrinsics/asyncex/test_registration.py index 6baffe166c..82d9e6b561 100644 --- a/tests/unit_tests/extrinsics/asyncex/test_registration.py +++ b/tests/unit_tests/extrinsics/asyncex/test_registration.py @@ -70,10 +70,10 @@ async def test_do_pow_register_success(subtensor, mocker): call=fake_call, keypair=fake_wallet.hotkey ) subtensor.substrate.submit_extrinsic.assert_awaited_once_with( - extrinsic=fake_extrinsic, wait_for_inclusion=True, wait_for_finalization=True + fake_extrinsic, wait_for_inclusion=True, wait_for_finalization=True ) assert result is True - assert error_message is None + assert error_message == "" @pytest.mark.asyncio @@ -105,7 +105,7 @@ async def test_do_pow_register_failure(subtensor, mocker): subtensor.substrate, "submit_extrinsic", return_value=fake_response ) mocked_format_error_message = mocker.patch.object( - async_registration, "format_error_message" + async_subtensor, "format_error_message" ) # Call @@ -124,10 +124,10 @@ async def test_do_pow_register_failure(subtensor, mocker): call=fake_call, keypair=fake_wallet.hotkey ) subtensor.substrate.submit_extrinsic.asseert_awaited_once_with( - extrinsic=fake_extrinsic, wait_for_inclusion=True, wait_for_finalization=True + fake_extrinsic, wait_for_inclusion=True, wait_for_finalization=True ) - mocked_format_error_message.assert_called_once_with(error_message=fake_err_message) + mocked_format_error_message.assert_called_once_with(fake_err_message) assert result_error_message == (False, mocked_format_error_message.return_value) @@ -173,7 +173,7 @@ async def test_do_pow_register_no_waiting(subtensor, mocker): fake_extrinsic, wait_for_inclusion=False, wait_for_finalization=False ) assert result is True - assert error_message is None + assert error_message == "" @pytest.mark.asyncio @@ -214,8 +214,15 @@ async def test_register_extrinsic_success(subtensor, mocker): ) # Asserts - mocked_subnet_exists.assert_called_once_with(1) - mocked_get_neuron.assert_called_once_with(hotkey_ss58="hotkey_ss58", netuid=1) + mocked_subnet_exists.assert_called_once_with( + 1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) + mocked_get_neuron.assert_called_once_with( + hotkey_ss58="hotkey_ss58", + netuid=1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) mocked_create_pow.assert_called_once() mocked_do_pow_register.assert_called_once() mocked_is_hotkey_registered.assert_called_once_with( @@ -264,8 +271,15 @@ async def test_register_extrinsic_success_with_cuda(subtensor, mocker): ) # Asserts - mocked_subnet_exists.assert_called_once_with(1) - mocked_get_neuron.assert_called_once_with(hotkey_ss58="hotkey_ss58", netuid=1) + mocked_subnet_exists.assert_called_once_with( + 1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) + mocked_get_neuron.assert_called_once_with( + hotkey_ss58="hotkey_ss58", + netuid=1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) mocked_create_pow.assert_called_once() mocked_do_pow_register.assert_called_once() mocked_is_hotkey_registered.assert_called_once_with( @@ -303,8 +317,15 @@ async def test_register_extrinsic_failed_with_cuda(subtensor, mocker): ) # Asserts - mocked_subnet_exists.assert_called_once_with(1) - mocked_get_neuron.assert_called_once_with(hotkey_ss58="hotkey_ss58", netuid=1) + mocked_subnet_exists.assert_called_once_with( + 1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) + mocked_get_neuron.assert_called_once_with( + hotkey_ss58="hotkey_ss58", + netuid=1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) assert result is False @@ -326,7 +347,10 @@ async def test_register_extrinsic_subnet_not_exists(subtensor, mocker): ) # Asserts - mocked_subnet_exists.assert_called_once_with(1) + mocked_subnet_exists.assert_called_once_with( + 1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) assert result is False @@ -350,7 +374,9 @@ async def test_register_extrinsic_already_registered(subtensor, mocker): # Asserts mocked_get_neuron.assert_called_once_with( - hotkey_ss58=fake_wallet.hotkey.ss58_address, netuid=1 + hotkey_ss58=fake_wallet.hotkey.ss58_address, + netuid=1, + block_hash=subtensor.substrate.get_chain_head.return_value, ) assert result is True @@ -400,8 +426,15 @@ async def is_stale_side_effect(*_, **__): ) # Asserts - mocked_subnet_exists.assert_called_once_with(1) - mocked_get_neuron.assert_called_once_with(hotkey_ss58="hotkey_ss58", netuid=1) + mocked_subnet_exists.assert_called_once_with( + 1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) + mocked_get_neuron.assert_called_once_with( + hotkey_ss58="hotkey_ss58", + netuid=1, + block_hash=subtensor.substrate.get_chain_head.return_value, + ) assert mocked_create_pow.call_count == 3 assert mocked_do_pow_register.call_count == 3 diff --git a/tests/unit_tests/extrinsics/asyncex/test_root.py b/tests/unit_tests/extrinsics/asyncex/test_root.py index bc258f1da9..c1aed1d6a4 100644 --- a/tests/unit_tests/extrinsics/asyncex/test_root.py +++ b/tests/unit_tests/extrinsics/asyncex/test_root.py @@ -81,7 +81,6 @@ async def test_root_register_extrinsic_success(subtensor, mocker): result = await async_root.root_register_extrinsic( subtensor=subtensor, wallet=fake_wallet, - netuid=1, wait_for_inclusion=True, wait_for_finalization=True, ) @@ -89,14 +88,14 @@ async def test_root_register_extrinsic_success(subtensor, mocker): # Asserts mocked_unlock_key.assert_called_once_with(fake_wallet) mocked_is_hotkey_registered.assert_called_once_with( - netuid=1, hotkey_ss58="fake_hotkey_address" + netuid=0, hotkey_ss58="fake_hotkey_address" ) mocked_compose_call.assert_called_once() mocked_sign_and_send_extrinsic.assert_called_once() mocked_query.assert_called_once_with( module="SubtensorModule", storage_function="Uids", - params=[1, "fake_hotkey_address"], + params=[0, "fake_hotkey_address"], ) assert result is True @@ -117,7 +116,6 @@ async def test_root_register_extrinsic_unlock_failed(subtensor, mocker): result = await async_root.root_register_extrinsic( subtensor=subtensor, wallet=fake_wallet, - netuid=1, wait_for_inclusion=True, wait_for_finalization=True, ) @@ -149,7 +147,6 @@ async def test_root_register_extrinsic_already_registered(subtensor, mocker): result = await async_root.root_register_extrinsic( subtensor=subtensor, wallet=fake_wallet, - netuid=1, wait_for_inclusion=True, wait_for_finalization=True, ) @@ -157,7 +154,7 @@ async def test_root_register_extrinsic_already_registered(subtensor, mocker): # Asserts mocked_unlock_key.assert_called_once_with(fake_wallet) mocked_is_hotkey_registered.assert_called_once_with( - netuid=1, hotkey_ss58="fake_hotkey_address" + netuid=0, hotkey_ss58="fake_hotkey_address" ) assert result is True @@ -190,7 +187,6 @@ async def test_root_register_extrinsic_transaction_failed(subtensor, mocker): result = await async_root.root_register_extrinsic( subtensor=subtensor, wallet=fake_wallet, - netuid=1, wait_for_inclusion=True, wait_for_finalization=True, ) @@ -198,7 +194,7 @@ async def test_root_register_extrinsic_transaction_failed(subtensor, mocker): # Asserts mocked_unlock_key.assert_called_once_with(fake_wallet) mocked_is_hotkey_registered.assert_called_once_with( - netuid=1, hotkey_ss58="fake_hotkey_address" + netuid=0, hotkey_ss58="fake_hotkey_address" ) mocked_compose_call.assert_called_once() mocked_sign_and_send_extrinsic.assert_called_once() @@ -238,7 +234,6 @@ async def test_root_register_extrinsic_uid_not_found(subtensor, mocker): result = await async_root.root_register_extrinsic( subtensor=subtensor, wallet=fake_wallet, - netuid=1, wait_for_inclusion=True, wait_for_finalization=True, ) @@ -246,14 +241,14 @@ async def test_root_register_extrinsic_uid_not_found(subtensor, mocker): # Asserts mocked_unlock_key.assert_called_once_with(fake_wallet) mocked_is_hotkey_registered.assert_called_once_with( - netuid=1, hotkey_ss58="fake_hotkey_address" + netuid=0, hotkey_ss58="fake_hotkey_address" ) mocked_compose_call.assert_called_once() mocked_sign_and_send_extrinsic.assert_called_once() mocked_query.assert_called_once_with( module="SubtensorModule", storage_function="Uids", - params=[1, "fake_hotkey_address"], + params=[0, "fake_hotkey_address"], ) assert result is False diff --git a/tests/unit_tests/extrinsics/asyncex/test_transfer.py b/tests/unit_tests/extrinsics/asyncex/test_transfer.py index df0e788734..0d15d7b577 100644 --- a/tests/unit_tests/extrinsics/asyncex/test_transfer.py +++ b/tests/unit_tests/extrinsics/asyncex/test_transfer.py @@ -220,7 +220,7 @@ async def test_transfer_extrinsic_success(subtensor, mocker): result = await async_transfer.transfer_extrinsic( subtensor=subtensor, wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=fake_amount, transfer_all=False, wait_for_inclusion=True, @@ -285,7 +285,7 @@ async def test_transfer_extrinsic_call_successful_with_failed_response( result = await async_transfer.transfer_extrinsic( subtensor=subtensor, wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=fake_amount, transfer_all=False, wait_for_inclusion=True, @@ -346,7 +346,7 @@ async def test_transfer_extrinsic_insufficient_balance(subtensor, mocker): result = await async_transfer.transfer_extrinsic( subtensor=subtensor, wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=fake_amount, transfer_all=False, wait_for_inclusion=True, @@ -384,7 +384,7 @@ async def test_transfer_extrinsic_invalid_destination(subtensor, mocker): result = await async_transfer.transfer_extrinsic( subtensor=subtensor, wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=fake_amount, transfer_all=False, wait_for_inclusion=True, @@ -422,7 +422,7 @@ async def test_transfer_extrinsic_unlock_key_false(subtensor, mocker): result = await async_transfer.transfer_extrinsic( subtensor=subtensor, wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=fake_amount, transfer_all=False, wait_for_inclusion=True, @@ -479,7 +479,7 @@ async def test_transfer_extrinsic_keep_alive_false_and_transfer_all_true( result = await async_transfer.transfer_extrinsic( subtensor=subtensor, wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=fake_amount, transfer_all=True, wait_for_inclusion=True, diff --git a/tests/unit_tests/extrinsics/asyncex/test_weights.py b/tests/unit_tests/extrinsics/asyncex/test_weights.py index 531ab802f7..3233519c16 100644 --- a/tests/unit_tests/extrinsics/asyncex/test_weights.py +++ b/tests/unit_tests/extrinsics/asyncex/test_weights.py @@ -56,7 +56,7 @@ async def fake_is_success(): # Asserts assert result is True - assert message == "Successfully set weights." + assert message is None @pytest.mark.asyncio @@ -79,7 +79,7 @@ async def fake_is_success(): fake_response.process_events = mocker.AsyncMock() - fake_response.error_message = mocker.Mock() + fake_response.error_message = mocker.AsyncMock(return_value="Error occurred")() fake_response.process_events = mocker.AsyncMock() mocked_format_error_message = mocker.Mock() @@ -108,7 +108,7 @@ async def fake_is_success(): # Asserts assert result is False - mocked_format_error_message.assert_called_once_with(fake_response.error_message) + mocked_format_error_message.assert_called_once_with("Error occurred") assert message == mocked_format_error_message.return_value @@ -146,7 +146,7 @@ async def test_do_set_weights_no_waiting(subtensor, mocker): # Asserts assert result is True - assert message == "Not waiting for finalization or inclusion." + assert message is None @pytest.mark.asyncio @@ -336,7 +336,7 @@ async def fake_is_success(): fake_response = mocker.Mock() fake_response.is_success = fake_is_success() fake_response.process_events = mocker.AsyncMock() - fake_response.error_message = "Error occurred" + fake_response.error_message = mocker.AsyncMock(return_value="Error occurred")() mocked_format_error_message = mocker.Mock(return_value="Formatted error") mocker.patch.object( @@ -363,7 +363,7 @@ async def fake_is_success(): # Asserts assert result is False - mocked_format_error_message.assert_called_once_with(fake_response.error_message) + mocked_format_error_message.assert_called_once_with("Error occurred") assert message == "Formatted error" diff --git a/tests/unit_tests/test_async_subtensor.py b/tests/unit_tests/test_async_subtensor.py index 99ca1c126f..950d2c7cb6 100644 --- a/tests/unit_tests/test_async_subtensor.py +++ b/tests/unit_tests/test_async_subtensor.py @@ -1153,6 +1153,7 @@ async def test_get_neuron_for_pubkey_and_subnet_success(subtensor, mocker): subtensor.substrate.rpc_request.assert_called_once_with( method="neuronInfo_getNeuron", params=[fake_netuid, fake_uid.value], + block_hash=None, reuse_block_hash=False, ) mocked_neuron_info.assert_called_once_with(fake_result) @@ -1230,6 +1231,7 @@ async def test_get_neuron_for_pubkey_and_subnet_rpc_result_empty(subtensor, mock subtensor.substrate.rpc_request.assert_called_once_with( method="neuronInfo_getNeuron", params=[fake_netuid, fake_uid], + block_hash=None, reuse_block_hash=False, ) mocked_get_null_neuron.assert_called_once() @@ -2504,7 +2506,7 @@ async def test_transfer_success(subtensor, mocker): mocked_transfer_extrinsic.assert_awaited_once_with( subtensor=subtensor, wallet=fake_wallet, - destination=fake_destination, + dest=fake_destination, amount=mocked_balance_from_tao, transfer_all=fake_transfer_all, wait_for_inclusion=True, diff --git a/tests/unit_tests/test_subtensor.py b/tests/unit_tests/test_subtensor.py index b43bd71b85..e720aa52d7 100644 --- a/tests/unit_tests/test_subtensor.py +++ b/tests/unit_tests/test_subtensor.py @@ -12,7 +12,7 @@ def test_methods_comparable(mocker): async_subtensor = AsyncSubtensor(_mock=True) # methods which lives in async subtensor only - excluded_async_subtensor_methods = ["sign_and_send_extrinsic", "initialize"] + excluded_async_subtensor_methods = ["initialize"] subtensor_methods = [m for m in dir(subtensor) if not m.startswith("_")] async_subtensor_methods = [ From 0e72ec08be0fa66f8f7a6b725e8ba2ca52cbe3a8 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 23 Jan 2025 16:58:35 +0200 Subject: [PATCH 26/52] Metagraph/tests --- bittensor/core/metagraph.py | 24 +++++++-------------- bittensor/core/subtensor.py | 2 +- tests/e2e_tests/utils/chain_interactions.py | 4 ++-- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/bittensor/core/metagraph.py b/bittensor/core/metagraph.py index ce696c536e..c761c64dbd 100644 --- a/bittensor/core/metagraph.py +++ b/bittensor/core/metagraph.py @@ -514,6 +514,14 @@ def __repr__(self) -> str: """ return self.__str__() + async def __aenter__(self): + if self.should_sync: + await self.sync(block=None, lite=self.lite, subtensor=self.subtensor) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + pass + def metadata(self) -> dict: """ Retrieves the metadata of the metagraph, providing key information about the current state of the Bittensor @@ -1133,14 +1141,6 @@ def __init__( self.subtensor = subtensor self.should_sync = sync - async def __aenter__(self): - if self.should_sync: - await self.sync(block=None, lite=self.lite, subtensor=self.subtensor) - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - pass - async def _set_metagraph_attributes(self, block: int, subtensor: "AsyncSubtensor"): """ Sets various attributes of the metagraph based on the latest network data fetched from the subtensor. @@ -1334,14 +1334,6 @@ def __init__( self.subtensor = subtensor self.should_sync = sync - async def __aenter__(self): - if self.should_sync: - await self.sync(block=None, lite=self.lite, subtensor=self.subtensor) - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - pass - async def _set_metagraph_attributes(self, block: int, subtensor: "AsyncSubtensor"): """ Sets various attributes of the metagraph based on the latest network data fetched from the subtensor. This diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index f1ae724eba..355b6fe9bc 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -2312,7 +2312,7 @@ def root_register( recycle_call = self.get_hyperparameter( param_name="Burn", netuid=0, block=block ) - balance = (self.get_balance(wallet.coldkeypub.ss58_address, block=block),) + balance = self.get_balance(wallet.coldkeypub.ss58_address, block=block) except TypeError as e: logging.error(f"Unable to retrieve current recycle. {e}") return False diff --git a/tests/e2e_tests/utils/chain_interactions.py b/tests/e2e_tests/utils/chain_interactions.py index cf28240139..af0c85b36e 100644 --- a/tests/e2e_tests/utils/chain_interactions.py +++ b/tests/e2e_tests/utils/chain_interactions.py @@ -153,7 +153,7 @@ async def wait_interval( and the provided tempo, then enters a loop where it periodically checks the current block number until the next tempo interval starts. """ - current_block = await subtensor.async_subtensor.get_current_block() + current_block = subtensor.get_current_block() next_tempo_block_start = next_tempo(current_block, tempo, netuid) last_reported = None @@ -161,7 +161,7 @@ async def wait_interval( await asyncio.sleep( 1 ) # Wait for 1 second before checking the block number again - current_block = await subtensor.async_subtensor.get_current_block() + current_block = subtensor.get_current_block() if last_reported is None or current_block - last_reported >= reporting_interval: last_reported = current_block print( From 3b498bcae2694308596bf61780dde7bb010c48d6 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 23 Jan 2025 17:10:13 +0200 Subject: [PATCH 27/52] Call function --- bittensor/core/extrinsics/serving.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/core/extrinsics/serving.py b/bittensor/core/extrinsics/serving.py index df73b21acc..b5c47637bf 100644 --- a/bittensor/core/extrinsics/serving.py +++ b/bittensor/core/extrinsics/serving.py @@ -218,7 +218,7 @@ def serve_axon_extrinsic( # ---- Get external ip ---- if axon.external_ip is None: try: - external_ip = net.get_external_ip + external_ip = net.get_external_ip() logging.success( f":white_heavy_check_mark: [green]Found external ip:[/green] [blue]{external_ip}[/blue]" ) From f3558ecca4a01962207bf1c307e94feb4e578470 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 23 Jan 2025 17:11:52 +0200 Subject: [PATCH 28/52] Raise correct error --- bittensor/core/extrinsics/asyncex/serving.py | 2 +- bittensor/core/extrinsics/serving.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/core/extrinsics/asyncex/serving.py b/bittensor/core/extrinsics/asyncex/serving.py index 189fcee6aa..70d10d4e78 100644 --- a/bittensor/core/extrinsics/asyncex/serving.py +++ b/bittensor/core/extrinsics/asyncex/serving.py @@ -227,7 +227,7 @@ async def serve_axon_extrinsic( f":white_heavy_check_mark: [green]Found external ip:[/green] [blue]{external_ip}[/blue]" ) except Exception as e: - raise RuntimeError( + raise ConnectionError( f"Unable to attain your external ip. Check your internet connection. error: {e}" ) from e else: diff --git a/bittensor/core/extrinsics/serving.py b/bittensor/core/extrinsics/serving.py index b5c47637bf..29af1e40e4 100644 --- a/bittensor/core/extrinsics/serving.py +++ b/bittensor/core/extrinsics/serving.py @@ -223,7 +223,7 @@ def serve_axon_extrinsic( f":white_heavy_check_mark: [green]Found external ip:[/green] [blue]{external_ip}[/blue]" ) except Exception as e: - raise RuntimeError( + raise ConnectionError( f"Unable to attain your external ip. Check your internet connection. error: {e}" ) from e else: From 7ec2df34cf6b429fa186202c8c7948523f062da3 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 23 Jan 2025 18:41:27 +0200 Subject: [PATCH 29/52] Metagraph rewrite --- bittensor/core/metagraph.py | 1055 ++++++++++------- .../extrinsics/asyncex/test_commit_reveal.py | 4 +- 2 files changed, 605 insertions(+), 454 deletions(-) diff --git a/bittensor/core/metagraph.py b/bittensor/core/metagraph.py index c761c64dbd..953dd833a5 100644 --- a/bittensor/core/metagraph.py +++ b/bittensor/core/metagraph.py @@ -1,5 +1,6 @@ import asyncio import copy +import importlib import os import pickle import typing @@ -142,7 +143,7 @@ def determine_chain_endpoint_and_network(network: str) -> tuple[str, str]: return "unknown", network -class AsyncMetagraphMixin(ABC): +class MetagraphMixin(ABC): """ The metagraph class is a core component of the Bittensor network, representing the neural graph that forms the backbone of the decentralized machine learning system. @@ -232,6 +233,7 @@ class AsyncMetagraphMixin(ABC): axons: list[AxonInfo] chain_endpoint: Optional[str] subtensor: Optional["AsyncSubtensor"] + _dtype_registry = {"int64": np.int64, "float32": np.float32, "bool": bool} @property def S(self) -> Union[NDArray, "torch.nn.Parameter"]: @@ -454,7 +456,7 @@ def __init__( network: str = settings.DEFAULT_NETWORK, lite: bool = True, sync: bool = True, - subtensor: "AsyncSubtensor" = None, + subtensor: Optional[Union["AsyncSubtensor", "Subtensor"]] = None, ): """ Initializes a new instance of the metagraph object, setting up the basic structure and parameters based on the @@ -514,14 +516,6 @@ def __repr__(self) -> str: """ return self.__str__() - async def __aenter__(self): - if self.should_sync: - await self.sync(block=None, lite=self.lite, subtensor=self.subtensor) - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - pass - def metadata(self) -> dict: """ Retrieves the metadata of the metagraph, providing key information about the current state of the Bittensor @@ -575,150 +569,6 @@ def state_dict(self): "neurons": self.neurons, } - async def sync( - self, - block: Optional[int] = None, - lite: bool = True, - subtensor: Optional["AsyncSubtensor"] = None, - ): - """ - Synchronizes the metagraph with the Bittensor network's current state. It updates the metagraph's attributes to - reflect the latest data from the network, ensuring the metagraph represents the most current state of the - network. - - Args: - block (Optional[int]): A specific block number to synchronize with. If None, the metagraph syncs with the - latest block. This allows for historical analysis or specific state examination of the network. - lite (bool): If True, a lite version of the metagraph is used for quicker synchronization. This is - beneficial when full detail is not necessary, allowing for reduced computational and time overhead. - subtensor (Optional[bittensor.core.subtensor.Subtensor]): An instance of the subtensor class from Bittensor, - providing an interface to the underlying blockchain data. If provided, this instance is used for data - retrieval during synchronization. - - Example: - Sync the metagraph with the latest block from the subtensor, using the lite version for efficiency:: - - from bittensor.core.subtensor import Subtensor - - subtensor = Subtensor() - metagraph.sync(subtensor=subtensor) - - Sync with a specific block number for detailed analysis:: - - from bittensor.core.subtensor import Subtensor - - subtensor = Subtensor() - metagraph.sync(block=12345, lite=False, subtensor=subtensor) - - NOTE: - If attempting to access data beyond the previous 300 blocks, you **must** use the ``archive`` network for - subtensor. Light nodes are configured only to store the previous 300 blocks if connecting to finney or - test networks. - - For example:: - - from bittensor.core.subtensor import Subtensor - - subtensor = Subtensor(network='archive') - current_block = subtensor.get_current_block() - history_block = current_block - 1200 - - metagraph.sync(block=history_block, lite=False, subtensor=subtensor) - """ - # Initialize subtensor - subtensor = self._initialize_subtensor(subtensor) - - if ( - subtensor.chain_endpoint != settings.ARCHIVE_ENTRYPOINT - or subtensor.network != "archive" - ): - cur_block = await subtensor.get_current_block() - if block and block < (cur_block - 300): - logging.warning( - "Attempting to sync longer than 300 blocks ago on a non-archive node. Please use the 'archive' " - "network for subtensor and retry." - ) - - # Assign neurons based on 'lite' flag - await self._assign_neurons(block, lite, subtensor) - - # Set attributes for metagraph - await self._set_metagraph_attributes(block, subtensor) - - # If not a 'lite' version, compute and set weights and bonds for each neuron - if not lite: - await self._set_weights_and_bonds(subtensor=subtensor) - - def _initialize_subtensor(self, subtensor: "AsyncSubtensor") -> "AsyncSubtensor": - """ - Initializes the subtensor to be used for syncing the metagraph. - - This method ensures that a subtensor instance is available and properly set up for data retrieval during the - synchronization process. - - If no subtensor is provided, this method is responsible for creating a new instance of the subtensor, configured - according to the current network settings. - - Args: - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance provided for - initialization. If ``None``, a new subtensor instance is created using the current network configuration. - - Returns: - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The initialized subtensor instance, ready to be - used for syncing the metagraph. - - Internal Usage: - Used internally during the sync process to ensure a valid subtensor instance is available:: - - subtensor = self._initialize_subtensor(subtensor) - """ - if subtensor and subtensor != self.subtensor: - self.subtensor = subtensor - if not subtensor and self.subtensor: - subtensor = self.subtensor - if not subtensor: - # TODO: Check and test the initialization of the new subtensor - # Lazy import due to circular import (subtensor -> metagraph, metagraph -> subtensor) - from bittensor.core.subtensor import AsyncSubtensor - - subtensor = AsyncSubtensor(network=self.chain_endpoint) - self.subtensor = subtensor - return subtensor - - async def _assign_neurons( - self, block: int, lite: bool, subtensor: "AsyncSubtensor" - ): - """ - Assigns neurons to the metagraph based on the provided block number and the lite flag. - - This method is responsible for fetching and setting the neuron data in the metagraph, which includes neuron - attributes like UID, stake, trust, and other relevant information. - - Args: - block (int): The block number for which the neuron data needs to be fetched. If ``None``, the latest block - data is used. - lite (bool): A boolean flag indicating whether to use a lite version of the neuron data. The lite version - typically includes essential information and is quicker to fetch and process. - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for fetching neuron - data from the network. - - Internal Usage: - Used internally during the sync process to fetch and set neuron data:: - - from bittensor.core.subtensor import Subtensor - - block = 12345 - lite = False - subtensor = Subtensor() - self._assign_neurons(block, lite, subtensor) - """ - if lite: - self.neurons = await subtensor.neurons_lite(block=block, netuid=self.netuid) - - else: - self.neurons = await subtensor.neurons(block=block, netuid=self.netuid) - self.lite = lite - @staticmethod def _create_tensor(data, dtype) -> Union[NDArray, "torch.nn.Parameter"]: """ @@ -744,38 +594,6 @@ def _create_tensor(data, dtype) -> Union[NDArray, "torch.nn.Parameter"]: else np.array(data, dtype=dtype) ) - async def _set_weights_and_bonds( - self, subtensor: Optional["AsyncSubtensor"] = None - ): - """ - Computes and sets the weights and bonds for each neuron in the metagraph. This method is responsible for - processing the raw weight and bond data obtained from the network and converting it into a structured format - suitable for the metagraph model. - - Args: - subtensor: The subtensor instance used for fetching weights and bonds data. If ``None``, the weights and - bonds are not updated. - - Internal Usage: - Used internally during the sync process to update the weights and bonds of the neurons:: - - self._set_weights_and_bonds(subtensor=subtensor) - """ - # TODO: Check and test the computation of weights and bonds - if self.netuid == 0: - self.weights = await self._process_root_weights( - [neuron.weights for neuron in self.neurons], - "weights", - subtensor, - ) - else: - self.weights = self._process_weights_or_bonds( - [neuron.weights for neuron in self.neurons], "weights" - ) - self.bonds = self._process_weights_or_bonds( - [neuron.bonds for neuron in self.neurons], "bonds" - ) - def _process_weights_or_bonds( self, data, attribute: str ) -> Union[NDArray, "torch.nn.Parameter"]: @@ -840,95 +658,106 @@ def _process_weights_or_bonds( ) return tensor_param - @abstractmethod - async def _set_metagraph_attributes(self, block, subtensor): - pass + def _set_metagraph_attributes(self, block: int): + """ + Sets various attributes of the metagraph based on the latest network data fetched from the subtensor. This + method updates parameters like the number of neurons, block number, stakes, trusts, ranks, and other + neuron-specific information. - async def _process_root_weights( - self, data: list, attribute: str, subtensor: "AsyncSubtensor" - ) -> Union[NDArray, "torch.nn.Parameter"]: + Args: + block (int): The block number for which the metagraph attributes need to be set. + + Internal Usage: + Used internally during the sync process to update the metagraph's attributes:: + + self._set_metagraph_attributes(block) """ - Specifically processes the root weights data for the metagraph. This method is similar to :func:`_process_weights_or_bonds` - but is tailored for processing root weights, which have a different structure and significance in the network. + self.n = self._create_tensor( + len(self.neurons), dtype=self._dtype_registry["int64"] + ) + self.version = self._create_tensor( + [settings.version_as_int], dtype=self._dtype_registry["int64"] + ) + self.block = self._create_tensor(block, dtype=self._dtype_registry["int64"]) + self.uids = self._create_tensor( + [neuron.uid for neuron in self.neurons], dtype=self._dtype_registry["int64"] + ) + self.trust = self._create_tensor( + [neuron.trust for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.consensus = self._create_tensor( + [neuron.consensus for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.incentive = self._create_tensor( + [neuron.incentive for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.dividends = self._create_tensor( + [neuron.dividends for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.ranks = self._create_tensor( + [neuron.rank for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.emission = self._create_tensor( + [neuron.emission for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.active = self._create_tensor( + [neuron.active for neuron in self.neurons], + dtype=self._dtype_registry["int64"], + ) + self.last_update = self._create_tensor( + [neuron.last_update for neuron in self.neurons], + dtype=self._dtype_registry["int64"], + ) + self.validator_permit = self._create_tensor( + [neuron.validator_permit for neuron in self.neurons], dtype=bool + ) + self.validator_trust = self._create_tensor( + [neuron.validator_trust for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.total_stake = self._create_tensor( + [neuron.total_stake.tao for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.stake = self._create_tensor( + [neuron.stake for neuron in self.neurons], + dtype=self._dtype_registry["float32"], + ) + self.axons = [n.axon_info for n in self.neurons] + + def save(self, root_dir: Optional[list[str]] = None) -> "AsyncMetagraph": + """ + Saves the current state of the metagraph to a file on disk. This function is crucial for persisting the current + state of the network's metagraph, which can later be reloaded or analyzed. The save operation includes all + neuron attributes and parameters, ensuring a complete snapshot of the metagraph's state. Args: - data (list): The raw root weights data to be processed. - attribute (str): A string indicating the attribute type, here it's typically ``weights``. - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for additional data - and context needed in processing. + root_dir: list to the file path for the root directory of your metagraph saves + (i.e. ['/', 'tmp', 'metagraphs'], defaults to ["~", ".bittensor", "metagraphs"] Returns: - A tensor parameter encapsulating the processed root weights data. + metagraph (bittensor.core.metagraph.Metagraph): The metagraph instance after saving its state. - Internal Usage: - Used internally to process and set root weights for the metagraph:: + Example: + Save the current state of the metagraph to the default directory:: - self.root_weights = self._process_root_weights(raw_root_weights_data, "weights", subtensor) - """ - data_array = [] - n_subnets = await subtensor.get_total_subnets() or 0 - subnets = await subtensor.get_subnets() - for item in data: - if len(item) == 0: - if use_torch(): - data_array.append(torch.zeros(n_subnets)) - else: - data_array.append(np.zeros(n_subnets, dtype=np.float32)) - else: - uids, values = zip(*item) - # TODO: Validate and test the conversion of uids and values to tensor - data_array.append( - convert_root_weight_uids_and_vals_to_tensor( - n_subnets, list(uids), list(values), subnets - ) - ) - - tensor_param: Union[NDArray, "torch.nn.Parameter"] = ( - ( - torch.nn.Parameter(torch.stack(data_array), requires_grad=False) - if len(data_array) - else torch.nn.Parameter() - ) - if use_torch() - else ( - np.stack(data_array) - if len(data_array) - else np.array([], dtype=np.float32) - ) - ) - if len(data_array) == 0: - logging.warning( - f"Empty {attribute}_array on metagraph.sync(). The '{attribute}' tensor is empty." - ) - return tensor_param - - def save(self, root_dir: Optional[list[str]] = None) -> "AsyncMetagraph": - """ - Saves the current state of the metagraph to a file on disk. This function is crucial for persisting the current - state of the network's metagraph, which can later be reloaded or analyzed. The save operation includes all - neuron attributes and parameters, ensuring a complete snapshot of the metagraph's state. - - Args: - root_dir: list to the file path for the root directory of your metagraph saves - (i.e. ['/', 'tmp', 'metagraphs'], defaults to ["~", ".bittensor", "metagraphs"] - - Returns: - metagraph (bittensor.core.metagraph.Metagraph): The metagraph instance after saving its state. - - Example: - Save the current state of the metagraph to the default directory:: - - metagraph.save() - - The saved state can later be loaded to restore or analyze the metagraph's state at this point. - - If using the default save path:: - - metagraph.load() - - If using a custom save path:: - - metagraph.load_from_path(dir_path) + metagraph.save() + + The saved state can later be loaded to restore or analyze the metagraph's state at this point. + + If using the default save path:: + + metagraph.load() + + If using a custom save path:: + + metagraph.load_from_path(dir_path) """ save_directory = get_save_dir(self.network, self.netuid, root_dir=root_dir) os.makedirs(save_directory, exist_ok=True) @@ -1044,14 +873,14 @@ def __copy__(self): """ -class AsyncTorchMetaGraph(AsyncMetagraphMixin, BaseClass): +class TorchMetagraph(MetagraphMixin, BaseClass): def __init__( self, netuid: int, network: str = settings.DEFAULT_NETWORK, lite: bool = True, sync: bool = True, - subtensor: "AsyncSubtensor" = None, + subtensor: Optional[Union["AsyncSubtensor", "Subtensor"]] = None, ): """ Initializes a new instance of the metagraph object, setting up the basic structure and parameters based on the @@ -1076,11 +905,16 @@ def __init__( metagraph = Metagraph(netuid=123, network="finney", lite=True, sync=True) """ torch.nn.Module.__init__(self) - AsyncMetagraphMixin.__init__(self, netuid, network, lite, sync, subtensor) + MetagraphMixin.__init__(self, netuid, network, lite, sync, subtensor) self.netuid = netuid self.network, self.chain_endpoint = determine_chain_endpoint_and_network( network ) + self._dtype_registry = { + "int64": torch.int64, + "float32": torch.float32, + "bool": torch.bool, + } self.version = torch.nn.Parameter( torch.tensor([settings.version_as_int], dtype=torch.int64), requires_grad=False, @@ -1141,74 +975,6 @@ def __init__( self.subtensor = subtensor self.should_sync = sync - async def _set_metagraph_attributes(self, block: int, subtensor: "AsyncSubtensor"): - """ - Sets various attributes of the metagraph based on the latest network data fetched from the subtensor. - This method updates parameters like the number of neurons, block number, stakes, trusts, ranks, and other - neuron-specific information. - - Args: - block (int): The block number for which the metagraph attributes need to be set. If ``None``, the latest - block data is used. - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for fetching the - latest network data. - - Internal Usage: - Used internally during the sync process to update the metagraph's attributes:: - - from bittensor.core.subtensor import Subtensor - - subtensor = Subtensor() - block = subtensor.get_current_block() - - self._set_metagraph_attributes(block, subtensor) - """ - self.n = self._create_tensor(len(self.neurons), dtype=torch.int64) - self.version = self._create_tensor([settings.version_as_int], dtype=torch.int64) - self.block = self._create_tensor( - block if block else await subtensor.block, dtype=torch.int64 - ) - self.uids = self._create_tensor( - [neuron.uid for neuron in self.neurons], dtype=torch.int64 - ) - self.trust = self._create_tensor( - [neuron.trust for neuron in self.neurons], dtype=torch.float32 - ) - self.consensus = self._create_tensor( - [neuron.consensus for neuron in self.neurons], dtype=torch.float32 - ) - self.incentive = self._create_tensor( - [neuron.incentive for neuron in self.neurons], dtype=torch.float32 - ) - self.dividends = self._create_tensor( - [neuron.dividends for neuron in self.neurons], dtype=torch.float32 - ) - self.ranks = self._create_tensor( - [neuron.rank for neuron in self.neurons], dtype=torch.float32 - ) - self.emission = self._create_tensor( - [neuron.emission for neuron in self.neurons], dtype=torch.float32 - ) - self.active = self._create_tensor( - [neuron.active for neuron in self.neurons], dtype=torch.int64 - ) - self.last_update = self._create_tensor( - [neuron.last_update for neuron in self.neurons], dtype=torch.int64 - ) - self.validator_permit = self._create_tensor( - [neuron.validator_permit for neuron in self.neurons], dtype=torch.bool - ) - self.validator_trust = self._create_tensor( - [neuron.validator_trust for neuron in self.neurons], dtype=torch.float32 - ) - self.total_stake = self._create_tensor( - [neuron.total_stake.tao for neuron in self.neurons], dtype=torch.float32 - ) - self.stake = self._create_tensor( - [neuron.stake for neuron in self.neurons], dtype=torch.float32 - ) - self.axons = [n.axon_info for n in self.neurons] - def load_from_path(self, dir_path: str) -> "AsyncMetagraph": """ Loads the metagraph state from a specified directory path. @@ -1273,14 +1039,14 @@ def load_from_path(self, dir_path: str) -> "AsyncMetagraph": return self -class AsyncNonTorchMetagraph(AsyncMetagraphMixin): +class NonTorchMetagraph(MetagraphMixin): def __init__( self, netuid: int, network: str = settings.DEFAULT_NETWORK, lite: bool = True, sync: bool = True, - subtensor: "AsyncSubtensor" = None, + subtensor: Optional[Union["AsyncSubtensor", "Subtensor"]] = None, ): """ Initializes a new instance of the metagraph object, setting up the basic structure and parameters based on the @@ -1334,70 +1100,6 @@ def __init__( self.subtensor = subtensor self.should_sync = sync - async def _set_metagraph_attributes(self, block: int, subtensor: "AsyncSubtensor"): - """ - Sets various attributes of the metagraph based on the latest network data fetched from the subtensor. This - method updates parameters like the number of neurons, block number, stakes, trusts, ranks, and other - neuron-specific information. - - Args: - block (int): The block number for which the metagraph attributes need to be set. If ``None``, the latest - block data is used. - subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for fetching the - latest network data. - - Internal Usage: - Used internally during the sync process to update the metagraph's attributes:: - - self._set_metagraph_attributes(block, subtensor) - """ - # TODO: Check and test the setting of each attribute - self.n = self._create_tensor(len(self.neurons), dtype=np.int64) - self.version = self._create_tensor([settings.version_as_int], dtype=np.int64) - self.block = self._create_tensor( - block if block else await subtensor.block, dtype=np.int64 - ) - self.uids = self._create_tensor( - [neuron.uid for neuron in self.neurons], dtype=np.int64 - ) - self.trust = self._create_tensor( - [neuron.trust for neuron in self.neurons], dtype=np.float32 - ) - self.consensus = self._create_tensor( - [neuron.consensus for neuron in self.neurons], dtype=np.float32 - ) - self.incentive = self._create_tensor( - [neuron.incentive for neuron in self.neurons], dtype=np.float32 - ) - self.dividends = self._create_tensor( - [neuron.dividends for neuron in self.neurons], dtype=np.float32 - ) - self.ranks = self._create_tensor( - [neuron.rank for neuron in self.neurons], dtype=np.float32 - ) - self.emission = self._create_tensor( - [neuron.emission for neuron in self.neurons], dtype=np.float32 - ) - self.active = self._create_tensor( - [neuron.active for neuron in self.neurons], dtype=np.int64 - ) - self.last_update = self._create_tensor( - [neuron.last_update for neuron in self.neurons], dtype=np.int64 - ) - self.validator_permit = self._create_tensor( - [neuron.validator_permit for neuron in self.neurons], dtype=bool - ) - self.validator_trust = self._create_tensor( - [neuron.validator_trust for neuron in self.neurons], dtype=np.float32 - ) - self.total_stake = self._create_tensor( - [neuron.total_stake.tao for neuron in self.neurons], dtype=np.float32 - ) - self.stake = self._create_tensor( - [neuron.stake for neuron in self.neurons], dtype=np.float32 - ) - self.axons = [n.axon_info for n in self.neurons] - def load_from_path(self, dir_path: str) -> "AsyncMetagraph": """ Loads the state of the Metagraph from a specified directory path. @@ -1461,9 +1163,9 @@ def load_from_path(self, dir_path: str) -> "AsyncMetagraph": if use_torch(): - AsyncMetagraph = AsyncTorchMetaGraph + NumpyOrTorch = TorchMetagraph else: - AsyncMetagraph = AsyncNonTorchMetagraph + NumpyOrTorch = NonTorchMetagraph """Metagraph class that uses :class:`TorchMetaGraph` if PyTorch is available; otherwise, it falls back to :class:`NonTorchMetagraph`. - **With PyTorch**: When `use_torch()` returns `True`, `Metagraph` is set to :class:`TorchMetaGraph`, which utilizes PyTorch functionalities. @@ -1471,18 +1173,9 @@ def load_from_path(self, dir_path: str) -> "AsyncMetagraph": """ -class Metagraph(AsyncMetagraph): +class AsyncMetagraph(NumpyOrTorch): """ - Represents a wrapper for the asynchronous metagraph functionality. - - This class provides a synchronous interface to interact with an asynchronous metagraph. It is initialized with - configuration related to the network and provides methods for synchronizing and accessing asynchronous metagraph - attributes. - - If you want to get the description of any method from the `bittensor.core.metagraph.Metagraph` class, then simply - get the corresponding method from the `bittensor.core.metagraph.AsyncMetagraph` class. - `AsyncMetagraph` is the class related with `AsyncTorchMetaGraph` or `AsyncNonTorchMetagraph` depending on the use - of the use of the env var `USE_TORCH` + TODO docstring. Advise user to use `async_metagraph` factory fn if they want to sync at init """ def __init__( @@ -1491,51 +1184,507 @@ def __init__( network: str = settings.DEFAULT_NETWORK, lite: bool = True, sync: bool = True, - subtensor: "Subtensor" = None, + subtensor: Optional["AsyncSubtensor"] = None, ): - self.subtensor: Optional["Subtensor"] = subtensor - self._async_metagraph = AsyncMetagraph( - netuid=netuid, - network=network, - lite=lite, - sync=sync, - subtensor=subtensor.async_subtensor if subtensor else None, - ) - if sync: - if self.subtensor: - self.subtensor.event_loop_mgr.run(self._async_metagraph.sync()) + NumpyOrTorch.__init__(self, netuid, network, lite, sync, subtensor) - def sync( + async def __aenter__(self): + if self.should_sync: + await self.sync(block=None, lite=self.lite, subtensor=self.subtensor) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + pass + + async def sync( self, block: Optional[int] = None, lite: bool = True, - subtensor: Optional["Subtensor"] = None, + subtensor: Optional["AsyncSubtensor"] = None, ): - """Synchronizes the metagraph to the specified block, lite, and subtensor instance if available.""" - if subtensor: - event_loop_mgr = subtensor.event_loop_mgr - elif self.subtensor: - event_loop_mgr = self.subtensor.event_loop_mgr - else: - event_loop_mgr = self.event_loop_mgr - event_loop_mgr.run( - self._async_metagraph.sync( - block=block, - lite=lite, - subtensor=subtensor.async_subtensor if subtensor else None, - ) - ) + """ + Synchronizes the metagraph with the Bittensor network's current state. It updates the metagraph's attributes to + reflect the latest data from the network, ensuring the metagraph represents the most current state of the + network. + + Args: + block (Optional[int]): A specific block number to synchronize with. If None, the metagraph syncs with the + latest block. This allows for historical analysis or specific state examination of the network. + lite (bool): If True, a lite version of the metagraph is used for quicker synchronization. This is + beneficial when full detail is not necessary, allowing for reduced computational and time overhead. + subtensor (Optional[bittensor.core.subtensor.Subtensor]): An instance of the subtensor class from Bittensor, + providing an interface to the underlying blockchain data. If provided, this instance is used for data + retrieval during synchronization. + + Example: + Sync the metagraph with the latest block from the subtensor, using the lite version for efficiency:: + + from bittensor.core.subtensor import Subtensor + + subtensor = Subtensor() + metagraph.sync(subtensor=subtensor) + + Sync with a specific block number for detailed analysis:: + + from bittensor.core.subtensor import Subtensor + + subtensor = Subtensor() + metagraph.sync(block=12345, lite=False, subtensor=subtensor) + + NOTE: + If attempting to access data beyond the previous 300 blocks, you **must** use the ``archive`` network for + subtensor. Light nodes are configured only to store the previous 300 blocks if connecting to finney or + test networks. + + For example:: + + from bittensor.core.subtensor import Subtensor + + subtensor = Subtensor(network='archive') + current_block = subtensor.get_current_block() + history_block = current_block - 1200 + + metagraph.sync(block=history_block, lite=False, subtensor=subtensor) + """ + subtensor = await self._initialize_subtensor(subtensor) + + if ( + subtensor.chain_endpoint != settings.ARCHIVE_ENTRYPOINT + or subtensor.network != "archive" + ): + cur_block = await subtensor.get_current_block() + if block and block < (cur_block - 300): + logging.warning( + "Attempting to sync longer than 300 blocks ago on a non-archive node. Please use the 'archive' " + "network for subtensor and retry." + ) + if block is None: + block = await subtensor.get_current_block() + + # Assign neurons based on 'lite' flag + await self._assign_neurons(block, lite, subtensor) + + # Set attributes for metagraph + self._set_metagraph_attributes(block) + + # If not a 'lite' version, compute and set weights and bonds for each neuron + if not lite: + await self._set_weights_and_bonds(subtensor=subtensor) + + async def _initialize_subtensor( + self, subtensor: "AsyncSubtensor" + ) -> "AsyncSubtensor": + """ + Initializes the subtensor to be used for syncing the metagraph. + + This method ensures that a subtensor instance is available and properly set up for data retrieval during the + synchronization process. + + If no subtensor is provided, this method is responsible for creating a new instance of the subtensor, configured + according to the current network settings. + + Args: + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance provided for + initialization. If ``None``, a new subtensor instance is created using the current network configuration. - def __getattr__(self, name): - attr = getattr(self._async_metagraph, name) - if callable(attr): - if asyncio.iscoroutinefunction(attr): + Returns: + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The initialized subtensor instance, ready to be + used for syncing the metagraph. - def wrapper(*args, **kwargs): - return self.event_loop_mgr.run(attr(*args, **kwargs)) + Internal Usage: + Used internally during the sync process to ensure a valid subtensor instance is available:: - return wrapper - return attr + subtensor = await self._initialize_subtensor(subtensor) + """ + if subtensor and subtensor != self.subtensor: + self.subtensor = subtensor + if not subtensor and self.subtensor: + subtensor = self.subtensor + if not subtensor: + # Lazy import due to circular import (subtensor -> metagraph, metagraph -> subtensor) + AsyncSubtensor = getattr( + importlib.import_module("bittensor.core.async_subtensor"), + "AsyncSubtensor", + ) + + async with AsyncSubtensor(network=self.chain_endpoint) as subtensor: + self.subtensor = subtensor + return subtensor + + async def _assign_neurons( + self, block: int, lite: bool, subtensor: "AsyncSubtensor" + ): + """ + Assigns neurons to the metagraph based on the provided block number and the lite flag. + + This method is responsible for fetching and setting the neuron data in the metagraph, which includes neuron + attributes like UID, stake, trust, and other relevant information. + + Args: + block (int): The block number for which the neuron data needs to be fetched. If ``None``, the latest block + data is used. + lite (bool): A boolean flag indicating whether to use a lite version of the neuron data. The lite version + typically includes essential information and is quicker to fetch and process. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for fetching neuron + data from the network. + + Internal Usage: + Used internally during the sync process to fetch and set neuron data:: + + from bittensor.core.subtensor import Subtensor + + block = 12345 + lite = False + subtensor = Subtensor() + self._assign_neurons(block, lite, subtensor) + """ + if lite: + self.neurons = await subtensor.neurons_lite(block=block, netuid=self.netuid) + + else: + self.neurons = await subtensor.neurons(block=block, netuid=self.netuid) + self.lite = lite + + async def _set_weights_and_bonds( + self, subtensor: Optional["AsyncSubtensor"] = None + ): + """ + Computes and sets the weights and bonds for each neuron in the metagraph. This method is responsible for + processing the raw weight and bond data obtained from the network and converting it into a structured format + suitable for the metagraph model. + + Args: + subtensor: The subtensor instance used for fetching weights and bonds data. If ``None``, the weights and + bonds are not updated. + + Internal Usage: + Used internally during the sync process to update the weights and bonds of the neurons:: + + self._set_weights_and_bonds(subtensor=subtensor) + """ + # TODO: Check and test the computation of weights and bonds + if self.netuid == 0: + self.weights = await self._process_root_weights( + [neuron.weights for neuron in self.neurons], + "weights", + subtensor, + ) + else: + self.weights = self._process_weights_or_bonds( + [neuron.weights for neuron in self.neurons], "weights" + ) + self.bonds = self._process_weights_or_bonds( + [neuron.bonds for neuron in self.neurons], "bonds" + ) + + async def _process_root_weights( + self, data: list, attribute: str, subtensor: "AsyncSubtensor" + ) -> Union[NDArray, "torch.nn.Parameter"]: + """ + Specifically processes the root weights data for the metagraph. This method is similar to :func:`_process_weights_or_bonds` + but is tailored for processing root weights, which have a different structure and significance in the network. + + Args: + data (list): The raw root weights data to be processed. + attribute (str): A string indicating the attribute type, here it's typically ``weights``. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for additional data + and context needed in processing. + + Returns: + A tensor parameter encapsulating the processed root weights data. + + Internal Usage: + Used internally to process and set root weights for the metagraph:: + + self.root_weights = self._process_root_weights(raw_root_weights_data, "weights", subtensor) + """ + data_array = [] + n_subnets = await subtensor.get_total_subnets() or 0 + subnets = await subtensor.get_subnets() + for item in data: + if len(item) == 0: + if use_torch(): + data_array.append(torch.zeros(n_subnets)) + else: + data_array.append(np.zeros(n_subnets, dtype=np.float32)) + else: + uids, values = zip(*item) + # TODO: Validate and test the conversion of uids and values to tensor + data_array.append( + convert_root_weight_uids_and_vals_to_tensor( + n_subnets, list(uids), list(values), subnets + ) + ) + + tensor_param: Union[NDArray, "torch.nn.Parameter"] = ( + ( + torch.nn.Parameter(torch.stack(data_array), requires_grad=False) + if len(data_array) + else torch.nn.Parameter() + ) + if use_torch() + else ( + np.stack(data_array) + if len(data_array) + else np.array([], dtype=np.float32) + ) + ) + if len(data_array) == 0: + logging.warning( + f"Empty {attribute}_array on metagraph.sync(). The '{attribute}' tensor is empty." + ) + return tensor_param + + +class Metagraph(NumpyOrTorch): + def __init__( + self, + netuid: int, + network: str = settings.DEFAULT_NETWORK, + lite: bool = True, + sync: bool = True, + subtensor: Optional["Subtensor"] = None, + ): + NumpyOrTorch.__init__(self, netuid, network, lite, sync, subtensor) + if sync: + self.sync() + + def sync( + self, + block: Optional[int] = None, + lite: bool = True, + subtensor: Optional["Subtensor"] = None, + ): + """ + Synchronizes the metagraph with the Bittensor network's current state. It updates the metagraph's attributes to + reflect the latest data from the network, ensuring the metagraph represents the most current state of the + network. + + Args: + block (Optional[int]): A specific block number to synchronize with. If None, the metagraph syncs with the + latest block. This allows for historical analysis or specific state examination of the network. + lite (bool): If True, a lite version of the metagraph is used for quicker synchronization. This is + beneficial when full detail is not necessary, allowing for reduced computational and time overhead. + subtensor (Optional[bittensor.core.subtensor.Subtensor]): An instance of the subtensor class from Bittensor, + providing an interface to the underlying blockchain data. If provided, this instance is used for data + retrieval during synchronization. + + Example: + Sync the metagraph with the latest block from the subtensor, using the lite version for efficiency:: + + from bittensor.core.subtensor import Subtensor + + subtensor = Subtensor() + metagraph.sync(subtensor=subtensor) + + Sync with a specific block number for detailed analysis:: + + from bittensor.core.subtensor import Subtensor + + subtensor = Subtensor() + metagraph.sync(block=12345, lite=False, subtensor=subtensor) + + NOTE: + If attempting to access data beyond the previous 300 blocks, you **must** use the ``archive`` network for + subtensor. Light nodes are configured only to store the previous 300 blocks if connecting to finney or + test networks. + + For example:: + + from bittensor.core.subtensor import Subtensor + + subtensor = Subtensor(network='archive') + current_block = subtensor.get_current_block() + history_block = current_block - 1200 + + metagraph.sync(block=history_block, lite=False, subtensor=subtensor) + """ + subtensor = self._initialize_subtensor(subtensor) + + if ( + subtensor.chain_endpoint != settings.ARCHIVE_ENTRYPOINT + or subtensor.network != "archive" + ): + cur_block = subtensor.get_current_block() + if block and block < (cur_block - 300): + logging.warning( + "Attempting to sync longer than 300 blocks ago on a non-archive node. Please use the 'archive' " + "network for subtensor and retry." + ) + + if block is None: + block = subtensor.get_current_block() + + # Assign neurons based on 'lite' flag + self._assign_neurons(block, lite, subtensor) + + # Set attributes for metagraph + self._set_metagraph_attributes(block) + + # If not a 'lite' version, compute and set weights and bonds for each neuron + if not lite: + self._set_weights_and_bonds(subtensor=subtensor) + + def _initialize_subtensor(self, subtensor: "Subtensor") -> "Subtensor": + """ + Initializes the subtensor to be used for syncing the metagraph. + + This method ensures that a subtensor instance is available and properly set up for data retrieval during the + synchronization process. + + If no subtensor is provided, this method is responsible for creating a new instance of the subtensor, configured + according to the current network settings. + + Args: + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance provided for + initialization. If ``None``, a new subtensor instance is created using the current network configuration. + + Returns: + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The initialized subtensor instance, ready to be + used for syncing the metagraph. + + Internal Usage: + Used internally during the sync process to ensure a valid subtensor instance is available:: + + subtensor = self._initialize_subtensor(subtensor) + """ + if subtensor and subtensor != self.subtensor: + self.subtensor = subtensor + if not subtensor and self.subtensor: + subtensor = self.subtensor + if not subtensor: + # Lazy import due to circular import (subtensor -> metagraph, metagraph -> subtensor) + Subtensor = getattr( + importlib.import_module("bittensor.core.subtensor"), "Subtensor" + ) + subtensor = Subtensor(network=self.chain_endpoint) + + self.subtensor = subtensor + return subtensor + + async def _assign_neurons( + self, block: int, lite: bool, subtensor: "AsyncSubtensor" + ): + """ + Assigns neurons to the metagraph based on the provided block number and the lite flag. + + This method is responsible for fetching and setting the neuron data in the metagraph, which includes neuron + attributes like UID, stake, trust, and other relevant information. + + Args: + block (int): The block number for which the neuron data needs to be fetched. If ``None``, the latest block + data is used. + lite (bool): A boolean flag indicating whether to use a lite version of the neuron data. The lite version + typically includes essential information and is quicker to fetch and process. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for fetching neuron + data from the network. + + Internal Usage: + Used internally during the sync process to fetch and set neuron data:: + + from bittensor.core.subtensor import Subtensor + + block = 12345 + lite = False + subtensor = Subtensor() + self._assign_neurons(block, lite, subtensor) + """ + if lite: + self.neurons = await subtensor.neurons_lite(block=block, netuid=self.netuid) + + else: + self.neurons = await subtensor.neurons(block=block, netuid=self.netuid) + self.lite = lite + + async def _set_weights_and_bonds( + self, subtensor: Optional["AsyncSubtensor"] = None + ): + """ + Computes and sets the weights and bonds for each neuron in the metagraph. This method is responsible for + processing the raw weight and bond data obtained from the network and converting it into a structured format + suitable for the metagraph model. + + Args: + subtensor: The subtensor instance used for fetching weights and bonds data. If ``None``, the weights and + bonds are not updated. + + Internal Usage: + Used internally during the sync process to update the weights and bonds of the neurons:: + + self._set_weights_and_bonds(subtensor=subtensor) + """ + # TODO: Check and test the computation of weights and bonds + if self.netuid == 0: + self.weights = await self._process_root_weights( + [neuron.weights for neuron in self.neurons], + "weights", + subtensor, + ) + else: + self.weights = self._process_weights_or_bonds( + [neuron.weights for neuron in self.neurons], "weights" + ) + self.bonds = self._process_weights_or_bonds( + [neuron.bonds for neuron in self.neurons], "bonds" + ) + + async def _process_root_weights( + self, data: list, attribute: str, subtensor: "AsyncSubtensor" + ) -> Union[NDArray, "torch.nn.Parameter"]: + """ + Specifically processes the root weights data for the metagraph. This method is similar to :func:`_process_weights_or_bonds` + but is tailored for processing root weights, which have a different structure and significance in the network. + + Args: + data (list): The raw root weights data to be processed. + attribute (str): A string indicating the attribute type, here it's typically ``weights``. + subtensor (bittensor.core.async_subtensor.AsyncSubtensor): The subtensor instance used for additional data + and context needed in processing. + + Returns: + A tensor parameter encapsulating the processed root weights data. + + Internal Usage: + Used internally to process and set root weights for the metagraph:: + + self.root_weights = self._process_root_weights(raw_root_weights_data, "weights", subtensor) + """ + data_array = [] + n_subnets = await subtensor.get_total_subnets() or 0 + subnets = await subtensor.get_subnets() + for item in data: + if len(item) == 0: + if use_torch(): + data_array.append(torch.zeros(n_subnets)) + else: + data_array.append(np.zeros(n_subnets, dtype=np.float32)) + else: + uids, values = zip(*item) + # TODO: Validate and test the conversion of uids and values to tensor + data_array.append( + convert_root_weight_uids_and_vals_to_tensor( + n_subnets, list(uids), list(values), subnets + ) + ) + + tensor_param: Union[NDArray, "torch.nn.Parameter"] = ( + ( + torch.nn.Parameter(torch.stack(data_array), requires_grad=False) + if len(data_array) + else torch.nn.Parameter() + ) + if use_torch() + else ( + np.stack(data_array) + if len(data_array) + else np.array([], dtype=np.float32) + ) + ) + if len(data_array) == 0: + logging.warning( + f"Empty {attribute}_array on metagraph.sync(). The '{attribute}' tensor is empty." + ) + return tensor_param async def async_metagraph( diff --git a/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py b/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py index 37ba957731..24ba13c707 100644 --- a/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py +++ b/tests/unit_tests/extrinsics/asyncex/test_commit_reveal.py @@ -121,7 +121,9 @@ async def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): ) mocked_format_error_message = mocker.patch.object( - subtensor_module, "format_error_message", return_value="Formatted error", + subtensor_module, + "format_error_message", + return_value="Formatted error", ) # Call From 9a83347c3b59aff9861128a88596c83931dd3d98 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 23 Jan 2025 20:01:00 +0200 Subject: [PATCH 30/52] Metagraph rewrite part 2 --- bittensor/core/metagraph.py | 32 +++++++++++++++---------------- tests/e2e_tests/test_incentive.py | 6 +++--- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bittensor/core/metagraph.py b/bittensor/core/metagraph.py index 953dd833a5..8d60573b33 100644 --- a/bittensor/core/metagraph.py +++ b/bittensor/core/metagraph.py @@ -1,4 +1,3 @@ -import asyncio import copy import importlib import os @@ -162,6 +161,8 @@ class MetagraphMixin(ABC): Args: netuid (int): A unique identifier that distinguishes between different instances or versions of the Bittensor network. network (str): The name of the network, signifying specific configurations or iterations within the Bittensor ecosystem. + + Attributes: version (NDArray): The version number of the network, integral for tracking network updates. n (NDArray): The total number of neurons in the network, reflecting its size and complexity. block (NDArray): The current block number in the blockchain, crucial for synchronizing with the network's latest state. @@ -1071,7 +1072,7 @@ def __init__( metagraph = Metagraph(netuid=123, network="finney", lite=True, sync=True) """ # super(metagraph, self).__init__() - AsyncMetagraphMixin.__init__(self, netuid, network, lite, sync, subtensor) + MetagraphMixin.__init__(self, netuid, network, lite, sync, subtensor) self.netuid = netuid self.network, self.chain_endpoint = determine_chain_endpoint_and_network( @@ -1186,7 +1187,7 @@ def __init__( sync: bool = True, subtensor: Optional["AsyncSubtensor"] = None, ): - NumpyOrTorch.__init__(self, netuid, network, lite, sync, subtensor) + super().__init__(netuid, network, lite, sync, subtensor) async def __aenter__(self): if self.should_sync: @@ -1445,7 +1446,7 @@ def __init__( sync: bool = True, subtensor: Optional["Subtensor"] = None, ): - NumpyOrTorch.__init__(self, netuid, network, lite, sync, subtensor) + super().__init__(netuid, network, lite, sync, subtensor) if sync: self.sync() @@ -1562,8 +1563,8 @@ def _initialize_subtensor(self, subtensor: "Subtensor") -> "Subtensor": self.subtensor = subtensor return subtensor - async def _assign_neurons( - self, block: int, lite: bool, subtensor: "AsyncSubtensor" + def _assign_neurons( + self, block: int, lite: bool, subtensor: "Subtensor" ): """ Assigns neurons to the metagraph based on the provided block number and the lite flag. @@ -1590,14 +1591,14 @@ async def _assign_neurons( self._assign_neurons(block, lite, subtensor) """ if lite: - self.neurons = await subtensor.neurons_lite(block=block, netuid=self.netuid) + self.neurons = subtensor.neurons_lite(block=block, netuid=self.netuid) else: - self.neurons = await subtensor.neurons(block=block, netuid=self.netuid) + self.neurons = subtensor.neurons(block=block, netuid=self.netuid) self.lite = lite - async def _set_weights_and_bonds( - self, subtensor: Optional["AsyncSubtensor"] = None + def _set_weights_and_bonds( + self, subtensor: Optional["Subtensor"] = None ): """ Computes and sets the weights and bonds for each neuron in the metagraph. This method is responsible for @@ -1613,9 +1614,8 @@ async def _set_weights_and_bonds( self._set_weights_and_bonds(subtensor=subtensor) """ - # TODO: Check and test the computation of weights and bonds if self.netuid == 0: - self.weights = await self._process_root_weights( + self.weights = self._process_root_weights( [neuron.weights for neuron in self.neurons], "weights", subtensor, @@ -1628,8 +1628,8 @@ async def _set_weights_and_bonds( [neuron.bonds for neuron in self.neurons], "bonds" ) - async def _process_root_weights( - self, data: list, attribute: str, subtensor: "AsyncSubtensor" + def _process_root_weights( + self, data: list, attribute: str, subtensor: "Subtensor" ) -> Union[NDArray, "torch.nn.Parameter"]: """ Specifically processes the root weights data for the metagraph. This method is similar to :func:`_process_weights_or_bonds` @@ -1650,8 +1650,8 @@ async def _process_root_weights( self.root_weights = self._process_root_weights(raw_root_weights_data, "weights", subtensor) """ data_array = [] - n_subnets = await subtensor.get_total_subnets() or 0 - subnets = await subtensor.get_subnets() + n_subnets = subtensor.get_total_subnets() or 0 + subnets = subtensor.get_subnets() for item in data: if len(item) == 0: if use_torch(): diff --git a/tests/e2e_tests/test_incentive.py b/tests/e2e_tests/test_incentive.py index d7a063cffa..7bddcd4e5d 100644 --- a/tests/e2e_tests/test_incentive.py +++ b/tests/e2e_tests/test_incentive.py @@ -15,7 +15,7 @@ templates_repo, ) from bittensor.utils.balance import Balance -from bittensor.core.extrinsics.asyncex.weights import _do_set_weights +from bittensor.core.extrinsics.set_weights import _do_set_weights from bittensor.core.metagraph import Metagraph @@ -156,8 +156,8 @@ async def test_incentive(local_chain): await wait_epoch(subtensor) # Set weights by Alice on the subnet - await _do_set_weights( - subtensor=subtensor.async_subtensor, + _do_set_weights( + subtensor=subtensor, wallet=alice_wallet, uids=[1], vals=[65535], From 74c8d3e37f515b037e26fc58beeeab207cbe428c Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 23 Jan 2025 21:42:13 +0200 Subject: [PATCH 31/52] Serving tests --- bittensor/core/async_subtensor.py | 6 +- bittensor/core/extrinsics/asyncex/serving.py | 65 ++++---------- bittensor/core/extrinsics/serving.py | 65 ++++---------- bittensor/core/metagraph.py | 8 +- bittensor/core/subtensor.py | 7 +- bittensor/core/types.py | 94 +++++++++++++++++++- 6 files changed, 140 insertions(+), 105 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index f275174fc8..b68971a873 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -1352,9 +1352,9 @@ async def get_neuron_certificate( ) try: if certificate: - return ( - chr(certificate["algorithm"]) - + bytes(certificate["public_key"][0]).decode() + tuple_ascii = certificate["public_key"][0] + return chr(certificate["algorithm"]) + "".join( + chr(i) for i in tuple_ascii ) except AttributeError: diff --git a/bittensor/core/extrinsics/asyncex/serving.py b/bittensor/core/extrinsics/asyncex/serving.py index 70d10d4e78..4dd4417f99 100644 --- a/bittensor/core/extrinsics/asyncex/serving.py +++ b/bittensor/core/extrinsics/asyncex/serving.py @@ -10,11 +10,11 @@ Certificate, ) from bittensor.utils.btlogging import logging +from bittensor.core.types import AxonServeCallParams if TYPE_CHECKING: from bittensor.core.axon import Axon from bittensor.core.async_subtensor import AsyncSubtensor - from bittensor.core.types import AxonServeCallParams from bittensor_wallet import Wallet @@ -43,30 +43,15 @@ async def do_serve_axon( decentralized computation capabilities of Bittensor. """ - if call_params["certificate"] is None: - call_params_ = { - "version": call_params["version"], - "ip": call_params["ip"], - "port": call_params["port"], - "ip_type": call_params["ip_type"], - "netuid": call_params["netuid"], - } + if call_params.certificate is None: call_function = "serve_axon" else: - call_params_ = { - "version": call_params["version"], - "ip": call_params["ip"], - "port": call_params["port"], - "ip_type": call_params["ip_type"], - "netuid": call_params["netuid"], - "certificate": call_params["certificate"], - } call_function = "serve_axon_tls" call = await subtensor.substrate.compose_call( call_module="SubtensorModule", call_function=call_function, - call_params=call_params_, + call_params=call_params.dict(), ) extrinsic = await subtensor.substrate.create_signed_extrinsic( call=call, keypair=wallet.hotkey @@ -125,38 +110,26 @@ async def serve_extrinsic( logging.error(unlock.message) return False - params: "AxonServeCallParams" = { - "version": version_as_int, - "ip": net.ip_to_int(ip), - "port": port, - "ip_type": net.ip_version(ip), - "netuid": netuid, - "hotkey": wallet.hotkey.ss58_address, - "coldkey": wallet.coldkeypub.ss58_address, - "protocol": protocol, - "placeholder1": placeholder1, - "placeholder2": placeholder2, - "certificate": certificate, - } + params = AxonServeCallParams( + **{ + "version": version_as_int, + "ip": net.ip_to_int(ip), + "port": port, + "ip_type": net.ip_version(ip), + "netuid": netuid, + "hotkey": wallet.hotkey.ss58_address, + "coldkey": wallet.coldkeypub.ss58_address, + "protocol": protocol, + "placeholder1": placeholder1, + "placeholder2": placeholder2, + "certificate": certificate, + } + ) logging.debug("Checking axon ...") neuron = await subtensor.get_neuron_for_pubkey_and_subnet( wallet.hotkey.ss58_address, netuid=netuid ) - neuron_up_to_date = not neuron.is_null and params == { - "version": neuron.axon_info.version, - "ip": net.ip_to_int(neuron.axon_info.ip), - "port": neuron.axon_info.port, - "ip_type": neuron.axon_info.ip_type, - "netuid": neuron.netuid, - "hotkey": neuron.hotkey, - "coldkey": neuron.coldkey, - "protocol": neuron.axon_info.protocol, - "placeholder1": neuron.axon_info.placeholder1, - "placeholder2": neuron.axon_info.placeholder2, - } - output = params.copy() - output["coldkey"] = wallet.coldkeypub.ss58_address - output["hotkey"] = wallet.hotkey.ss58_address + neuron_up_to_date = not neuron.is_null and params == neuron if neuron_up_to_date: logging.debug( f"Axon already served on: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) " diff --git a/bittensor/core/extrinsics/serving.py b/bittensor/core/extrinsics/serving.py index 29af1e40e4..2dff1e61f2 100644 --- a/bittensor/core/extrinsics/serving.py +++ b/bittensor/core/extrinsics/serving.py @@ -9,12 +9,12 @@ Certificate, ) from bittensor.utils.btlogging import logging +from bittensor.core.types import AxonServeCallParams if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.axon import Axon from bittensor.core.subtensor import Subtensor - from bittensor.core.types import AxonServeCallParams def do_serve_axon( @@ -41,30 +41,15 @@ def do_serve_axon( This function is crucial for initializing and announcing a neuron's ``Axon`` service on the network, enhancing the decentralized computation capabilities of Bittensor. """ - if call_params["certificate"] is None: - call_params_ = { - "version": call_params["version"], - "ip": call_params["ip"], - "port": call_params["port"], - "ip_type": call_params["ip_type"], - "netuid": call_params["netuid"], - } + if call_params.certificate is None: call_function = "serve_axon" else: - call_params_ = { - "version": call_params["version"], - "ip": call_params["ip"], - "port": call_params["port"], - "ip_type": call_params["ip_type"], - "netuid": call_params["netuid"], - "certificate": call_params["certificate"], - } call_function = "serve_axon_tls" call = subtensor.substrate.compose_call( call_module="SubtensorModule", call_function=call_function, - call_params=call_params_, + call_params=call_params.dict(), ) extrinsic = subtensor.substrate.create_signed_extrinsic( call=call, keypair=wallet.hotkey @@ -123,38 +108,26 @@ def serve_extrinsic( logging.error(unlock.message) return False - params: "AxonServeCallParams" = { - "version": version_as_int, - "ip": net.ip_to_int(ip), - "port": port, - "ip_type": net.ip_version(ip), - "netuid": netuid, - "hotkey": wallet.hotkey.ss58_address, - "coldkey": wallet.coldkeypub.ss58_address, - "protocol": protocol, - "placeholder1": placeholder1, - "placeholder2": placeholder2, - "certificate": certificate, - } + params = AxonServeCallParams( + **{ + "version": version_as_int, + "ip": net.ip_to_int(ip), + "port": port, + "ip_type": net.ip_version(ip), + "netuid": netuid, + "hotkey": wallet.hotkey.ss58_address, + "coldkey": wallet.coldkeypub.ss58_address, + "protocol": protocol, + "placeholder1": placeholder1, + "placeholder2": placeholder2, + "certificate": certificate, + } + ) logging.debug("Checking axon ...") neuron = subtensor.get_neuron_for_pubkey_and_subnet( wallet.hotkey.ss58_address, netuid=netuid ) - neuron_up_to_date = not neuron.is_null and params == { - "version": neuron.axon_info.version, - "ip": net.ip_to_int(neuron.axon_info.ip), - "port": neuron.axon_info.port, - "ip_type": neuron.axon_info.ip_type, - "netuid": neuron.netuid, - "hotkey": neuron.hotkey, - "coldkey": neuron.coldkey, - "protocol": neuron.axon_info.protocol, - "placeholder1": neuron.axon_info.placeholder1, - "placeholder2": neuron.axon_info.placeholder2, - } - output = params.copy() - output["coldkey"] = wallet.coldkeypub.ss58_address - output["hotkey"] = wallet.hotkey.ss58_address + neuron_up_to_date = not neuron.is_null and params == neuron if neuron_up_to_date: logging.debug( f"Axon already served on: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) " diff --git a/bittensor/core/metagraph.py b/bittensor/core/metagraph.py index 8d60573b33..08d758f887 100644 --- a/bittensor/core/metagraph.py +++ b/bittensor/core/metagraph.py @@ -1563,9 +1563,7 @@ def _initialize_subtensor(self, subtensor: "Subtensor") -> "Subtensor": self.subtensor = subtensor return subtensor - def _assign_neurons( - self, block: int, lite: bool, subtensor: "Subtensor" - ): + def _assign_neurons(self, block: int, lite: bool, subtensor: "Subtensor"): """ Assigns neurons to the metagraph based on the provided block number and the lite flag. @@ -1597,9 +1595,7 @@ def _assign_neurons( self.neurons = subtensor.neurons(block=block, netuid=self.netuid) self.lite = lite - def _set_weights_and_bonds( - self, subtensor: Optional["Subtensor"] = None - ): + def _set_weights_and_bonds(self, subtensor: Optional["Subtensor"] = None): """ Computes and sets the weights and bonds for each neuron in the metagraph. This method is responsible for processing the raw weight and bond data obtained from the network and converting it into a structured format diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 355b6fe9bc..20fb8c7f12 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1049,9 +1049,10 @@ def get_neuron_certificate( ) try: if certificate: - public_key = bytes(certificate["public_key"][0]).hex() - return chr(certificate["algorithm"]) + f"0x{public_key}" - + tuple_ascii = certificate["public_key"][0] + return chr(certificate["algorithm"]) + "".join( + chr(i) for i in tuple_ascii + ) except AttributeError: return None return None diff --git a/bittensor/core/types.py b/bittensor/core/types.py index 601dc471cb..407d212b7a 100644 --- a/bittensor/core/types.py +++ b/bittensor/core/types.py @@ -22,6 +22,7 @@ from bittensor.utils.btlogging import logging from bittensor.core import settings from bittensor.core.config import Config +from bittensor.core.chain_data import NeuronInfo, NeuronInfoLite class SubtensorMixin(ABC): @@ -217,7 +218,7 @@ def determine_chain_endpoint_and_network( return "unknown", network -class AxonServeCallParams(TypedDict): +class AxonServeCallParams_(TypedDict): """Axon serve chain call parameters.""" version: int @@ -228,6 +229,97 @@ class AxonServeCallParams(TypedDict): certificate: Optional[Certificate] +class AxonServeCallParams: + def __init__( + self, + version: int, + ip: int, + port: int, + ip_type: int, + netuid: int, + hotkey: str, + coldkey: str, + protocol: int, + placeholder1: int, + placeholder2: int, + certificate: Optional[Certificate], + ): + self.version = version + self.ip = ip + self.port = port + self.ip_type = ip_type + self.netuid = netuid + self.hotkey = hotkey + self.coldkey = coldkey + self.protocol = protocol + self.placeholder1 = placeholder1 + self.placeholder2 = placeholder2 + self.certificate = certificate + + def __eq__(self, other): + if isinstance(other, self.__class__): + return all( + getattr(self, attr) == getattr(other, attr) for attr in self.__dict__ + ) + elif isinstance(other, dict): + return all(getattr(self, attr) == other.get(attr) for attr in self.__dict__) + elif isinstance(other, (NeuronInfo, NeuronInfoLite)): + return all( + [ + self.version == other.axon_info.version, + self.ip == networking.ip_to_int(other.axon_info.ip), + self.port == other.axon_info.port, + self.ip_type == other.axon_info.ip_type, + self.netuid == other.netuid, + self.hotkey == other.hotkey, + self.coldkey == other.coldkey, + self.protocol == other.axon_info.protocol, + self.placeholder1 == other.axon_info.placeholder1, + self.placeholder2 == other.axon_info.placeholder2, + ] + ) + else: + raise NotImplementedError( + f"AxonServeCallParams equality not implemented for {type(other)}" + ) + + def copy(self) -> "AxonServeCallParams": + return self.__class__( + self.version, + self.ip, + self.port, + self.ip_type, + self.netuid, + self.hotkey, + self.coldkey, + self.protocol, + self.placeholder1, + self.placeholder2, + self.certificate, + ) + + def dict(self) -> dict: + """ + Returns a dict representation of this object. If `self.certificate` is `None`, + it is not included in this. + """ + d = { + "version": self.version, + "ip": self.ip, + "port": self.port, + "ip_type": self.ip_type, + "netuid": self.netuid, + "hotkey": self.hotkey, + "coldkey": self.coldkey, + "protocol": self.protocol, + "placeholder1": self.placeholder1, + "placeholder2": self.placeholder2, + } + if self.certificate is not None: + d["certificate"] = self.certificate + return d + + class PrometheusServeCallParams(TypedDict): """Prometheus serve chain call parameters.""" From 35f1a125bac6bb8404e907f71095305cfd53198b Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 23 Jan 2025 23:33:58 +0200 Subject: [PATCH 32/52] E2E tests passing. --- tests/e2e_tests/test_root_set_weights.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e_tests/test_root_set_weights.py b/tests/e2e_tests/test_root_set_weights.py index 46fe6bbfe3..37993cf78a 100644 --- a/tests/e2e_tests/test_root_set_weights.py +++ b/tests/e2e_tests/test_root_set_weights.py @@ -8,7 +8,7 @@ wait_epoch, sudo_set_hyperparameter_values, ) -from bittensor.core.extrinsics.asyncex.root import _do_set_root_weights +from bittensor.core.extrinsics.root import _do_set_root_weights from tests.e2e_tests.utils.e2e_test_utils import ( setup_wallet, template_path, @@ -154,8 +154,8 @@ async def test_root_reg_hyperparams(local_chain): await wait_epoch(subtensor) # Set root weights to root network (0) and sn 1 - assert await _do_set_root_weights( - subtensor=subtensor.async_subtensor, + assert _do_set_root_weights( + subtensor=subtensor, wallet=alice_wallet, netuids=[0, 1], weights=weights, From 2eab16ca681ce758e3c8dad45f7b2070b920d91d Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Fri, 24 Jan 2025 00:00:12 +0200 Subject: [PATCH 33/52] Unit tests --- tests/unit_tests/test_axon.py | 12 +++++------- tests/unit_tests/test_chain_data.py | 15 ++++++++++++++- tests/unit_tests/test_metagraph.py | 3 ++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/unit_tests/test_axon.py b/tests/unit_tests/test_axon.py index 868e89ee01..512b4635ae 100644 --- a/tests/unit_tests/test_axon.py +++ b/tests/unit_tests/test_axon.py @@ -283,16 +283,16 @@ async def test_verify_body_integrity_happy_path( @pytest.mark.parametrize( - "body, expected_exception_message", + "body, expected_exception_name", [ - (b"", "Expecting value: line 1 column 1 (char 0)"), # Empty body - (b"not_json", "Expecting value: line 1 column 1 (char 0)"), # Non-JSON body + (b"", "JSONDecodeError"), # Empty body + (b"not_json", "JSONDecodeError"), # Non-JSON body ], ids=["empty_body", "non_json_body"], ) @pytest.mark.asyncio async def test_verify_body_integrity_edge_cases( - mock_request, axon_instance, body, expected_exception_message + mock_request, axon_instance, body, expected_exception_name ): # Arrange mock_request.body.return_value = body @@ -300,9 +300,7 @@ async def test_verify_body_integrity_edge_cases( # Act & Assert with pytest.raises(Exception) as exc_info: await axon_instance.verify_body_integrity(mock_request) - assert expected_exception_message in str( - exc_info.value - ), "Expected specific exception message." + assert exc_info.typename == expected_exception_name, "Expected specific exception" @pytest.mark.parametrize( diff --git a/tests/unit_tests/test_chain_data.py b/tests/unit_tests/test_chain_data.py index ec5c44ef94..63d8a69365 100644 --- a/tests/unit_tests/test_chain_data.py +++ b/tests/unit_tests/test_chain_data.py @@ -1,6 +1,7 @@ import pytest import torch +from async_substrate_interface.utils import json from bittensor.core.chain_data import AxonInfo, DelegateInfo from bittensor.core.chain_data.utils import ChainDataType @@ -102,7 +103,19 @@ def test_eq(other, expected, test_case): hotkey="hot", coldkey="cold", ), - '{"version": 1, "ip": "127.0.0.1", "port": 8080, "ip_type": 4, "hotkey": "hot", "coldkey": "cold", "protocol": 4, "placeholder1": 0, "placeholder2": 0}', + json.dumps( + { + "version": 1, + "ip": "127.0.0.1", + "port": 8080, + "ip_type": 4, + "hotkey": "hot", + "coldkey": "cold", + "protocol": 4, + "placeholder1": 0, + "placeholder2": 0, + } + ), "ID_to_string", ), ], diff --git a/tests/unit_tests/test_metagraph.py b/tests/unit_tests/test_metagraph.py index bbfd08c04f..1c8efe4fcc 100644 --- a/tests/unit_tests/test_metagraph.py +++ b/tests/unit_tests/test_metagraph.py @@ -47,7 +47,7 @@ async def test_set_metagraph_attributes(mock_environment): subtensor, neurons = mock_environment metagraph = Metagraph(1, sync=False) metagraph.neurons = neurons - await metagraph._set_metagraph_attributes(block=5, subtensor=subtensor) + metagraph._set_metagraph_attributes(block=5) # Check the attributes are set as expected assert metagraph.n.item() == len(neurons) @@ -158,6 +158,7 @@ def __contains__(self, item): ], ) def test_sync_warning_cases(block, test_id, metagraph_instance, mock_subtensor, caplog): + mock_subtensor.get_current_block.return_value = 601 metagraph_instance.sync(block=block, lite=True, subtensor=mock_subtensor) expected_message = "Attempting to sync longer than 300 blocks ago on a non-archive node. Please use the 'archive' network for subtensor and retry." From 4fb0586c1ef3a0f28a09221034733ecbe26ab6d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 13:04:42 +0100 Subject: [PATCH 34/52] Revert "update `bittensor/core/extrinsics/serving.py` unit tests" This reverts commit 87e53c12da4c6e5bfebaa59a5faa3454f9c5cc54. --- tests/unit_tests/extrinsics/test_serving.py | 499 ++++++++++++++------ 1 file changed, 362 insertions(+), 137 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_serving.py b/tests/unit_tests/extrinsics/test_serving.py index 2793844254..46eef17888 100644 --- a/tests/unit_tests/extrinsics/test_serving.py +++ b/tests/unit_tests/extrinsics/test_serving.py @@ -1,151 +1,376 @@ +# The MIT License (MIT) +# Copyright © 2024 Opentensor Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from unittest.mock import MagicMock, patch + +import pytest +from bittensor_wallet import Wallet + +from bittensor.core.axon import Axon +from bittensor.core.subtensor import Subtensor from bittensor.core.extrinsics import serving -def test_do_serve_axon(mocker): - """Verify that sync `do_serve_axon` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - call_params = mocker.Mock() - wait_for_inclusion = True - wait_for_finalization = True - - mocked_do_serve_axon = mocker.Mock() - serving.async_do_serve_axon = mocked_do_serve_axon - - # Call - result = serving.do_serve_axon( - subtensor=fake_subtensor, - wallet=fake_wallet, - call_params=call_params, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) +@pytest.fixture +def mock_subtensor(mocker): + mock_subtensor = mocker.MagicMock(spec=Subtensor) + mock_subtensor.network = "test_network" + mock_subtensor.substrate = mocker.MagicMock() + return mock_subtensor - # Asserts - mocked_do_serve_axon.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - call_params=call_params, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) +@pytest.fixture +def mock_wallet(mocker): + wallet = mocker.MagicMock(spec=Wallet) + wallet.hotkey.ss58_address = "hotkey_address" + wallet.coldkeypub.ss58_address = "coldkey_address" + return wallet -def test_serve_axon_extrinsic(mocker): - """Verify that sync `serve_axon_extrinsic` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - netuid = 2 - axon = mocker.Mock() - wait_for_inclusion = True - wait_for_finalization = True - certificate = mocker.Mock() - - mocked_execute_coroutine = mocker.patch.object(serving, "execute_coroutine") - mocked_serve_axon_extrinsic = mocker.Mock() - serving.async_serve_axon_extrinsic = mocked_serve_axon_extrinsic - - # Call - result = serving.serve_axon_extrinsic( - subtensor=fake_subtensor, - netuid=netuid, - axon=axon, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - certificate=certificate, - ) - # Asserts +@pytest.fixture +def mock_axon(mock_wallet, mocker): + axon = mocker.MagicMock(spec=Axon) + axon.wallet = mock_wallet() + axon.external_port = 9221 + return axon - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_serve_axon_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, - ) - mocked_serve_axon_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - netuid=netuid, - axon=axon, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - certificate=certificate, - ) - assert result == mocked_execute_coroutine.return_value - - -def test_publish_metadata(mocker): - """Verify that `publish_metadata` calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - netuid = 2 - data_type = "data_type" - data = b"data" - wait_for_inclusion = True - wait_for_finalization = True - - mocked_execute_coroutine = mocker.patch.object(serving, "execute_coroutine") - mocked_publish_metadata = mocker.Mock() - serving.async_publish_metadata = mocked_publish_metadata - - # Call - result = serving.publish_metadata( - subtensor=fake_subtensor, - wallet=fake_wallet, - netuid=netuid, - data_type=data_type, - data=data, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + +@pytest.mark.parametrize( + "ip,port,protocol,netuid,placeholder1,placeholder2,wait_for_inclusion,wait_for_finalization,expected,test_id,", + [ + ( + "192.168.1.1", + 9221, + 1, + 0, + 0, + 0, + False, + True, + True, + "happy-path-no-wait", + ), + ( + "192.168.1.2", + 9222, + 2, + 1, + 1, + 1, + True, + False, + True, + "happy-path-wait-for-inclusion", + ), + ( + "192.168.1.3", + 9223, + 3, + 2, + 2, + 2, + False, + True, + True, + "happy-path-wait-for-finalization", + ), + ], + ids=[ + "happy-path-no-wait", + "happy-path-wait-for-inclusion", + "happy-path-wait-for-finalization", + ], +) +def test_serve_extrinsic_happy_path( + mock_subtensor, + mock_wallet, + ip, + port, + protocol, + netuid, + placeholder1, + placeholder2, + wait_for_inclusion, + wait_for_finalization, + expected, + test_id, + mocker, +): + # Arrange + serving.do_serve_axon = mocker.MagicMock(return_value=(True, "")) + # Act + result = serving.serve_extrinsic( + mock_subtensor, + mock_wallet, + ip, + port, + protocol, + netuid, + placeholder1, + placeholder2, + wait_for_inclusion, + wait_for_finalization, ) - # Asserts + # Assert + assert result == expected, f"Test ID: {test_id}" - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_publish_metadata.return_value, - event_loop=fake_subtensor.event_loop, - ) - mocked_publish_metadata.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - netuid=netuid, - data_type=data_type, - data=data, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) - assert result == mocked_execute_coroutine.return_value - - -def test_get_metadata(mocker): - """Verify that `get_metadata` calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - netuid = 2 - hotkey = "hotkey" - block = 123 - - mocked_execute_coroutine = mocker.patch.object(serving, "execute_coroutine") - mocked_get_metadata = mocker.Mock() - serving.async_get_metadata = mocked_get_metadata - - # Call - result = serving.get_metadata( - subtensor=fake_subtensor, - netuid=netuid, - hotkey=hotkey, - block=block, - ) - # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_get_metadata.return_value, - event_loop=fake_subtensor.event_loop, +# Various edge cases +@pytest.mark.parametrize( + "ip,port,protocol,netuid,placeholder1,placeholder2,wait_for_inclusion,wait_for_finalization,expected,test_id,", + [ + ( + "192.168.1.4", + 9224, + 4, + 3, + 3, + 3, + True, + True, + True, + "edge_case_max_values", + ), + ], + ids=["edge-case-max-values"], +) +def test_serve_extrinsic_edge_cases( + mock_subtensor, + mock_wallet, + ip, + port, + protocol, + netuid, + placeholder1, + placeholder2, + wait_for_inclusion, + wait_for_finalization, + expected, + test_id, + mocker, +): + # Arrange + serving.do_serve_axon = mocker.MagicMock(return_value=(True, "")) + # Act + result = serving.serve_extrinsic( + mock_subtensor, + mock_wallet, + ip, + port, + protocol, + netuid, + placeholder1, + placeholder2, + wait_for_inclusion, + wait_for_finalization, ) - mocked_get_metadata.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - netuid=netuid, - hotkey=hotkey, - block=block, + + # Assert + assert result == expected, f"Test ID: {test_id}" + + +# Various error cases +@pytest.mark.parametrize( + "ip,port,protocol,netuid,placeholder1,placeholder2,wait_for_inclusion,wait_for_finalization,expected_error_message,test_id,", + [ + ( + "192.168.1.5", + 9225, + 5, + 4, + 4, + 4, + True, + True, + False, + "error-case-failed-serve", + ), + ], + ids=["error-case-failed-serve"], +) +def test_serve_extrinsic_error_cases( + mock_subtensor, + mock_wallet, + ip, + port, + protocol, + netuid, + placeholder1, + placeholder2, + wait_for_inclusion, + wait_for_finalization, + expected_error_message, + test_id, + mocker, +): + # Arrange + serving.do_serve_axon = mocker.MagicMock(return_value=(False, "Error serving axon")) + # Act + result = serving.serve_extrinsic( + mock_subtensor, + mock_wallet, + ip, + port, + protocol, + netuid, + placeholder1, + placeholder2, + wait_for_inclusion, + wait_for_finalization, ) - assert result == mocked_execute_coroutine.return_value + + # Assert + assert result == expected_error_message, f"Test ID: {test_id}" + + +@pytest.mark.parametrize( + "netuid, wait_for_inclusion, wait_for_finalization, external_ip, external_ip_success, serve_success, expected_result, test_id", + [ + # Happy path test + (1, False, True, "192.168.1.1", True, True, True, "happy-ext-ip"), + (1, False, True, None, True, True, True, "happy-net-external-ip"), + # Edge cases + (1, True, True, "192.168.1.1", True, True, True, "edge-case-wait"), + # Error cases + (1, False, True, None, False, True, False, "error-fetching-external-ip"), + ( + 1, + False, + True, + "192.168.1.1", + True, + False, + False, + "error-serving-axon", + ), + ], + ids=[ + "happy-axon-external-ip", + "happy-net-external-ip", + "edge-case-wait", + "error-fetching-external-ip", + "error-serving-axon", + ], +) +def test_serve_axon_extrinsic( + mock_subtensor, + mock_axon, + netuid, + wait_for_inclusion, + wait_for_finalization, + external_ip, + external_ip_success, + serve_success, + expected_result, + test_id, + mocker, +): + mock_axon.external_ip = external_ip + # Arrange + with patch( + "bittensor.utils.networking.get_external_ip", + side_effect=Exception("Failed to fetch IP") + if not external_ip_success + else MagicMock(return_value="192.168.1.1"), + ): + serving.do_serve_axon = mocker.MagicMock(return_value=(serve_success, "")) + # Act + if not external_ip_success: + with pytest.raises(RuntimeError): + serving.serve_axon_extrinsic( + mock_subtensor, + netuid, + mock_axon, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + else: + result = serving.serve_axon_extrinsic( + mock_subtensor, + netuid, + mock_axon, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + # Assert + assert result == expected_result, f"Test ID: {test_id}" + + +@pytest.mark.parametrize( + "wait_for_inclusion, wait_for_finalization, net_uid, type_u, data, response_success, expected_result, test_id", + [ + ( + True, + True, + 1, + "Sha256", + b"mock_bytes_data", + True, + True, + "happy-path-wait", + ), + ( + False, + False, + 1, + "Sha256", + b"mock_bytes_data", + True, + True, + "happy-path-no-wait", + ), + ], + ids=["happy-path-wait", "happy-path-no-wait"], +) +def test_publish_metadata( + mock_subtensor, + mock_wallet, + wait_for_inclusion, + wait_for_finalization, + net_uid, + type_u, + data, + response_success, + expected_result, + test_id, +): + # Arrange + with patch.object(mock_subtensor.substrate, "compose_call"), patch.object( + mock_subtensor.substrate, "create_signed_extrinsic" + ), patch.object( + mock_subtensor.substrate, + "submit_extrinsic", + return_value=MagicMock( + is_success=response_success, + process_events=MagicMock(), + error_message="error", + ), + ): + # Act + result = serving.publish_metadata( + self=mock_subtensor, + wallet=mock_wallet, + netuid=net_uid, + data_type=type_u, + data=data, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # Assert + assert result == expected_result, f"Test ID: {test_id}" From dcc6e5b2e273ae3cf05728eed971e83af0ab3534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 13:37:11 +0100 Subject: [PATCH 35/52] Revert "update `bittensor/core/extrinsics/transfer.py` unit tests" This reverts commit d10b688b45ceaf53014b374afd88efeea8534616. --- tests/unit_tests/extrinsics/test_transfer.py | 173 ++++++++++++++----- 1 file changed, 134 insertions(+), 39 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_transfer.py b/tests/unit_tests/extrinsics/test_transfer.py index f85e3f267e..af59d5769b 100644 --- a/tests/unit_tests/extrinsics/test_transfer.py +++ b/tests/unit_tests/extrinsics/test_transfer.py @@ -1,47 +1,142 @@ -from bittensor.core.extrinsics import transfer +import pytest +from bittensor.core import subtensor as subtensor_module +from bittensor.core.extrinsics.transfer import do_transfer +from bittensor.core.subtensor import Subtensor +from bittensor.utils.balance import Balance -def test_transfer_extrinsic(mocker): - """Verify that sync `transfer_extrinsic` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - dest = "hotkey" - amount = 1.1 - transfer_all = True - wait_for_inclusion = True - wait_for_finalization = True - keep_alive = False - mocked_execute_coroutine = mocker.patch.object(transfer, "execute_coroutine") - mocked_transfer_extrinsic = mocker.Mock() - transfer.async_transfer_extrinsic = mocked_transfer_extrinsic +@pytest.fixture +def subtensor(mocker): + fake_substrate = mocker.MagicMock() + fake_substrate.websocket.sock.getsockopt.return_value = 0 + mocker.patch.object( + subtensor_module, "SubstrateInterface", return_value=fake_substrate + ) + return Subtensor() + + +def test_do_transfer_is_success_true(subtensor, mocker): + """Successful do_transfer call.""" + # Prep + fake_wallet = mocker.MagicMock() + fake_dest = "SS58PUBLICKEY" + fake_transfer_balance = Balance(1) + fake_wait_for_inclusion = True + fake_wait_for_finalization = True + + subtensor.substrate.submit_extrinsic.return_value.is_success = True + + # Call + result = do_transfer( + subtensor, + fake_wallet, + fake_dest, + fake_transfer_balance, + fake_wait_for_inclusion, + fake_wait_for_finalization, + ) + + # Asserts + subtensor.substrate.compose_call.assert_called_once_with( + call_module="Balances", + call_function="transfer_allow_death", + call_params={"dest": fake_dest, "value": fake_transfer_balance.rao}, + ) + subtensor.substrate.create_signed_extrinsic.assert_called_once_with( + call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.coldkey + ) + subtensor.substrate.submit_extrinsic.assert_called_once_with( + subtensor.substrate.create_signed_extrinsic.return_value, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, + ) + subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + assert result == ( + True, + subtensor.substrate.submit_extrinsic.return_value.block_hash, + None, + ) + + +def test_do_transfer_is_success_false(subtensor, mocker): + """Successful do_transfer call.""" + # Prep + fake_wallet = mocker.MagicMock() + fake_dest = "SS58PUBLICKEY" + fake_transfer_balance = Balance(1) + fake_wait_for_inclusion = True + fake_wait_for_finalization = True + + subtensor.substrate.submit_extrinsic.return_value.is_success = False + + mocked_format_error_message = mocker.MagicMock() + subtensor_module.format_error_message = mocked_format_error_message + + # Call + result = do_transfer( + subtensor, + fake_wallet, + fake_dest, + fake_transfer_balance, + fake_wait_for_inclusion, + fake_wait_for_finalization, + ) + + # Asserts + subtensor.substrate.compose_call.assert_called_once_with( + call_module="Balances", + call_function="transfer_allow_death", + call_params={"dest": fake_dest, "value": fake_transfer_balance.rao}, + ) + subtensor.substrate.create_signed_extrinsic.assert_called_once_with( + call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.coldkey + ) + subtensor.substrate.submit_extrinsic.assert_called_once_with( + subtensor.substrate.create_signed_extrinsic.return_value, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, + ) + subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + + assert result == ( + False, + None, + subtensor.substrate.submit_extrinsic.return_value.error_message, + ) + + +def test_do_transfer_no_waits(subtensor, mocker): + """Successful do_transfer call.""" + # Prep + fake_wallet = mocker.MagicMock() + fake_dest = "SS58PUBLICKEY" + fake_transfer_balance = Balance(1) + fake_wait_for_inclusion = False + fake_wait_for_finalization = False # Call - result = transfer.transfer_extrinsic( - subtensor=fake_subtensor, - wallet=fake_wallet, - dest=dest, - amount=amount, - transfer_all=transfer_all, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - keep_alive=keep_alive, + result = do_transfer( + subtensor, + fake_wallet, + fake_dest, + fake_transfer_balance, + fake_wait_for_inclusion, + fake_wait_for_finalization, ) # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_transfer_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, - ) - mocked_transfer_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - destination=dest, - amount=amount, - transfer_all=transfer_all, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - keep_alive=keep_alive, - ) - assert result == mocked_execute_coroutine.return_value + subtensor.substrate.compose_call.assert_called_once_with( + call_module="Balances", + call_function="transfer_allow_death", + call_params={"dest": fake_dest, "value": fake_transfer_balance.rao}, + ) + subtensor.substrate.create_signed_extrinsic.assert_called_once_with( + call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.coldkey + ) + subtensor.substrate.submit_extrinsic.assert_called_once_with( + subtensor.substrate.create_signed_extrinsic.return_value, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, + ) + assert result == (True, None, None) From e4cde0f8a0db3f2c4ea457b106998ec046eb623e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 15:15:54 +0100 Subject: [PATCH 36/52] Revert "update `bittensor/core/extrinsics/set_weights.py` unit tests" This reverts commit 41cfd9995fdaa65ca872a92f0cbefb2844ffbbcb. --- .../unit_tests/extrinsics/test_set_weights.py | 275 +++++++++++++++--- 1 file changed, 239 insertions(+), 36 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_set_weights.py b/tests/unit_tests/extrinsics/test_set_weights.py index 116065463f..6c070bf5c4 100644 --- a/tests/unit_tests/extrinsics/test_set_weights.py +++ b/tests/unit_tests/extrinsics/test_set_weights.py @@ -1,47 +1,250 @@ -from bittensor.core.extrinsics import set_weights +from unittest.mock import MagicMock, patch +import pytest +import torch +from bittensor_wallet import Wallet -def test_set_weights_extrinsic(mocker): - """ "Verify that sync `set_weights_extrinsic` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - netuid = 2 - uids = [1, 2, 3, 4] - weights = [0.1, 0.2, 0.3, 0.4] - version_key = 2 - wait_for_inclusion = True - wait_for_finalization = True +from bittensor.core import subtensor as subtensor_module +from bittensor.core.extrinsics.set_weights import ( + do_set_weights, + set_weights_extrinsic, +) +from bittensor.core.settings import version_as_int +from bittensor.core.subtensor import Subtensor - mocked_execute_coroutine = mocker.patch.object(set_weights, "execute_coroutine") - mocked_set_weights_extrinsic = mocker.Mock() - set_weights.async_set_weights_extrinsic = mocked_set_weights_extrinsic + +@pytest.fixture +def mock_subtensor(): + mock = MagicMock(spec=Subtensor) + mock.network = "mock_network" + mock.substrate = MagicMock() + return mock + + +@pytest.fixture +def mock_wallet(): + mock = MagicMock(spec=Wallet) + return mock + + +@pytest.mark.parametrize( + "uids, weights, version_key, wait_for_inclusion, wait_for_finalization, expected_success, expected_message", + [ + ( + [1, 2], + [0.5, 0.5], + 0, + True, + False, + True, + "Successfully set weights and Finalized.", + ), + ( + [1, 2], + [0.5, 0.4], + 0, + False, + False, + True, + "Not waiting for finalization or inclusion.", + ), + ( + [1, 2], + [0.5, 0.5], + 0, + True, + False, + False, + "Mock error message", + ), + ], + ids=[ + "happy-flow", + "not-waiting-finalization-inclusion", + "error-flow", + ], +) +def test_set_weights_extrinsic( + mock_subtensor, + mock_wallet, + uids, + weights, + version_key, + wait_for_inclusion, + wait_for_finalization, + expected_success, + expected_message, +): + uids_tensor = torch.tensor(uids, dtype=torch.int64) + weights_tensor = torch.tensor(weights, dtype=torch.float32) + with patch( + "bittensor.utils.weight_utils.convert_weights_and_uids_for_emit", + return_value=(uids_tensor, weights_tensor), + ), patch( + "bittensor.core.extrinsics.set_weights.do_set_weights", + return_value=(expected_success, "Mock error message"), + ) as mock_do_set_weights: + result, message = set_weights_extrinsic( + subtensor=mock_subtensor, + wallet=mock_wallet, + netuid=123, + uids=uids, + weights=weights, + version_key=version_key, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + assert result == expected_success, f"Test {expected_message} failed." + assert message == expected_message, f"Test {expected_message} failed." + + +def test_do_set_weights_is_success(mock_subtensor, mocker): + """Successful _do_set_weights call.""" + # Prep + fake_wallet = mocker.MagicMock() + fake_uids = [1, 2, 3] + fake_vals = [4, 5, 6] + fake_netuid = 1 + fake_wait_for_inclusion = True + fake_wait_for_finalization = True + + mock_subtensor.substrate.submit_extrinsic.return_value.is_success = True # Call - result = set_weights.set_weights_extrinsic( - subtensor=fake_subtensor, + result = do_set_weights( + self=mock_subtensor, wallet=fake_wallet, - netuid=netuid, - uids=uids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + uids=fake_uids, + vals=fake_vals, + netuid=fake_netuid, + version_key=version_as_int, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, ) # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_set_weights_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + mock_subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="set_weights", + call_params={ + "dests": fake_uids, + "weights": fake_vals, + "netuid": fake_netuid, + "version_key": version_as_int, + }, ) - mocked_set_weights_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, + + mock_subtensor.substrate.create_signed_extrinsic.assert_called_once() + _, kwargs = mock_subtensor.substrate.create_signed_extrinsic.call_args + assert kwargs["call"] == mock_subtensor.substrate.compose_call.return_value + assert kwargs["keypair"] == fake_wallet.hotkey + assert kwargs["era"] == {"period": 5} + + mock_subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + assert result == (True, "Successfully set weights.") + + +def test_do_set_weights_is_not_success(mock_subtensor, mocker): + """Unsuccessful _do_set_weights call.""" + # Prep + fake_wallet = mocker.MagicMock() + fake_uids = [1, 2, 3] + fake_vals = [4, 5, 6] + fake_netuid = 1 + fake_wait_for_inclusion = True + fake_wait_for_finalization = True + + mock_subtensor.substrate.submit_extrinsic.return_value.is_success = False + mocked_format_error_message = mocker.MagicMock() + subtensor_module.format_error_message = mocked_format_error_message + + # Call + result = do_set_weights( + self=mock_subtensor, wallet=fake_wallet, - netuid=netuid, - uids=uids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) - assert result == mocked_execute_coroutine.return_value + uids=fake_uids, + vals=fake_vals, + netuid=fake_netuid, + version_key=version_as_int, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, + ) + + # Asserts + mock_subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="set_weights", + call_params={ + "dests": fake_uids, + "weights": fake_vals, + "netuid": fake_netuid, + "version_key": version_as_int, + }, + ) + + mock_subtensor.substrate.create_signed_extrinsic.assert_called_once() + _, kwargs = mock_subtensor.substrate.create_signed_extrinsic.call_args + assert kwargs["call"] == mock_subtensor.substrate.compose_call.return_value + assert kwargs["keypair"] == fake_wallet.hotkey + assert kwargs["era"] == {"period": 5} + + mock_subtensor.substrate.submit_extrinsic.assert_called_once_with( + mock_subtensor.substrate.create_signed_extrinsic.return_value, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, + ) + + mock_subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + assert result == ( + False, + "Subtensor returned `UnknownError(UnknownType)` error. This means: `Unknown Description`.", + ) + + +def test_do_set_weights_no_waits(mock_subtensor, mocker): + """Successful _do_set_weights call without wait flags for fake_wait_for_inclusion and fake_wait_for_finalization.""" + # Prep + fake_wallet = mocker.MagicMock() + fake_uids = [1, 2, 3] + fake_vals = [4, 5, 6] + fake_netuid = 1 + fake_wait_for_inclusion = False + fake_wait_for_finalization = False + + # Call + result = do_set_weights( + self=mock_subtensor, + wallet=fake_wallet, + uids=fake_uids, + vals=fake_vals, + netuid=fake_netuid, + version_key=version_as_int, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, + ) + + # Asserts + mock_subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="set_weights", + call_params={ + "dests": fake_uids, + "weights": fake_vals, + "netuid": fake_netuid, + "version_key": version_as_int, + }, + ) + + mock_subtensor.substrate.create_signed_extrinsic.assert_called_once() + _, kwargs = mock_subtensor.substrate.create_signed_extrinsic.call_args + assert kwargs["call"] == mock_subtensor.substrate.compose_call.return_value + assert kwargs["keypair"] == fake_wallet.hotkey + assert kwargs["era"] == {"period": 5} + + mock_subtensor.substrate.submit_extrinsic.assert_called_once_with( + mock_subtensor.substrate.create_signed_extrinsic.return_value, + wait_for_inclusion=fake_wait_for_inclusion, + wait_for_finalization=fake_wait_for_finalization, + ) + assert result == (True, "Not waiting for finalization or inclusion.") From d593bcfa4b32c3130a39a604bc12bf5a5aa6629f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 15:22:42 +0100 Subject: [PATCH 37/52] Revert "update `bittensor/core/extrinsics/root.py` unit tests" This reverts commit 785f5089c87f14c36bb62a6f214a4045b74b2b1e. --- tests/unit_tests/extrinsics/test_root.py | 269 ++++++++++++++++++----- 1 file changed, 214 insertions(+), 55 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_root.py b/tests/unit_tests/extrinsics/test_root.py index 7fae887011..96d90fe09a 100644 --- a/tests/unit_tests/extrinsics/test_root.py +++ b/tests/unit_tests/extrinsics/test_root.py @@ -3,81 +3,240 @@ from bittensor.core.extrinsics import root -def test_root_register_extrinsic(mocker): - """Verify that sync `root_register_extrinsic` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - wait_for_inclusion = True - wait_for_finalization = True +@pytest.fixture +def mock_subtensor(mocker): + mock = mocker.MagicMock(spec=Subtensor) + mock.network = "magic_mock" + return mock - mocked_execute_coroutine = mocker.patch.object(root, "execute_coroutine") - mocked_root_register_extrinsic = mocker.Mock() - root.async_root_register_extrinsic = mocked_root_register_extrinsic - # Call - result = root.root_register_extrinsic( - subtensor=fake_subtensor, - wallet=fake_wallet, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - ) +@pytest.fixture +def mock_wallet(mocker): + mock = mocker.MagicMock() + mock.hotkey.ss58_address = "fake_hotkey_address" + return mock - # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_root_register_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, +@pytest.mark.parametrize( + "wait_for_inclusion, wait_for_finalization, hotkey_registered, registration_success, expected_result", + [ + ( + False, + True, + [True, None], + True, + True, + ), # Already registered after attempt + ( + False, + True, + [False, True], + True, + True, + ), # Registration succeeds with user confirmation + (False, True, [False, False], False, None), # Registration fails + ( + False, + True, + [False, False], + True, + None, + ), # Registration succeeds but neuron not found + ], + ids=[ + "success-already-registered", + "success-registration-succeeds", + "failure-registration-failed", + "failure-neuron-not-found", + ], +) +def test_root_register_extrinsic( + mock_subtensor, + mock_wallet, + wait_for_inclusion, + wait_for_finalization, + hotkey_registered, + registration_success, + expected_result, + mocker, +): + # Arrange + mock_subtensor.is_hotkey_registered.side_effect = hotkey_registered + + # Preps + mock_register = mocker.Mock( + return_value=(registration_success, "Error registering") ) - mocked_root_register_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - netuid=0, + root._do_root_register = mock_register + + # Act + result = root.root_register_extrinsic( + subtensor=mock_subtensor, + wallet=mock_wallet, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - assert result == mocked_execute_coroutine.return_value + # Assert + assert result == expected_result + if not hotkey_registered[0]: + mock_register.assert_called_once() -def test_set_root_weights_extrinsic(mocker): - """Verify that sync `set_root_weights_extrinsic` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - netuids = [1, 2, 3, 4] - weights = [0.1, 0.2, 0.3, 0.4] - version_key = 2 - wait_for_inclusion = True - wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(root, "execute_coroutine") - mocked_set_root_weights_extrinsic = mocker.Mock() - root.async_set_root_weights_extrinsic = mocked_set_root_weights_extrinsic +@pytest.mark.parametrize( + "wait_for_inclusion, wait_for_finalization, netuids, weights, expected_success", + [ + (True, False, [1, 2], [0.5, 0.5], True), # Success - weights set + ( + False, + False, + [1, 2], + [0.5, 0.5], + True, + ), # Success - weights set no wait + ( + True, + False, + [1, 2], + [2000, 20], + True, + ), # Success - large value to be normalized + ( + True, + False, + [1, 2], + [2000, 0], + True, + ), # Success - single large value + ( + True, + False, + [1, 2], + [0.5, 0.5], + False, + ), # Failure - setting weights failed + ( + True, + False, + [], + [], + False, + ), # Exception catched - ValueError 'min() arg is an empty sequence' + ], + ids=[ + "success-weights-set", + "success-not-wait", + "success-large-value", + "success-single-value", + "failure-setting-weights", + "failure-value-error-exception", + ], +) +def test_set_root_weights_extrinsic( + mock_subtensor, + mock_wallet, + wait_for_inclusion, + wait_for_finalization, + netuids, + weights, + expected_success, + mocker, +): + # Preps + root._do_set_root_weights = mocker.Mock( + return_value=(expected_success, "Mock error") + ) + mock_subtensor.min_allowed_weights = mocker.Mock(return_value=0) + mock_subtensor.max_weight_limit = mocker.Mock(return_value=1) # Call result = root.set_root_weights_extrinsic( - subtensor=fake_subtensor, - wallet=fake_wallet, + subtensor=mock_subtensor, + wallet=mock_wallet, netuids=netuids, weights=weights, - version_key=version_key, + version_key=0, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) # Asserts + assert result == expected_success - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_set_root_weights_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, - ) - mocked_set_root_weights_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - netuids=netuids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + +@pytest.mark.parametrize( + "wait_for_inclusion, wait_for_finalization, netuids, weights, user_response, expected_success", + [ + (True, False, [1, 2], [0.5, 0.5], True, True), # Success - weights set + ( + False, + False, + [1, 2], + [0.5, 0.5], + None, + True, + ), # Success - weights set no wait + ( + True, + False, + [1, 2], + [2000, 20], + True, + True, + ), # Success - large value to be normalized + ( + True, + False, + [1, 2], + [2000, 0], + True, + True, + ), # Success - single large value + ( + True, + False, + [1, 2], + [0.5, 0.5], + None, + False, + ), # Failure - setting weights failed + ( + True, + False, + [], + [], + False, + False, + ), # Exception catched - ValueError 'min() arg is an empty sequence' + ], + ids=[ + "success-weights-set", + "success-not-wait", + "success-large-value", + "success-single-value", + "failure-setting-weights", + "failure-value-error-exception", + ], +) +def test_set_root_weights_extrinsic_torch( + mock_subtensor, + mock_wallet, + wait_for_inclusion, + wait_for_finalization, + netuids, + weights, + user_response, + expected_success, + force_legacy_torch_compatible_api, + mocker, +): + test_set_root_weights_extrinsic( + mock_subtensor, + mock_wallet, + wait_for_inclusion, + wait_for_finalization, + netuids, + weights, + expected_success, + mocker, ) - assert result == mocked_execute_coroutine.return_value From 7f64dd1bde174b86fdd9257236e56cbf2afcf93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 15:31:50 +0100 Subject: [PATCH 38/52] Revert "typo" This reverts commit 09751b1a8140a903403be79ece8eadfe9ffffe48. --- tests/unit_tests/extrinsics/test_commit_weights.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/extrinsics/test_commit_weights.py b/tests/unit_tests/extrinsics/test_commit_weights.py index 54a0b80b74..fcb55835fb 100644 --- a/tests/unit_tests/extrinsics/test_commit_weights.py +++ b/tests/unit_tests/extrinsics/test_commit_weights.py @@ -43,7 +43,7 @@ def test_commit_weights_extrinsic(mocker): def test_reveal_weights_extrinsic(mocker): - """Verify that sync `reveal_weights_extrinsic` method calls proper async method.""" + """ "Verify that sync `reveal_weights_extrinsic` method calls proper async method.""" # Preps fake_subtensor = mocker.Mock() fake_wallet = mocker.Mock() From 57e92a7ba3800dc03d91dee19f1ca880174a7afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 15:31:59 +0100 Subject: [PATCH 39/52] Revert "ruff" This reverts commit 50e383eca50ff26d764824437788fd5ad48f8b74. --- .../unit_tests/extrinsics/test_commit_reveal.py | 8 ++++---- .../extrinsics/test_commit_weights.py | 16 ++++++++-------- .../unit_tests/extrinsics/test_registration.py | 17 +++++++++-------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_commit_reveal.py b/tests/unit_tests/extrinsics/test_commit_reveal.py index 30bd7c0e63..9302ed0391 100644 --- a/tests/unit_tests/extrinsics/test_commit_reveal.py +++ b/tests/unit_tests/extrinsics/test_commit_reveal.py @@ -2,7 +2,7 @@ def test_commit_reveal_v3_extrinsic(mocker): - """ "Verify that sync `commit_reveal_v3_extrinsic` method calls proper async method.""" + """"Verify that sync `commit_reveal_v3_extrinsic` method calls proper async method.""" # Preps fake_subtensor = mocker.Mock() fake_wallet = mocker.Mock() @@ -26,14 +26,14 @@ def test_commit_reveal_v3_extrinsic(mocker): weights=weights, version_key=version_key, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) # Asserts mocked_execute_coroutine.assert_called_once_with( coroutine=mocked_commit_reveal_v3_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + event_loop=fake_subtensor.event_loop ) mocked_commit_reveal_v3_extrinsic.assert_called_once_with( subtensor=fake_subtensor.async_subtensor, @@ -43,6 +43,6 @@ def test_commit_reveal_v3_extrinsic(mocker): weights=weights, version_key=version_key, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) assert result == mocked_execute_coroutine.return_value diff --git a/tests/unit_tests/extrinsics/test_commit_weights.py b/tests/unit_tests/extrinsics/test_commit_weights.py index fcb55835fb..01d7691770 100644 --- a/tests/unit_tests/extrinsics/test_commit_weights.py +++ b/tests/unit_tests/extrinsics/test_commit_weights.py @@ -2,7 +2,7 @@ def test_commit_weights_extrinsic(mocker): - """ "Verify that sync `commit_weights_extrinsic` method calls proper async method.""" + """"Verify that sync `commit_weights_extrinsic` method calls proper async method.""" # Preps fake_subtensor = mocker.Mock() fake_wallet = mocker.Mock() @@ -22,14 +22,14 @@ def test_commit_weights_extrinsic(mocker): netuid=netuid, commit_hash=commit_hash, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) # Asserts mocked_execute_coroutine.assert_called_once_with( coroutine=mocked_commit_weights_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + event_loop=fake_subtensor.event_loop ) mocked_commit_weights_extrinsic.assert_called_once_with( subtensor=fake_subtensor.async_subtensor, @@ -37,13 +37,13 @@ def test_commit_weights_extrinsic(mocker): netuid=netuid, commit_hash=commit_hash, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) assert result == mocked_execute_coroutine.return_value def test_reveal_weights_extrinsic(mocker): - """ "Verify that sync `reveal_weights_extrinsic` method calls proper async method.""" + """"Verify that sync `reveal_weights_extrinsic` method calls proper async method.""" # Preps fake_subtensor = mocker.Mock() fake_wallet = mocker.Mock() @@ -69,14 +69,14 @@ def test_reveal_weights_extrinsic(mocker): salt=salt, version_key=version_key, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) # Asserts mocked_execute_coroutine.assert_called_once_with( coroutine=mocked_reveal_weights_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + event_loop=fake_subtensor.event_loop ) mocked_reveal_weights_extrinsic.assert_called_once_with( subtensor=fake_subtensor.async_subtensor, @@ -87,6 +87,6 @@ def test_reveal_weights_extrinsic(mocker): salt=salt, version_key=version_key, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) assert result == mocked_execute_coroutine.return_value diff --git a/tests/unit_tests/extrinsics/test_registration.py b/tests/unit_tests/extrinsics/test_registration.py index 99b2c3cd1d..2edc20b07d 100644 --- a/tests/unit_tests/extrinsics/test_registration.py +++ b/tests/unit_tests/extrinsics/test_registration.py @@ -2,7 +2,7 @@ def test_burned_register_extrinsic(mocker): - """ "Verify that sync `burned_register_extrinsic` method calls proper async method.""" + """"Verify that sync `burned_register_extrinsic` method calls proper async method.""" # Preps fake_subtensor = mocker.Mock() fake_wallet = mocker.Mock() @@ -20,27 +20,27 @@ def test_burned_register_extrinsic(mocker): wallet=fake_wallet, netuid=netuid, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) # Asserts mocked_execute_coroutine.assert_called_once_with( coroutine=mocked_burned_register_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + event_loop=fake_subtensor.event_loop ) mocked_burned_register_extrinsic.assert_called_once_with( subtensor=fake_subtensor.async_subtensor, wallet=fake_wallet, netuid=netuid, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + wait_for_finalization=wait_for_finalization ) assert result == mocked_execute_coroutine.return_value def test_register_extrinsic(mocker): - """ "Verify that sync `register_extrinsic` method calls proper async method.""" + """"Verify that sync `register_extrinsic` method calls proper async method.""" # Preps fake_subtensor = mocker.Mock() fake_wallet = mocker.Mock() @@ -74,14 +74,14 @@ def test_register_extrinsic(mocker): tpb=tpb, num_processes=num_processes, update_interval=update_interval, - log_verbose=log_verbose, + log_verbose=log_verbose ) # Asserts mocked_execute_coroutine.assert_called_once_with( coroutine=mocked_register_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + event_loop=fake_subtensor.event_loop ) mocked_register_extrinsic.assert_called_once_with( subtensor=fake_subtensor.async_subtensor, @@ -96,6 +96,7 @@ def test_register_extrinsic(mocker): tpb=tpb, num_processes=num_processes, update_interval=update_interval, - log_verbose=log_verbose, + log_verbose=log_verbose ) assert result == mocked_execute_coroutine.return_value + From 901ad69a2a9d3f71c17e4e45186598cd0fdea51c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 15:32:06 +0100 Subject: [PATCH 40/52] Revert "update `commit_weights.py` unit test" This reverts commit f940e22298cbe74b0a70011f55b49dc62e9d9ae4. --- .../extrinsics/test_commit_weights.py | 146 +++++++++++------- 1 file changed, 94 insertions(+), 52 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_commit_weights.py b/tests/unit_tests/extrinsics/test_commit_weights.py index 01d7691770..57d78a8013 100644 --- a/tests/unit_tests/extrinsics/test_commit_weights.py +++ b/tests/unit_tests/extrinsics/test_commit_weights.py @@ -1,92 +1,134 @@ -from bittensor.core.extrinsics import commit_weights +import pytest +from bittensor.core import subtensor as subtensor_module +from bittensor.core.settings import version_as_int +from bittensor.core.subtensor import Subtensor +from bittensor.core.extrinsics.commit_weights import ( + do_commit_weights, + do_reveal_weights, +) -def test_commit_weights_extrinsic(mocker): - """"Verify that sync `commit_weights_extrinsic` method calls proper async method.""" + +@pytest.fixture +def subtensor(mocker): + fake_substrate = mocker.MagicMock() + fake_substrate.websocket.sock.getsockopt.return_value = 0 + mocker.patch.object( + subtensor_module, "SubstrateInterface", return_value=fake_substrate + ) + return Subtensor() + + +def test_do_commit_weights(subtensor, mocker): + """Successful _do_commit_weights call.""" # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() + fake_wallet = mocker.MagicMock() netuid = 1 - commit_hash = "0x1234567890abcdef" + commit_hash = "fake_commit_hash" wait_for_inclusion = True wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(commit_weights, "execute_coroutine") - mocked_commit_weights_extrinsic = mocker.Mock() - commit_weights.async_commit_weights_extrinsic = mocked_commit_weights_extrinsic + subtensor.substrate.submit_extrinsic.return_value.is_success = None + + mocked_format_error_message = mocker.MagicMock() + subtensor_module.format_error_message = mocked_format_error_message # Call - result = commit_weights.commit_weights_extrinsic( - subtensor=fake_subtensor, + result = do_commit_weights( + self=subtensor, wallet=fake_wallet, netuid=netuid, commit_hash=commit_hash, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization + wait_for_finalization=wait_for_finalization, ) - # Asserts - - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_commit_weights_extrinsic.return_value, - event_loop=fake_subtensor.event_loop + # Assertions + subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="commit_weights", + call_params={ + "netuid": netuid, + "commit_hash": commit_hash, + }, ) - mocked_commit_weights_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - netuid=netuid, - commit_hash=commit_hash, + + subtensor.substrate.create_signed_extrinsic.assert_called_once() + _, kwargs = subtensor.substrate.create_signed_extrinsic.call_args + assert kwargs["call"] == subtensor.substrate.compose_call.return_value + assert kwargs["keypair"] == fake_wallet.hotkey + + subtensor.substrate.submit_extrinsic.assert_called_once_with( + subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization + wait_for_finalization=wait_for_finalization, + ) + + subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + + assert result == ( + False, + subtensor.substrate.submit_extrinsic.return_value.error_message, ) - assert result == mocked_execute_coroutine.return_value -def test_reveal_weights_extrinsic(mocker): - """"Verify that sync `reveal_weights_extrinsic` method calls proper async method.""" +def test_do_reveal_weights(subtensor, mocker): + """Verifies that the `_do_reveal_weights` method interacts with the right substrate methods.""" # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() + fake_wallet = mocker.MagicMock() + fake_wallet.hotkey = "hotkey" + netuid = 1 uids = [1, 2, 3, 4] - weights = [5, 6, 7, 8] - salt = [1, 2, 3, 4] - version_key = 2 + values = [1, 2, 3, 4] + salt = [4, 2, 2, 1] wait_for_inclusion = True wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(commit_weights, "execute_coroutine") - mocked_reveal_weights_extrinsic = mocker.Mock() - commit_weights.async_reveal_weights_extrinsic = mocked_reveal_weights_extrinsic + subtensor.substrate.submit_extrinsic.return_value.is_success = None + + mocked_format_error_message = mocker.MagicMock() + subtensor_module.format_error_message = mocked_format_error_message # Call - result = commit_weights.reveal_weights_extrinsic( - subtensor=fake_subtensor, + result = do_reveal_weights( + self=subtensor, wallet=fake_wallet, netuid=netuid, uids=uids, - weights=weights, + values=values, salt=salt, - version_key=version_key, + version_key=version_as_int, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization + wait_for_finalization=wait_for_finalization, ) # Asserts + subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="reveal_weights", + call_params={ + "netuid": netuid, + "uids": uids, + "values": values, + "salt": salt, + "version_key": version_as_int, + }, + ) - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_reveal_weights_extrinsic.return_value, - event_loop=fake_subtensor.event_loop + subtensor.substrate.create_signed_extrinsic.assert_called_once_with( + call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.hotkey ) - mocked_reveal_weights_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - netuid=netuid, - uids=uids, - weights=weights, - salt=salt, - version_key=version_key, + + subtensor.substrate.submit_extrinsic.assert_called_once_with( + subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization + wait_for_finalization=wait_for_finalization, + ) + + subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + + assert result == ( + False, + subtensor.substrate.submit_extrinsic.return_value.error_message, ) - assert result == mocked_execute_coroutine.return_value From f674a017949c298d65cc1bec3439aee9953c7be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 15:32:15 +0100 Subject: [PATCH 41/52] Revert "update `commit_reveal` unit test" This reverts commit 86a46d1b7ecf75a025d03758d58eaffc520cf5e5. --- .../extrinsics/test_commit_reveal.py | 373 ++++++++++++++++-- 1 file changed, 339 insertions(+), 34 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_commit_reveal.py b/tests/unit_tests/extrinsics/test_commit_reveal.py index 9302ed0391..e1f7c9f877 100644 --- a/tests/unit_tests/extrinsics/test_commit_reveal.py +++ b/tests/unit_tests/extrinsics/test_commit_reveal.py @@ -1,48 +1,353 @@ +from bittensor.core import subtensor as subtensor_module +from bittensor.core.chain_data import SubnetHyperparameters +from bittensor.core.subtensor import Subtensor from bittensor.core.extrinsics import commit_reveal +import pytest +import torch +import numpy as np -def test_commit_reveal_v3_extrinsic(mocker): - """"Verify that sync `commit_reveal_v3_extrinsic` method calls proper async method.""" +@pytest.fixture +def subtensor(mocker): + fake_substrate = mocker.MagicMock() + fake_substrate.websocket.sock.getsockopt.return_value = 0 + mocker.patch.object( + subtensor_module, "SubstrateInterface", return_value=fake_substrate + ) + yield Subtensor() + + +@pytest.fixture +def hyperparams(): + yield SubnetHyperparameters( + rho=0, + kappa=0, + immunity_period=0, + min_allowed_weights=0, + max_weight_limit=0.0, + tempo=0, + min_difficulty=0, + max_difficulty=0, + weights_version=0, + weights_rate_limit=0, + adjustment_interval=0, + activity_cutoff=0, + registration_allowed=False, + target_regs_per_interval=0, + min_burn=0, + max_burn=0, + bonds_moving_avg=0, + max_regs_per_block=0, + serving_rate_limit=0, + max_validators=0, + adjustment_alpha=0, + difficulty=0, + commit_reveal_weights_interval=0, + commit_reveal_weights_enabled=True, + alpha_high=0, + alpha_low=0, + liquid_alpha_enabled=False, + ) + + +def test_do_commit_reveal_v3_success(mocker, subtensor): + """Test successful commit-reveal with wait for finalization.""" + # Preps + fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_netuid = 1 + fake_commit = b"fake_commit" + fake_reveal_round = 1 + + mocked_compose_call = mocker.patch.object(subtensor.substrate, "compose_call") + mocked_create_signed_extrinsic = mocker.patch.object( + subtensor.substrate, "create_signed_extrinsic" + ) + mocked_submit_extrinsic = mocker.patch.object(commit_reveal, "submit_extrinsic") + + # Call + result = commit_reveal._do_commit_reveal_v3( + self=subtensor, + wallet=fake_wallet, + netuid=fake_netuid, + commit=fake_commit, + reveal_round=fake_reveal_round, + ) + + # Asserts + mocked_compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="commit_crv3_weights", + call_params={ + "netuid": fake_netuid, + "commit": fake_commit, + "reveal_round": fake_reveal_round, + }, + ) + mocked_create_signed_extrinsic.assert_called_once_with( + call=mocked_compose_call.return_value, keypair=fake_wallet.hotkey + ) + mocked_submit_extrinsic.assert_called_once_with( + subtensor=subtensor, + extrinsic=mocked_create_signed_extrinsic.return_value, + wait_for_inclusion=False, + wait_for_finalization=False, + ) + assert result == (True, "Not waiting for finalization or inclusion.") + + +def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): + """Test commit-reveal fails due to an error in submission.""" + # Preps + fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_netuid = 1 + fake_commit = b"fake_commit" + fake_reveal_round = 1 + + mocked_compose_call = mocker.patch.object(subtensor.substrate, "compose_call") + mocked_create_signed_extrinsic = mocker.patch.object( + subtensor.substrate, "create_signed_extrinsic" + ) + mocked_submit_extrinsic = mocker.patch.object( + commit_reveal, + "submit_extrinsic", + return_value=mocker.Mock(is_success=False, error_message="Mocked error"), + ) + mocked_format_error_message = mocker.patch.object( + commit_reveal, "format_error_message", return_value="Formatted error" + ) + + # Call + result = commit_reveal._do_commit_reveal_v3( + self=subtensor, + wallet=fake_wallet, + netuid=fake_netuid, + commit=fake_commit, + reveal_round=fake_reveal_round, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + # Asserts + mocked_compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="commit_crv3_weights", + call_params={ + "netuid": fake_netuid, + "commit": fake_commit, + "reveal_round": fake_reveal_round, + }, + ) + mocked_create_signed_extrinsic.assert_called_once_with( + call=mocked_compose_call.return_value, keypair=fake_wallet.hotkey + ) + mocked_submit_extrinsic.assert_called_once_with( + subtensor=subtensor, + extrinsic=mocked_create_signed_extrinsic.return_value, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + mocked_format_error_message.assert_called_once_with("Mocked error") + assert result == (False, "Formatted error") + + +def test_commit_reveal_v3_extrinsic_success_with_torch(mocker, subtensor, hyperparams): + """Test successful commit-reveal with torch tensors.""" + # Preps + fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_netuid = 1 + fake_uids = torch.tensor([1, 2, 3], dtype=torch.int64) + fake_weights = torch.tensor([0.1, 0.2, 0.7], dtype=torch.float32) + fake_commit_for_reveal = b"mock_commit_for_reveal" + fake_reveal_round = 1 + + # Mocks + + mocked_uids = mocker.Mock() + mocked_weights = mocker.Mock() + mocked_convert_weights_and_uids_for_emit = mocker.patch.object( + commit_reveal, + "convert_weights_and_uids_for_emit", + return_value=(mocked_uids, mocked_weights), + ) + mocked_get_subnet_reveal_period_epochs = mocker.patch.object( + subtensor, "get_subnet_reveal_period_epochs" + ) + mocked_get_encrypted_commit = mocker.patch.object( + commit_reveal, + "get_encrypted_commit", + return_value=(fake_commit_for_reveal, fake_reveal_round), + ) + mock_do_commit_reveal_v3 = mocker.patch.object( + commit_reveal, "_do_commit_reveal_v3", return_value=(True, "Success") + ) + mock_block = mocker.patch.object(subtensor, "get_current_block", return_value=1) + mock_hyperparams = mocker.patch.object( + subtensor, + "get_subnet_hyperparameters", + return_value=hyperparams, + ) + + # Call + success, message = commit_reveal.commit_reveal_v3_extrinsic( + subtensor=subtensor, + wallet=fake_wallet, + netuid=fake_netuid, + uids=fake_uids, + weights=fake_weights, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + # Asserts + assert success is True + assert message == "reveal_round:1" + mocked_convert_weights_and_uids_for_emit.assert_called_once_with( + fake_uids, fake_weights + ) + mocked_get_encrypted_commit.assert_called_once_with( + uids=mocked_uids, + weights=mocked_weights, + subnet_reveal_period_epochs=mock_hyperparams.return_value.commit_reveal_weights_interval, + version_key=commit_reveal.version_as_int, + tempo=mock_hyperparams.return_value.tempo, + netuid=fake_netuid, + current_block=mock_block.return_value, + ) + mock_do_commit_reveal_v3.assert_called_once_with( + self=subtensor, + wallet=fake_wallet, + netuid=fake_netuid, + commit=fake_commit_for_reveal, + reveal_round=fake_reveal_round, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + +def test_commit_reveal_v3_extrinsic_success_with_numpy(mocker, subtensor, hyperparams): + """Test successful commit-reveal with numpy arrays.""" + # Preps + fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_netuid = 1 + fake_uids = np.array([1, 2, 3], dtype=np.int64) + fake_weights = np.array([0.1, 0.2, 0.7], dtype=np.float32) + + mock_convert = mocker.patch.object( + commit_reveal, + "convert_weights_and_uids_for_emit", + return_value=(fake_uids, fake_weights), + ) + mock_encode_drand = mocker.patch.object( + commit_reveal, "get_encrypted_commit", return_value=(b"commit", 0) + ) + mock_do_commit = mocker.patch.object( + commit_reveal, "_do_commit_reveal_v3", return_value=(True, "Committed!") + ) + mocker.patch.object(subtensor, "get_current_block", return_value=1) + mocker.patch.object( + subtensor, + "get_subnet_hyperparameters", + return_value=hyperparams, + ) + + # Call + success, message = commit_reveal.commit_reveal_v3_extrinsic( + subtensor=subtensor, + wallet=fake_wallet, + netuid=fake_netuid, + uids=fake_uids, + weights=fake_weights, + wait_for_inclusion=False, + wait_for_finalization=False, + ) + + # Asserts + assert success is True + assert message == "reveal_round:0" + mock_convert.assert_called_once_with(fake_uids, fake_weights) + mock_encode_drand.assert_called_once() + mock_do_commit.assert_called_once() + + +def test_commit_reveal_v3_extrinsic_response_false(mocker, subtensor, hyperparams): + """Test unsuccessful commit-reveal with torch.""" # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - netuid = 1 - uids = [1, 2, 3, 4] - weights = [0.1, 0.2, 0.3, 0.4] - version_key = 2 - wait_for_inclusion = True - wait_for_finalization = True - - mocked_execute_coroutine = mocker.patch.object(commit_reveal, "execute_coroutine") - mocked_commit_reveal_v3_extrinsic = mocker.Mock() - commit_reveal.async_commit_reveal_v3_extrinsic = mocked_commit_reveal_v3_extrinsic + fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_netuid = 1 + fake_uids = torch.tensor([1, 2, 3], dtype=torch.int64) + fake_weights = torch.tensor([0.1, 0.2, 0.7], dtype=torch.float32) + fake_commit_for_reveal = b"mock_commit_for_reveal" + fake_reveal_round = 1 + + # Mocks + mocker.patch.object( + commit_reveal, + "convert_weights_and_uids_for_emit", + return_value=(fake_uids, fake_weights), + ) + mocker.patch.object( + commit_reveal, + "get_encrypted_commit", + return_value=(fake_commit_for_reveal, fake_reveal_round), + ) + mock_do_commit_reveal_v3 = mocker.patch.object( + commit_reveal, "_do_commit_reveal_v3", return_value=(False, "Failed") + ) + mocker.patch.object(subtensor, "get_current_block", return_value=1) + mocker.patch.object( + subtensor, + "get_subnet_hyperparameters", + return_value=hyperparams, + ) # Call - result = commit_reveal.commit_reveal_v3_extrinsic( - subtensor=fake_subtensor, + success, message = commit_reveal.commit_reveal_v3_extrinsic( + subtensor=subtensor, wallet=fake_wallet, - netuid=netuid, - uids=uids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization + netuid=fake_netuid, + uids=fake_uids, + weights=fake_weights, + wait_for_inclusion=True, + wait_for_finalization=True, ) # Asserts + assert success is False + assert message == "Failed" + mock_do_commit_reveal_v3.assert_called_once_with( + self=subtensor, + wallet=fake_wallet, + netuid=fake_netuid, + commit=fake_commit_for_reveal, + reveal_round=fake_reveal_round, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + +def test_commit_reveal_v3_extrinsic_exception(mocker, subtensor): + """Test exception handling in commit-reveal.""" + # Preps + fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_netuid = 1 + fake_uids = [1, 2, 3] + fake_weights = [0.1, 0.2, 0.7] - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_commit_reveal_v3_extrinsic.return_value, - event_loop=fake_subtensor.event_loop + mocker.patch.object( + commit_reveal, + "convert_weights_and_uids_for_emit", + side_effect=Exception("Test Error"), ) - mocked_commit_reveal_v3_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, + + # Call + success, message = commit_reveal.commit_reveal_v3_extrinsic( + subtensor=subtensor, wallet=fake_wallet, - netuid=netuid, - uids=uids, - weights=weights, - version_key=version_key, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization + netuid=fake_netuid, + uids=fake_uids, + weights=fake_weights, ) - assert result == mocked_execute_coroutine.return_value + + # Asserts + assert success is False + assert "Test Error" in message From ef50ff547dc300fb938d014a9ed27db9a04ee9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Thu, 23 Jan 2025 15:32:22 +0100 Subject: [PATCH 42/52] Revert "update `registration.py` unit test" This reverts commit 7ab41121cac14b5f46ecf01dd97b7518d7c5a059. --- .../extrinsics/test_registration.py | 318 ++++++++++++------ 1 file changed, 220 insertions(+), 98 deletions(-) diff --git a/tests/unit_tests/extrinsics/test_registration.py b/tests/unit_tests/extrinsics/test_registration.py index 2edc20b07d..18676619de 100644 --- a/tests/unit_tests/extrinsics/test_registration.py +++ b/tests/unit_tests/extrinsics/test_registration.py @@ -1,102 +1,224 @@ +# The MIT License (MIT) +# Copyright © 2024 Opentensor Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +import pytest +from bittensor_wallet import Wallet + from bittensor.core.extrinsics import registration +from bittensor.core.subtensor import Subtensor +from bittensor.utils.registration import POWSolution + + +# Mocking external dependencies +@pytest.fixture +def mock_subtensor(mocker): + mock = mocker.MagicMock(spec=Subtensor) + mock.network = "mock_network" + mock.substrate = mocker.MagicMock() + return mock + + +@pytest.fixture +def mock_wallet(mocker): + mock = mocker.MagicMock(spec=Wallet) + mock.coldkeypub.ss58_address = "mock_address" + mock.coldkey = mocker.MagicMock() + mock.hotkey = mocker.MagicMock() + mock.hotkey.ss58_address = "fake_ss58_address" + return mock + + +@pytest.fixture +def mock_pow_solution(mocker): + mock = mocker.MagicMock(spec=POWSolution) + mock.block_number = 123 + mock.nonce = 456 + mock.seal = [0, 1, 2, 3] + mock.is_stale.return_value = False + return mock + + +@pytest.fixture +def mock_new_wallet(mocker): + mock = mocker.MagicMock(spec=Wallet) + mock.coldkeypub.ss58_address = "mock_address" + mock.coldkey = mocker.MagicMock() + mock.hotkey = mocker.MagicMock() + return mock + + +@pytest.mark.parametrize( + "subnet_exists, neuron_is_null, cuda_available, expected_result, test_id", + [ + (False, True, True, False, "subnet-does-not-exist"), + (True, False, True, True, "neuron-already-registered"), + (True, True, False, False, "cuda-unavailable"), + ], +) +def test_register_extrinsic_without_pow( + mock_subtensor, + mock_wallet, + subnet_exists, + neuron_is_null, + cuda_available, + expected_result, + test_id, + mocker, +): + # Arrange + with ( + mocker.patch.object( + mock_subtensor, "subnet_exists", return_value=subnet_exists + ), + mocker.patch.object( + mock_subtensor, + "get_neuron_for_pubkey_and_subnet", + return_value=mocker.MagicMock(is_null=neuron_is_null), + ), + mocker.patch("torch.cuda.is_available", return_value=cuda_available), + mocker.patch( + "bittensor.utils.registration.pow._get_block_with_retry", + return_value=(0, 0, "00ff11ee"), + ), + ): + # Act + result = registration.register_extrinsic( + subtensor=mock_subtensor, + wallet=mock_wallet, + netuid=123, + wait_for_inclusion=True, + wait_for_finalization=True, + max_allowed_attempts=3, + output_in_place=True, + cuda=True, + dev_id=0, + tpb=256, + num_processes=None, + update_interval=None, + log_verbose=False, + ) + + # Assert + assert result == expected_result, f"Test failed for test_id: {test_id}" + + +@pytest.mark.parametrize( + "pow_success, pow_stale, registration_success, cuda, hotkey_registered, expected_result, test_id", + [ + (True, False, True, False, False, True, "successful-with-valid-pow"), + (True, False, True, True, False, True, "successful-with-valid-cuda-pow"), + # Pow failed but key was registered already + (False, False, False, False, True, True, "hotkey-registered"), + # Pow was a success but registration failed with error 'key already registered' + (True, False, False, False, False, True, "registration-fail-key-registered"), + ], +) +def test_register_extrinsic_with_pow( + mock_subtensor, + mock_wallet, + mock_pow_solution, + pow_success, + pow_stale, + registration_success, + cuda, + hotkey_registered, + expected_result, + test_id, + mocker, +): + # Arrange + with mocker.patch( + "bittensor.utils.registration.pow._solve_for_difficulty_fast", + return_value=mock_pow_solution if pow_success else None, + ), mocker.patch( + "bittensor.utils.registration.pow._solve_for_difficulty_fast_cuda", + return_value=mock_pow_solution if pow_success else None, + ), mocker.patch( + "bittensor.core.extrinsics.registration._do_pow_register", + return_value=(registration_success, "HotKeyAlreadyRegisteredInSubNet"), + ), mocker.patch("torch.cuda.is_available", return_value=cuda): + # Act + if pow_success: + mock_pow_solution.is_stale.return_value = pow_stale + + if not pow_success and hotkey_registered: + mock_subtensor.is_hotkey_registered = mocker.MagicMock( + return_value=hotkey_registered + ) + + result = registration.register_extrinsic( + subtensor=mock_subtensor, + wallet=mock_wallet, + netuid=123, + wait_for_inclusion=True, + wait_for_finalization=True, + max_allowed_attempts=3, + output_in_place=True, + cuda=cuda, + dev_id=0, + tpb=256, + num_processes=None, + update_interval=None, + log_verbose=False, + ) + # Assert + assert result == expected_result, f"Test failed for test_id: {test_id}." -def test_burned_register_extrinsic(mocker): - """"Verify that sync `burned_register_extrinsic` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - netuid = 1 - wait_for_inclusion = True - wait_for_finalization = True - - mocked_execute_coroutine = mocker.patch.object(registration, "execute_coroutine") - mocked_burned_register_extrinsic = mocker.Mock() - registration.async_burned_register_extrinsic = mocked_burned_register_extrinsic - - # Call - result = registration.burned_register_extrinsic( - subtensor=fake_subtensor, - wallet=fake_wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization - ) - - # Asserts - - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_burned_register_extrinsic.return_value, - event_loop=fake_subtensor.event_loop - ) - mocked_burned_register_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization - ) - assert result == mocked_execute_coroutine.return_value - - -def test_register_extrinsic(mocker): - """"Verify that sync `register_extrinsic` method calls proper async method.""" - # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() - netuid = 1 - wait_for_inclusion = True - wait_for_finalization = True - max_allowed_attempts = 7 - output_in_place = True - cuda = True - dev_id = 5 - tpb = 12 - num_processes = 8 - update_interval = 2 - log_verbose = True - - mocked_execute_coroutine = mocker.patch.object(registration, "execute_coroutine") - mocked_register_extrinsic = mocker.Mock() - registration.async_register_extrinsic = mocked_register_extrinsic - - # Call - result = registration.register_extrinsic( - subtensor=fake_subtensor, - wallet=fake_wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - max_allowed_attempts=max_allowed_attempts, - output_in_place=output_in_place, - cuda=cuda, - dev_id=dev_id, - tpb=tpb, - num_processes=num_processes, - update_interval=update_interval, - log_verbose=log_verbose - ) - - # Asserts - - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_register_extrinsic.return_value, - event_loop=fake_subtensor.event_loop - ) - mocked_register_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - netuid=netuid, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, - max_allowed_attempts=max_allowed_attempts, - output_in_place=output_in_place, - cuda=cuda, - dev_id=dev_id, - tpb=tpb, - num_processes=num_processes, - update_interval=update_interval, - log_verbose=log_verbose - ) - assert result == mocked_execute_coroutine.return_value +@pytest.mark.parametrize( + "subnet_exists, neuron_is_null, recycle_success, is_registered, expected_result, test_id", + [ + # Happy paths + (True, False, None, None, True, "neuron-not-null"), + (True, True, True, True, True, "happy-path-wallet-registered"), + # Error paths + (False, True, False, None, False, "subnet-non-existence"), + (True, True, False, False, False, "error-path-recycling-failed"), + (True, True, True, False, False, "error-path-not-registered"), + ], +) +def test_burned_register_extrinsic( + mock_subtensor, + mock_wallet, + subnet_exists, + neuron_is_null, + recycle_success, + is_registered, + expected_result, + test_id, + mocker, +): + # Arrange + with mocker.patch.object( + mock_subtensor, "subnet_exists", return_value=subnet_exists + ), mocker.patch.object( + mock_subtensor, + "get_neuron_for_pubkey_and_subnet", + return_value=mocker.MagicMock(is_null=neuron_is_null), + ), mocker.patch( + "bittensor.core.extrinsics.registration._do_burned_register", + return_value=(recycle_success, "Mock error message"), + ), mocker.patch.object( + mock_subtensor, "is_hotkey_registered", return_value=is_registered + ): + # Act + result = registration.burned_register_extrinsic( + subtensor=mock_subtensor, wallet=mock_wallet, netuid=123 + ) + # Assert + assert result == expected_result, f"Test failed for test_id: {test_id}" From b9e70f088a632a6414e80ee47b7a0f212d2d5dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Fri, 24 Jan 2025 19:33:25 +0100 Subject: [PATCH 43/52] test: fix old (sync subtensor) unittests --- bittensor/core/extrinsics/registration.py | 4 +- .../extrinsics/test_commit_reveal.py | 44 ++++++++-------- .../extrinsics/test_commit_weights.py | 51 ++++++++++++------- tests/unit_tests/extrinsics/test_serving.py | 4 +- .../unit_tests/extrinsics/test_set_weights.py | 24 ++++----- tests/unit_tests/extrinsics/test_transfer.py | 33 ++++++------ 6 files changed, 87 insertions(+), 73 deletions(-) diff --git a/bittensor/core/extrinsics/registration.py b/bittensor/core/extrinsics/registration.py index eef316f387..d5211ee199 100644 --- a/bittensor/core/extrinsics/registration.py +++ b/bittensor/core/extrinsics/registration.py @@ -11,12 +11,12 @@ from bittensor.utils import unlock_key from bittensor.utils.btlogging import logging -from bittensor.utils.registration import log_no_torch_error, torch +from bittensor.utils.registration import create_pow, log_no_torch_error, torch if TYPE_CHECKING: from bittensor_wallet import Wallet from bittensor.core.subtensor import Subtensor - from bittensor.utils.registration.pow import POWSolution, create_pow + from bittensor.utils.registration.pow import POWSolution def _do_burned_register( diff --git a/tests/unit_tests/extrinsics/test_commit_reveal.py b/tests/unit_tests/extrinsics/test_commit_reveal.py index e1f7c9f877..c0beedc6ab 100644 --- a/tests/unit_tests/extrinsics/test_commit_reveal.py +++ b/tests/unit_tests/extrinsics/test_commit_reveal.py @@ -1,10 +1,12 @@ +import numpy as np +import pytest +import torch +from bittensor_wallet import Wallet + from bittensor.core import subtensor as subtensor_module from bittensor.core.chain_data import SubnetHyperparameters -from bittensor.core.subtensor import Subtensor from bittensor.core.extrinsics import commit_reveal -import pytest -import torch -import numpy as np +from bittensor.core.subtensor import Subtensor @pytest.fixture @@ -53,7 +55,7 @@ def hyperparams(): def test_do_commit_reveal_v3_success(mocker, subtensor): """Test successful commit-reveal with wait for finalization.""" # Preps - fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_wallet = mocker.Mock(autospec=Wallet) fake_netuid = 1 fake_commit = b"fake_commit" fake_reveal_round = 1 @@ -62,11 +64,11 @@ def test_do_commit_reveal_v3_success(mocker, subtensor): mocked_create_signed_extrinsic = mocker.patch.object( subtensor.substrate, "create_signed_extrinsic" ) - mocked_submit_extrinsic = mocker.patch.object(commit_reveal, "submit_extrinsic") + mocked_submit_extrinsic = mocker.patch.object(subtensor.substrate, "submit_extrinsic") # Call result = commit_reveal._do_commit_reveal_v3( - self=subtensor, + subtensor=subtensor, wallet=fake_wallet, netuid=fake_netuid, commit=fake_commit, @@ -87,18 +89,17 @@ def test_do_commit_reveal_v3_success(mocker, subtensor): call=mocked_compose_call.return_value, keypair=fake_wallet.hotkey ) mocked_submit_extrinsic.assert_called_once_with( - subtensor=subtensor, - extrinsic=mocked_create_signed_extrinsic.return_value, + mocked_create_signed_extrinsic.return_value, wait_for_inclusion=False, wait_for_finalization=False, ) - assert result == (True, "Not waiting for finalization or inclusion.") + assert result == (True, "") def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): """Test commit-reveal fails due to an error in submission.""" # Preps - fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_wallet = mocker.Mock(autospec=Wallet) fake_netuid = 1 fake_commit = b"fake_commit" fake_reveal_round = 1 @@ -108,17 +109,17 @@ def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): subtensor.substrate, "create_signed_extrinsic" ) mocked_submit_extrinsic = mocker.patch.object( - commit_reveal, + subtensor.substrate, "submit_extrinsic", return_value=mocker.Mock(is_success=False, error_message="Mocked error"), ) mocked_format_error_message = mocker.patch.object( - commit_reveal, "format_error_message", return_value="Formatted error" + subtensor_module, "format_error_message", return_value="Formatted error" ) # Call result = commit_reveal._do_commit_reveal_v3( - self=subtensor, + subtensor=subtensor, wallet=fake_wallet, netuid=fake_netuid, commit=fake_commit, @@ -141,8 +142,7 @@ def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): call=mocked_compose_call.return_value, keypair=fake_wallet.hotkey ) mocked_submit_extrinsic.assert_called_once_with( - subtensor=subtensor, - extrinsic=mocked_create_signed_extrinsic.return_value, + mocked_create_signed_extrinsic.return_value, wait_for_inclusion=True, wait_for_finalization=True, ) @@ -153,7 +153,7 @@ def test_do_commit_reveal_v3_failure_due_to_error(mocker, subtensor): def test_commit_reveal_v3_extrinsic_success_with_torch(mocker, subtensor, hyperparams): """Test successful commit-reveal with torch tensors.""" # Preps - fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_wallet = mocker.Mock(autospec=Wallet) fake_netuid = 1 fake_uids = torch.tensor([1, 2, 3], dtype=torch.int64) fake_weights = torch.tensor([0.1, 0.2, 0.7], dtype=torch.float32) @@ -214,7 +214,7 @@ def test_commit_reveal_v3_extrinsic_success_with_torch(mocker, subtensor, hyperp current_block=mock_block.return_value, ) mock_do_commit_reveal_v3.assert_called_once_with( - self=subtensor, + subtensor=subtensor, wallet=fake_wallet, netuid=fake_netuid, commit=fake_commit_for_reveal, @@ -227,7 +227,7 @@ def test_commit_reveal_v3_extrinsic_success_with_torch(mocker, subtensor, hyperp def test_commit_reveal_v3_extrinsic_success_with_numpy(mocker, subtensor, hyperparams): """Test successful commit-reveal with numpy arrays.""" # Preps - fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_wallet = mocker.Mock(autospec=Wallet) fake_netuid = 1 fake_uids = np.array([1, 2, 3], dtype=np.int64) fake_weights = np.array([0.1, 0.2, 0.7], dtype=np.float32) @@ -272,7 +272,7 @@ def test_commit_reveal_v3_extrinsic_success_with_numpy(mocker, subtensor, hyperp def test_commit_reveal_v3_extrinsic_response_false(mocker, subtensor, hyperparams): """Test unsuccessful commit-reveal with torch.""" # Preps - fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_wallet = mocker.Mock(autospec=Wallet) fake_netuid = 1 fake_uids = torch.tensor([1, 2, 3], dtype=torch.int64) fake_weights = torch.tensor([0.1, 0.2, 0.7], dtype=torch.float32) @@ -315,7 +315,7 @@ def test_commit_reveal_v3_extrinsic_response_false(mocker, subtensor, hyperparam assert success is False assert message == "Failed" mock_do_commit_reveal_v3.assert_called_once_with( - self=subtensor, + subtensor=subtensor, wallet=fake_wallet, netuid=fake_netuid, commit=fake_commit_for_reveal, @@ -328,7 +328,7 @@ def test_commit_reveal_v3_extrinsic_response_false(mocker, subtensor, hyperparam def test_commit_reveal_v3_extrinsic_exception(mocker, subtensor): """Test exception handling in commit-reveal.""" # Preps - fake_wallet = mocker.Mock(autospec=subtensor_module.Wallet) + fake_wallet = mocker.Mock(autospec=Wallet) fake_netuid = 1 fake_uids = [1, 2, 3] fake_weights = [0.1, 0.2, 0.7] diff --git a/tests/unit_tests/extrinsics/test_commit_weights.py b/tests/unit_tests/extrinsics/test_commit_weights.py index 57d78a8013..42cc1f2311 100644 --- a/tests/unit_tests/extrinsics/test_commit_weights.py +++ b/tests/unit_tests/extrinsics/test_commit_weights.py @@ -1,11 +1,12 @@ import pytest +from bittensor_wallet import Wallet from bittensor.core import subtensor as subtensor_module from bittensor.core.settings import version_as_int from bittensor.core.subtensor import Subtensor from bittensor.core.extrinsics.commit_weights import ( - do_commit_weights, - do_reveal_weights, + _do_commit_weights, + _do_reveal_weights, ) @@ -30,12 +31,15 @@ def test_do_commit_weights(subtensor, mocker): subtensor.substrate.submit_extrinsic.return_value.is_success = None - mocked_format_error_message = mocker.MagicMock() - subtensor_module.format_error_message = mocked_format_error_message + mocked_format_error_message = mocker.Mock() + mocker.patch( + "bittensor.core.extrinsics.commit_weights.format_error_message", + mocked_format_error_message, + ) # Call - result = do_commit_weights( - self=subtensor, + result = _do_commit_weights( + subtensor=subtensor, wallet=fake_wallet, netuid=netuid, commit_hash=commit_hash, @@ -59,24 +63,26 @@ def test_do_commit_weights(subtensor, mocker): assert kwargs["keypair"] == fake_wallet.hotkey subtensor.substrate.submit_extrinsic.assert_called_once_with( - subtensor.substrate.create_signed_extrinsic.return_value, + extrinsic=subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + mocked_format_error_message.assert_called_once_with( + subtensor.substrate.submit_extrinsic.return_value.error_message, + ) assert result == ( False, - subtensor.substrate.submit_extrinsic.return_value.error_message, + mocked_format_error_message.return_value, ) def test_do_reveal_weights(subtensor, mocker): """Verifies that the `_do_reveal_weights` method interacts with the right substrate methods.""" # Preps - fake_wallet = mocker.MagicMock() - fake_wallet.hotkey = "hotkey" + fake_wallet = mocker.MagicMock(autospec=Wallet) + fake_wallet.hotkey.ss58_address = "hotkey" netuid = 1 uids = [1, 2, 3, 4] @@ -87,12 +93,15 @@ def test_do_reveal_weights(subtensor, mocker): subtensor.substrate.submit_extrinsic.return_value.is_success = None - mocked_format_error_message = mocker.MagicMock() - subtensor_module.format_error_message = mocked_format_error_message + mocked_format_error_message = mocker.Mock() + mocker.patch( + "bittensor.core.extrinsics.commit_weights.format_error_message", + mocked_format_error_message, + ) # Call - result = do_reveal_weights( - self=subtensor, + result = _do_reveal_weights( + subtensor=subtensor, wallet=fake_wallet, netuid=netuid, uids=uids, @@ -117,18 +126,22 @@ def test_do_reveal_weights(subtensor, mocker): ) subtensor.substrate.create_signed_extrinsic.assert_called_once_with( - call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.hotkey + call=subtensor.substrate.compose_call.return_value, + keypair=fake_wallet.hotkey, + nonce=subtensor.substrate.get_account_next_index.return_value, ) subtensor.substrate.submit_extrinsic.assert_called_once_with( - subtensor.substrate.create_signed_extrinsic.return_value, + extrinsic=subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) - subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + mocked_format_error_message.assert_called_once_with( + subtensor.substrate.submit_extrinsic.return_value.error_message, + ) assert result == ( False, - subtensor.substrate.submit_extrinsic.return_value.error_message, + mocked_format_error_message.return_value, ) diff --git a/tests/unit_tests/extrinsics/test_serving.py b/tests/unit_tests/extrinsics/test_serving.py index 46eef17888..6d00e97629 100644 --- a/tests/unit_tests/extrinsics/test_serving.py +++ b/tests/unit_tests/extrinsics/test_serving.py @@ -291,7 +291,7 @@ def test_serve_axon_extrinsic( serving.do_serve_axon = mocker.MagicMock(return_value=(serve_success, "")) # Act if not external_ip_success: - with pytest.raises(RuntimeError): + with pytest.raises(ConnectionError): serving.serve_axon_extrinsic( mock_subtensor, netuid, @@ -364,7 +364,7 @@ def test_publish_metadata( ): # Act result = serving.publish_metadata( - self=mock_subtensor, + subtensor=mock_subtensor, wallet=mock_wallet, netuid=net_uid, data_type=type_u, diff --git a/tests/unit_tests/extrinsics/test_set_weights.py b/tests/unit_tests/extrinsics/test_set_weights.py index 6c070bf5c4..a2aaa4aaab 100644 --- a/tests/unit_tests/extrinsics/test_set_weights.py +++ b/tests/unit_tests/extrinsics/test_set_weights.py @@ -6,7 +6,7 @@ from bittensor.core import subtensor as subtensor_module from bittensor.core.extrinsics.set_weights import ( - do_set_weights, + _do_set_weights, set_weights_extrinsic, ) from bittensor.core.settings import version_as_int @@ -81,9 +81,9 @@ def test_set_weights_extrinsic( "bittensor.utils.weight_utils.convert_weights_and_uids_for_emit", return_value=(uids_tensor, weights_tensor), ), patch( - "bittensor.core.extrinsics.set_weights.do_set_weights", + "bittensor.core.extrinsics.set_weights._do_set_weights", return_value=(expected_success, "Mock error message"), - ) as mock_do_set_weights: + ): result, message = set_weights_extrinsic( subtensor=mock_subtensor, wallet=mock_wallet, @@ -112,8 +112,8 @@ def test_do_set_weights_is_success(mock_subtensor, mocker): mock_subtensor.substrate.submit_extrinsic.return_value.is_success = True # Call - result = do_set_weights( - self=mock_subtensor, + result = _do_set_weights( + subtensor=mock_subtensor, wallet=fake_wallet, uids=fake_uids, vals=fake_vals, @@ -141,7 +141,6 @@ def test_do_set_weights_is_success(mock_subtensor, mocker): assert kwargs["keypair"] == fake_wallet.hotkey assert kwargs["era"] == {"period": 5} - mock_subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() assert result == (True, "Successfully set weights.") @@ -160,8 +159,8 @@ def test_do_set_weights_is_not_success(mock_subtensor, mocker): subtensor_module.format_error_message = mocked_format_error_message # Call - result = do_set_weights( - self=mock_subtensor, + result = _do_set_weights( + subtensor=mock_subtensor, wallet=fake_wallet, uids=fake_uids, vals=fake_vals, @@ -190,12 +189,11 @@ def test_do_set_weights_is_not_success(mock_subtensor, mocker): assert kwargs["era"] == {"period": 5} mock_subtensor.substrate.submit_extrinsic.assert_called_once_with( - mock_subtensor.substrate.create_signed_extrinsic.return_value, + extrinsic=mock_subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=fake_wait_for_inclusion, wait_for_finalization=fake_wait_for_finalization, ) - mock_subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() assert result == ( False, "Subtensor returned `UnknownError(UnknownType)` error. This means: `Unknown Description`.", @@ -213,8 +211,8 @@ def test_do_set_weights_no_waits(mock_subtensor, mocker): fake_wait_for_finalization = False # Call - result = do_set_weights( - self=mock_subtensor, + result = _do_set_weights( + subtensor=mock_subtensor, wallet=fake_wallet, uids=fake_uids, vals=fake_vals, @@ -243,7 +241,7 @@ def test_do_set_weights_no_waits(mock_subtensor, mocker): assert kwargs["era"] == {"period": 5} mock_subtensor.substrate.submit_extrinsic.assert_called_once_with( - mock_subtensor.substrate.create_signed_extrinsic.return_value, + extrinsic=mock_subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=fake_wait_for_inclusion, wait_for_finalization=fake_wait_for_finalization, ) diff --git a/tests/unit_tests/extrinsics/test_transfer.py b/tests/unit_tests/extrinsics/test_transfer.py index af59d5769b..b5cb97483d 100644 --- a/tests/unit_tests/extrinsics/test_transfer.py +++ b/tests/unit_tests/extrinsics/test_transfer.py @@ -1,7 +1,7 @@ import pytest from bittensor.core import subtensor as subtensor_module -from bittensor.core.extrinsics.transfer import do_transfer +from bittensor.core.extrinsics.transfer import _do_transfer from bittensor.core.subtensor import Subtensor from bittensor.utils.balance import Balance @@ -28,7 +28,7 @@ def test_do_transfer_is_success_true(subtensor, mocker): subtensor.substrate.submit_extrinsic.return_value.is_success = True # Call - result = do_transfer( + result = _do_transfer( subtensor, fake_wallet, fake_dest, @@ -47,15 +47,15 @@ def test_do_transfer_is_success_true(subtensor, mocker): call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.coldkey ) subtensor.substrate.submit_extrinsic.assert_called_once_with( - subtensor.substrate.create_signed_extrinsic.return_value, + extrinsic=subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=fake_wait_for_inclusion, wait_for_finalization=fake_wait_for_finalization, ) - subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + # subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() assert result == ( True, subtensor.substrate.submit_extrinsic.return_value.block_hash, - None, + "Success with response.", ) @@ -70,11 +70,14 @@ def test_do_transfer_is_success_false(subtensor, mocker): subtensor.substrate.submit_extrinsic.return_value.is_success = False - mocked_format_error_message = mocker.MagicMock() - subtensor_module.format_error_message = mocked_format_error_message + mocked_format_error_message = mocker.Mock() + mocker.patch( + "bittensor.core.extrinsics.transfer.format_error_message", + mocked_format_error_message, + ) # Call - result = do_transfer( + result = _do_transfer( subtensor, fake_wallet, fake_dest, @@ -93,16 +96,16 @@ def test_do_transfer_is_success_false(subtensor, mocker): call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.coldkey ) subtensor.substrate.submit_extrinsic.assert_called_once_with( - subtensor.substrate.create_signed_extrinsic.return_value, + extrinsic=subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=fake_wait_for_inclusion, wait_for_finalization=fake_wait_for_finalization, ) - subtensor.substrate.submit_extrinsic.return_value.process_events.assert_called_once() + mocked_format_error_message.assert_called_once_with(subtensor.substrate.submit_extrinsic.return_value.error_message) assert result == ( False, - None, - subtensor.substrate.submit_extrinsic.return_value.error_message, + "", + mocked_format_error_message.return_value, ) @@ -116,7 +119,7 @@ def test_do_transfer_no_waits(subtensor, mocker): fake_wait_for_finalization = False # Call - result = do_transfer( + result = _do_transfer( subtensor, fake_wallet, fake_dest, @@ -135,8 +138,8 @@ def test_do_transfer_no_waits(subtensor, mocker): call=subtensor.substrate.compose_call.return_value, keypair=fake_wallet.coldkey ) subtensor.substrate.submit_extrinsic.assert_called_once_with( - subtensor.substrate.create_signed_extrinsic.return_value, + extrinsic=subtensor.substrate.create_signed_extrinsic.return_value, wait_for_inclusion=fake_wait_for_inclusion, wait_for_finalization=fake_wait_for_finalization, ) - assert result == (True, None, None) + assert result == (True, "", "Success, extrinsic submitted without waiting.") From 2b747bc80fbedb4d12c5599d58ebbc5762afaac1 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 24 Jan 2025 10:53:12 -0800 Subject: [PATCH 44/52] fix `tests/integration_tests/test_metagraph_integration.py` --- tests/integration_tests/test_metagraph_integration.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration_tests/test_metagraph_integration.py b/tests/integration_tests/test_metagraph_integration.py index 45ce51a6b8..3344ab6ae4 100644 --- a/tests/integration_tests/test_metagraph_integration.py +++ b/tests/integration_tests/test_metagraph_integration.py @@ -33,18 +33,14 @@ def test_sync_block_0(self): self.metagraph.sync(lite=True, block=0, subtensor=self.sub) def test_load_sync_save(self): - with mock.patch.object( - self.sub.async_subtensor, "neurons_lite", return_value=[] - ): + with mock.patch.object(self.sub, "neurons_lite", return_value=[]): self.metagraph.sync(lite=True, subtensor=self.sub) self.metagraph.save() self.metagraph.load() self.metagraph.save() def test_load_sync_save_from_torch(self): - with mock.patch.object( - self.sub.async_subtensor, "neurons_lite", return_value=[] - ): + with mock.patch.object(self.sub, "neurons_lite", return_value=[]): self.metagraph.sync(lite=True, subtensor=self.sub) def deprecated_save_torch(metagraph): From f546cd81e1a0ba0b7309d553daaa02b44f652a2d Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 24 Jan 2025 12:38:52 -0800 Subject: [PATCH 45/52] add `metagraph[chain_getBlockHash]` result --- tests/helpers/integration_websocket_data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/helpers/integration_websocket_data.py b/tests/helpers/integration_websocket_data.py index 6bd2e926e5..340072ee96 100644 --- a/tests/helpers/integration_websocket_data.py +++ b/tests/helpers/integration_websocket_data.py @@ -6423,6 +6423,12 @@ } }, "system_chain": {"[]": {"jsonrpc": "2.0", "result": "Bittensor"}}, + "chain_getBlockHash": { + "[3264143]": { + "jsonrpc": "2.0", + "result": None, + } + }, }, "min_allowed_weights": { "chain_getHead": { From ababac2a4676fba4e48704e27a04549cf6a68d27 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 24 Jan 2025 12:39:16 -0800 Subject: [PATCH 46/52] fix `tests/integration_tests/test_subtensor_integration.py` --- .../test_subtensor_integration.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index dcf149e62e..0b8763dcd5 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -1,14 +1,14 @@ -import asyncio import os.path import pytest -from bittensor.utils.balance import Balance -from bittensor.core.chain_data.axon_info import AxonInfo +from async_substrate_interface import sync_substrate +from bt_decode import PortableRegistry, MetadataV15 from bittensor import NeuronInfo +from bittensor.core.chain_data.axon_info import AxonInfo from bittensor.core.subtensor import Subtensor -from bt_decode import PortableRegistry, MetadataV15 -from tests.helpers.helpers import FakeWebsocket +from bittensor.utils.balance import Balance +from tests.helpers.helpers import FakeConnectContextManager @pytest.fixture @@ -32,12 +32,8 @@ async def prepare_test(mocker, seed): MetadataV15.decode_from_metadata_option(f.read()) ) subtensor = Subtensor("unknown", _mock=True) - mocker.patch.object(subtensor.substrate.ws, "ws", FakeWebsocket(seed=seed)) - mocker.patch.object(subtensor.substrate.ws, "_initialized", True) - mocker.patch.object(subtensor.substrate._async_instance, "registry", registry) - subtensor.substrate.ws._receiving_task = asyncio.create_task( - subtensor.substrate.ws._start_receiving() - ) + mocker.patch.object(sync_substrate, "connect", mocker.Mock(return_value=FakeConnectContextManager(seed=seed))) + mocker.patch.object(subtensor.substrate, "registry", registry) return subtensor @@ -55,6 +51,7 @@ async def test_get_all_subnets_info(mocker): @pytest.mark.asyncio async def test_metagraph(mocker): subtensor = await prepare_test(mocker, "metagraph") + # with mocker.patch.object(subtensor, "get_block_hash", return_value={"n": 19}): result = subtensor.metagraph(23) assert result.n == 19 assert result.netuid == 23 From e841b411e6d1cdd34b2a9591b57528b7b46699b6 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 24 Jan 2025 12:39:42 -0800 Subject: [PATCH 47/52] improve helper to use with sync integration tests --- tests/helpers/helpers.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/helpers/helpers.py b/tests/helpers/helpers.py index a6e9f292df..da5cabb1d2 100644 --- a/tests/helpers/helpers.py +++ b/tests/helpers/helpers.py @@ -1,18 +1,18 @@ import asyncio -from collections import deque import json +import time +from collections import deque from typing import Union -from websockets.asyncio.client import ClientConnection, ClientProtocol -from websockets.uri import parse_uri - from bittensor_wallet.mock.wallet_mock import MockWallet as _MockWallet from bittensor_wallet.mock.wallet_mock import get_mock_coldkey from bittensor_wallet.mock.wallet_mock import get_mock_hotkey from bittensor_wallet.mock.wallet_mock import get_mock_wallet +from websockets.asyncio.client import ClientConnection, ClientProtocol +from websockets.uri import parse_uri -from bittensor.utils.balance import Balance from bittensor.core.chain_data import AxonInfo, NeuronInfo, PrometheusInfo +from bittensor.utils.balance import Balance from tests.helpers.integration_websocket_data import WEBSOCKET_RESPONSES, METADATA @@ -118,17 +118,15 @@ def __init__(self, *args, seed, **kwargs): self.received = deque() self._lock = asyncio.Lock() - async def send(self, payload: str, *args, **kwargs): + def send(self, payload: str, *args, **kwargs): received = json.loads(payload) id_ = received.pop("id") - async with self._lock: - self.received.append((received, id_)) + self.received.append((received, id_)) - async def recv(self, *args, **kwargs): + def recv(self, *args, **kwargs): while len(self.received) == 0: - await asyncio.sleep(0.1) - async with self._lock: - item, _id = self.received.pop() + time.sleep(0.1) + item, _id = self.received.pop() try: if item["method"] == "state_getMetadata": response = {"jsonrpc": "2.0", "id": _id, "result": METADATA} @@ -142,5 +140,17 @@ async def recv(self, *args, **kwargs): print("ERROR", self.seed, item["method"], item["params"]) raise - async def close(self, *args, **kwargs): + def close(self, *args, **kwargs): + pass + + +class FakeConnectContextManager: + + def __init__(self, seed): + self.seed = seed + + def __enter__(self): + return FakeWebsocket(seed=self.seed) + + def __exit__(self, exc_type, exc, tb): pass From b0eba572c38be0615c4d52d61de63dcc11658689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Fri, 24 Jan 2025 21:42:03 +0100 Subject: [PATCH 48/52] test: fix extrinsics unittests --- bittensor/core/extrinsics/staking.py | 4 +- tests/unit_tests/extrinsics/test_root.py | 55 +++++---- tests/unit_tests/extrinsics/test_staking.py | 116 +++++++++++++----- tests/unit_tests/extrinsics/test_unstaking.py | 89 +++++++++----- 4 files changed, 169 insertions(+), 95 deletions(-) diff --git a/bittensor/core/extrinsics/staking.py b/bittensor/core/extrinsics/staking.py index 9e85613bb4..9b35cda557 100644 --- a/bittensor/core/extrinsics/staking.py +++ b/bittensor/core/extrinsics/staking.py @@ -93,8 +93,10 @@ def _check_threshold_amount( if amount is None: # Stake it all. staking_balance = Balance.from_tao(old_balance.tao) + elif not isinstance(amount, Balance): + staking_balance = Balance.from_tao(amount) else: - staking_balance = Balance.from_tao(amount.tao) + staking_balance = amount # Leave existential balance to keep key alive. if staking_balance > old_balance - existential_deposit: diff --git a/tests/unit_tests/extrinsics/test_root.py b/tests/unit_tests/extrinsics/test_root.py index 96d90fe09a..3150de2400 100644 --- a/tests/unit_tests/extrinsics/test_root.py +++ b/tests/unit_tests/extrinsics/test_root.py @@ -7,6 +7,7 @@ def mock_subtensor(mocker): mock = mocker.MagicMock(spec=Subtensor) mock.network = "magic_mock" + mock.substrate = mocker.Mock() return mock @@ -30,17 +31,17 @@ def mock_wallet(mocker): ( False, True, - [False, True], + [False, 1], True, True, ), # Registration succeeds with user confirmation - (False, True, [False, False], False, None), # Registration fails + (False, True, [False, None], False, False), # Registration fails ( False, True, - [False, False], + [False, None], True, - None, + False, ), # Registration succeeds but neuron not found ], ids=[ @@ -61,13 +62,19 @@ def test_root_register_extrinsic( mocker, ): # Arrange - mock_subtensor.is_hotkey_registered.side_effect = hotkey_registered + mock_subtensor.is_hotkey_registered.return_value = hotkey_registered[0] # Preps - mock_register = mocker.Mock( + mocked_sign_and_send_extrinsic = mocker.patch.object( + mock_subtensor, + "sign_and_send_extrinsic", return_value=(registration_success, "Error registering") ) - root._do_root_register = mock_register + mocker.patch.object( + mock_subtensor.substrate, + "query", + return_value=hotkey_registered[1], + ) # Act result = root.root_register_extrinsic( @@ -80,7 +87,17 @@ def test_root_register_extrinsic( assert result == expected_result if not hotkey_registered[0]: - mock_register.assert_called_once() + mock_subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="root_register", + call_params={"hotkey": "fake_hotkey_address"}, + ) + mocked_sign_and_send_extrinsic.assert_called_once_with( + mock_subtensor.substrate.compose_call.return_value, + wallet=mock_wallet, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization + ) @pytest.mark.parametrize( @@ -115,13 +132,6 @@ def test_root_register_extrinsic( [0.5, 0.5], False, ), # Failure - setting weights failed - ( - True, - False, - [], - [], - False, - ), # Exception catched - ValueError 'min() arg is an empty sequence' ], ids=[ "success-weights-set", @@ -129,7 +139,6 @@ def test_root_register_extrinsic( "success-large-value", "success-single-value", "failure-setting-weights", - "failure-value-error-exception", ], ) def test_set_root_weights_extrinsic( @@ -146,8 +155,9 @@ def test_set_root_weights_extrinsic( root._do_set_root_weights = mocker.Mock( return_value=(expected_success, "Mock error") ) - mock_subtensor.min_allowed_weights = mocker.Mock(return_value=0) - mock_subtensor.max_weight_limit = mocker.Mock(return_value=1) + root._get_limits = mocker.Mock( + return_value=(0, 1), + ) # Call result = root.set_root_weights_extrinsic( @@ -200,14 +210,6 @@ def test_set_root_weights_extrinsic( None, False, ), # Failure - setting weights failed - ( - True, - False, - [], - [], - False, - False, - ), # Exception catched - ValueError 'min() arg is an empty sequence' ], ids=[ "success-weights-set", @@ -215,7 +217,6 @@ def test_set_root_weights_extrinsic( "success-large-value", "success-single-value", "failure-setting-weights", - "failure-value-error-exception", ], ) def test_set_root_weights_extrinsic_torch( diff --git a/tests/unit_tests/extrinsics/test_staking.py b/tests/unit_tests/extrinsics/test_staking.py index d30d225ebd..b6fc9cb38f 100644 --- a/tests/unit_tests/extrinsics/test_staking.py +++ b/tests/unit_tests/extrinsics/test_staking.py @@ -1,20 +1,28 @@ from bittensor.core.extrinsics import staking +from bittensor.utils.balance import Balance def test_add_stake_extrinsic(mocker): """Verify that sync `add_stake_extrinsic` method calls proper async method.""" # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() + fake_subtensor = mocker.Mock( + **{ + "get_balance.return_value": Balance(10), + "get_existential_deposit.return_value": Balance(1), + "get_hotkey_owner.return_value": "hotkey_owner", + "sign_and_send_extrinsic.return_value": (True, ""), + } + ) + fake_wallet = mocker.Mock( + **{ + "coldkeypub.ss58_address": "hotkey_owner", + } + ) hotkey_ss58 = "hotkey" amount = 1.1 wait_for_inclusion = True wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(staking, "execute_coroutine") - mocked_add_stake_extrinsic = mocker.Mock() - staking.async_add_stake_extrinsic = mocked_add_stake_extrinsic - # Call result = staking.add_stake_extrinsic( subtensor=fake_subtensor, @@ -26,35 +34,62 @@ def test_add_stake_extrinsic(mocker): ) # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_add_stake_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + assert result is True + + fake_subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="add_stake", + call_params={ + "hotkey": "hotkey", + "amount_staked": 9, + }, ) - mocked_add_stake_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - hotkey_ss58=hotkey_ss58, - amount=amount, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + fake_subtensor.sign_and_send_extrinsic.assert_called_once_with( + fake_subtensor.substrate.compose_call.return_value, + fake_wallet, + True, + True, ) - assert result == mocked_execute_coroutine.return_value def test_add_stake_multiple_extrinsic(mocker): """Verify that sync `add_stake_multiple_extrinsic` method calls proper async method.""" # Preps - fake_subtensor = mocker.Mock() - fake_wallet = mocker.Mock() + fake_subtensor = mocker.Mock( + **{ + "get_balance.return_value": Balance(10.0), + "sign_and_send_extrinsic.return_value": (True, ""), + "substrate.query_multi.return_value": [ + ( + mocker.Mock( + **{ + "params": ["hotkey1"], + }, + ), + 0, + ), + ( + mocker.Mock( + **{ + "params": ["hotkey2"], + }, + ), + 0, + ), + ], + "substrate.query.return_value": 0, + } + ) + fake_wallet = mocker.Mock( + **{ + "coldkeypub.ss58_address": "hotkey_owner", + } + ) hotkey_ss58s = ["hotkey1", "hotkey2"] amounts = [1.1, 2.2] wait_for_inclusion = True wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(staking, "execute_coroutine") - mocked_add_stake_multiple_extrinsic = mocker.Mock() - staking.async_add_stake_multiple_extrinsic = mocked_add_stake_multiple_extrinsic - # Call result = staking.add_stake_multiple_extrinsic( subtensor=fake_subtensor, @@ -66,16 +101,29 @@ def test_add_stake_multiple_extrinsic(mocker): ) # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_add_stake_multiple_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + assert result is True + assert fake_subtensor.substrate.compose_call.call_count == 2 + assert fake_subtensor.sign_and_send_extrinsic.call_count == 2 + + fake_subtensor.substrate.compose_call.assert_any_call( + call_module="SubtensorModule", + call_function="add_stake", + call_params={ + "hotkey": "hotkey1", + "amount_staked": 1099999666, + }, ) - mocked_add_stake_multiple_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - hotkey_ss58s=hotkey_ss58s, - amounts=amounts, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + fake_subtensor.substrate.compose_call.assert_any_call( + call_module="SubtensorModule", + call_function="add_stake", + call_params={ + "hotkey": "hotkey2", + "amount_staked": 2199999333, + }, + ) + fake_subtensor.sign_and_send_extrinsic.assert_called_with( + fake_subtensor.substrate.compose_call.return_value, + fake_wallet, + True, + True, ) - assert result == mocked_execute_coroutine.return_value diff --git a/tests/unit_tests/extrinsics/test_unstaking.py b/tests/unit_tests/extrinsics/test_unstaking.py index afd3c23e76..e8c84c3612 100644 --- a/tests/unit_tests/extrinsics/test_unstaking.py +++ b/tests/unit_tests/extrinsics/test_unstaking.py @@ -1,20 +1,23 @@ from bittensor.core.extrinsics import unstaking +from bittensor.utils.balance import Balance def test_unstake_extrinsic(mocker): - """Verify that sync `unstake_extrinsic` method calls proper async method.""" # Preps - fake_subtensor = mocker.Mock() + fake_subtensor = mocker.Mock( + **{ + "get_hotkey_owner.return_value": "hotkey_owner", + "get_stake_for_coldkey_and_hotkey.return_value": Balance(10.0), + "sign_and_send_extrinsic.return_value": (True, ""), + } + ) fake_wallet = mocker.Mock() + fake_wallet.coldkeypub.ss58_address = "hotkey_owner" hotkey_ss58 = "hotkey" amount = 1.1 wait_for_inclusion = True wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(unstaking, "execute_coroutine") - mocked_unstake_extrinsic = mocker.Mock() - unstaking.async_unstake_extrinsic = mocked_unstake_extrinsic - # Call result = unstaking.unstake_extrinsic( subtensor=fake_subtensor, @@ -26,35 +29,42 @@ def test_unstake_extrinsic(mocker): ) # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_unstake_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + assert result is True + + fake_subtensor.substrate.compose_call.assert_called_once_with( + call_module="SubtensorModule", + call_function="remove_stake", + call_params={ + "hotkey": "hotkey", + "amount_unstaked": 1100000000, + }, ) - mocked_unstake_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - hotkey_ss58=hotkey_ss58, - amount=amount, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + fake_subtensor.sign_and_send_extrinsic.assert_called_once_with( + fake_subtensor.substrate.compose_call.return_value, + fake_wallet, + True, + True, ) - assert result == mocked_execute_coroutine.return_value def test_unstake_multiple_extrinsic(mocker): """Verify that sync `unstake_multiple_extrinsic` method calls proper async method.""" # Preps - fake_subtensor = mocker.Mock() + fake_subtensor = mocker.Mock( + **{ + "get_hotkey_owner.return_value": "hotkey_owner", + "get_stake_for_coldkey_and_hotkey.return_value": Balance(10.0), + "sign_and_send_extrinsic.return_value": (True, ""), + "tx_rate_limit.return_value": 0, + } + ) fake_wallet = mocker.Mock() + fake_wallet.coldkeypub.ss58_address = "hotkey_owner" hotkey_ss58s = ["hotkey1", "hotkey2"] amounts = [1.1, 1.2] wait_for_inclusion = True wait_for_finalization = True - mocked_execute_coroutine = mocker.patch.object(unstaking, "execute_coroutine") - mocked_unstake_multiple_extrinsic = mocker.Mock() - unstaking.async_unstake_multiple_extrinsic = mocked_unstake_multiple_extrinsic - # Call result = unstaking.unstake_multiple_extrinsic( subtensor=fake_subtensor, @@ -66,16 +76,29 @@ def test_unstake_multiple_extrinsic(mocker): ) # Asserts - mocked_execute_coroutine.assert_called_once_with( - coroutine=mocked_unstake_multiple_extrinsic.return_value, - event_loop=fake_subtensor.event_loop, + assert result is True + assert fake_subtensor.substrate.compose_call.call_count == 2 + assert fake_subtensor.sign_and_send_extrinsic.call_count == 2 + + fake_subtensor.substrate.compose_call.assert_any_call( + call_module="SubtensorModule", + call_function="remove_stake", + call_params={ + "hotkey": "hotkey1", + "amount_unstaked": 1100000000, + }, ) - mocked_unstake_multiple_extrinsic.assert_called_once_with( - subtensor=fake_subtensor.async_subtensor, - wallet=fake_wallet, - hotkey_ss58s=hotkey_ss58s, - amounts=amounts, - wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization, + fake_subtensor.substrate.compose_call.assert_any_call( + call_module="SubtensorModule", + call_function="remove_stake", + call_params={ + "hotkey": "hotkey2", + "amount_unstaked": 1200000000, + }, + ) + fake_subtensor.sign_and_send_extrinsic.assert_called_with( + fake_subtensor.substrate.compose_call.return_value, + fake_wallet, + True, + True, ) - assert result == mocked_execute_coroutine.return_value From af26e989c927edeed38d2979700750ec44f6a194 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 24 Jan 2025 13:16:23 -0800 Subject: [PATCH 49/52] ruff --- tests/helpers/helpers.py | 1 - tests/integration_tests/test_subtensor_integration.py | 6 +++++- tests/unit_tests/extrinsics/test_commit_reveal.py | 4 +++- tests/unit_tests/extrinsics/test_root.py | 4 ++-- tests/unit_tests/extrinsics/test_transfer.py | 4 +++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/helpers/helpers.py b/tests/helpers/helpers.py index da5cabb1d2..70c8342009 100644 --- a/tests/helpers/helpers.py +++ b/tests/helpers/helpers.py @@ -145,7 +145,6 @@ def close(self, *args, **kwargs): class FakeConnectContextManager: - def __init__(self, seed): self.seed = seed diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index 0b8763dcd5..e0a0ba7527 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -32,7 +32,11 @@ async def prepare_test(mocker, seed): MetadataV15.decode_from_metadata_option(f.read()) ) subtensor = Subtensor("unknown", _mock=True) - mocker.patch.object(sync_substrate, "connect", mocker.Mock(return_value=FakeConnectContextManager(seed=seed))) + mocker.patch.object( + sync_substrate, + "connect", + mocker.Mock(return_value=FakeConnectContextManager(seed=seed)), + ) mocker.patch.object(subtensor.substrate, "registry", registry) return subtensor diff --git a/tests/unit_tests/extrinsics/test_commit_reveal.py b/tests/unit_tests/extrinsics/test_commit_reveal.py index c0beedc6ab..f3e3266d64 100644 --- a/tests/unit_tests/extrinsics/test_commit_reveal.py +++ b/tests/unit_tests/extrinsics/test_commit_reveal.py @@ -64,7 +64,9 @@ def test_do_commit_reveal_v3_success(mocker, subtensor): mocked_create_signed_extrinsic = mocker.patch.object( subtensor.substrate, "create_signed_extrinsic" ) - mocked_submit_extrinsic = mocker.patch.object(subtensor.substrate, "submit_extrinsic") + mocked_submit_extrinsic = mocker.patch.object( + subtensor.substrate, "submit_extrinsic" + ) # Call result = commit_reveal._do_commit_reveal_v3( diff --git a/tests/unit_tests/extrinsics/test_root.py b/tests/unit_tests/extrinsics/test_root.py index 3150de2400..21395735fc 100644 --- a/tests/unit_tests/extrinsics/test_root.py +++ b/tests/unit_tests/extrinsics/test_root.py @@ -68,7 +68,7 @@ def test_root_register_extrinsic( mocked_sign_and_send_extrinsic = mocker.patch.object( mock_subtensor, "sign_and_send_extrinsic", - return_value=(registration_success, "Error registering") + return_value=(registration_success, "Error registering"), ) mocker.patch.object( mock_subtensor.substrate, @@ -96,7 +96,7 @@ def test_root_register_extrinsic( mock_subtensor.substrate.compose_call.return_value, wallet=mock_wallet, wait_for_inclusion=wait_for_inclusion, - wait_for_finalization=wait_for_finalization + wait_for_finalization=wait_for_finalization, ) diff --git a/tests/unit_tests/extrinsics/test_transfer.py b/tests/unit_tests/extrinsics/test_transfer.py index b5cb97483d..607d703758 100644 --- a/tests/unit_tests/extrinsics/test_transfer.py +++ b/tests/unit_tests/extrinsics/test_transfer.py @@ -100,7 +100,9 @@ def test_do_transfer_is_success_false(subtensor, mocker): wait_for_inclusion=fake_wait_for_inclusion, wait_for_finalization=fake_wait_for_finalization, ) - mocked_format_error_message.assert_called_once_with(subtensor.substrate.submit_extrinsic.return_value.error_message) + mocked_format_error_message.assert_called_once_with( + subtensor.substrate.submit_extrinsic.return_value.error_message + ) assert result == ( False, From 28b3fdceaad4a473e21936d0749cada8920ba75d Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 24 Jan 2025 14:28:52 -0800 Subject: [PATCH 50/52] improve integration subtensor test --- tests/integration_tests/test_subtensor_integration.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index e0a0ba7527..1c33de39bc 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -1,7 +1,6 @@ import os.path import pytest -from async_substrate_interface import sync_substrate from bt_decode import PortableRegistry, MetadataV15 from bittensor import NeuronInfo @@ -32,9 +31,8 @@ async def prepare_test(mocker, seed): MetadataV15.decode_from_metadata_option(f.read()) ) subtensor = Subtensor("unknown", _mock=True) - mocker.patch.object( - sync_substrate, - "connect", + mocker.patch( + "async_substrate_interface.sync_substrate.connect", mocker.Mock(return_value=FakeConnectContextManager(seed=seed)), ) mocker.patch.object(subtensor.substrate, "registry", registry) From 64065d2157bd0ec3805df40a4ca6b155bfe85606 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 24 Jan 2025 14:34:06 -0800 Subject: [PATCH 51/52] remove comment --- tests/integration_tests/test_subtensor_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index 1c33de39bc..a2154b6615 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -53,7 +53,6 @@ async def test_get_all_subnets_info(mocker): @pytest.mark.asyncio async def test_metagraph(mocker): subtensor = await prepare_test(mocker, "metagraph") - # with mocker.patch.object(subtensor, "get_block_hash", return_value={"n": 19}): result = subtensor.metagraph(23) assert result.n == 19 assert result.netuid == 23 From 59fc0b5581b844d3de91daea2184626cc8fcf58f Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 27 Jan 2025 22:34:16 +0200 Subject: [PATCH 52/52] Mypy --- bittensor/core/async_subtensor.py | 2 +- bittensor/core/subtensor.py | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index b68971a873..c3ab32e6e1 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -414,7 +414,7 @@ async def query_runtime_api( self, runtime_api: str, method: str, - params: Optional[Union[list[list[int]], dict[str, int], list[int]]], + params: Optional[Union[list[list[int]], dict[str, int], list[int]]] = None, block: Optional[int] = None, block_hash: Optional[str] = None, reuse_block: bool = False, diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 20fb8c7f12..bd7aca83d3 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1,6 +1,6 @@ import copy from functools import lru_cache -from typing import TYPE_CHECKING, Any, Iterable, Optional, Union +from typing import TYPE_CHECKING, Any, Iterable, Optional, Union, cast from async_substrate_interface.errors import SubstrateRequestException from async_substrate_interface.sync_substrate import SubstrateInterface @@ -257,7 +257,7 @@ def query_runtime_api( self, runtime_api: str, method: str, - params: Optional[Union[list[int], dict[str, int]]] = None, + params: Optional[Union[list[list[int]], dict[str, int], list[int]]] = None, block: Optional[int] = None, ) -> Optional[str]: """ @@ -1206,8 +1206,11 @@ def get_subnet_reveal_period_epochs( self, netuid: int, block: Optional[int] = None ) -> int: """Retrieve the SubnetRevealPeriodEpochs hyperparameter.""" - return self.get_hyperparameter( - param_name="RevealPeriodEpochs", block=block, netuid=netuid + return cast( + int, + self.get_hyperparameter( + param_name="RevealPeriodEpochs", block=block, netuid=netuid + ), ) def get_subnets(self, block: Optional[int] = None) -> list[int]: @@ -1770,7 +1773,7 @@ def neurons_lite( return NeuronInfoLite.list_from_vec_u8(hex_to_bytes(hex_bytes_result)) - def query_identity(self, key: str, block: Optional[int] = None) -> Optional[str]: + def query_identity(self, key: str, block: Optional[int] = None) -> dict: """ Queries the identity of a neuron on the Bittensor blockchain using the given key. This function retrieves detailed identity information about a specific neuron, which is a crucial aspect of the network's @@ -2310,8 +2313,8 @@ def root_register( block = self.get_current_block() try: - recycle_call = self.get_hyperparameter( - param_name="Burn", netuid=0, block=block + recycle_call = cast( + int, self.get_hyperparameter(param_name="Burn", netuid=0, block=block) ) balance = self.get_balance(wallet.coldkeypub.ss58_address, block=block) except TypeError as e: @@ -2412,8 +2415,8 @@ def set_weights( """ def _blocks_weight_limit() -> bool: - bslu = self.blocks_since_last_update(netuid, uid) - wrl = self.weights_rate_limit(netuid) + bslu = cast(int, self.blocks_since_last_update(netuid, cast(int, uid))) + wrl = cast(int, self.weights_rate_limit(netuid)) return bslu > wrl retries = 0