diff --git a/bin/btcli b/bin/btcli index b302727073..68bb4fecd7 100755 --- a/bin/btcli +++ b/bin/btcli @@ -1,11 +1,16 @@ #!/usr/bin/env python3 -import bittensor +import sys + +import bittensor + if __name__ == '__main__': - bittensor.cli().run() + args = sys.argv[1:] + bittensor.cli(args=args).run() # The MIT License (MIT) # Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation diff --git a/bittensor/__init__.py b/bittensor/__init__.py index c92a69c810..091e5896d4 100644 --- a/bittensor/__init__.py +++ b/bittensor/__init__.py @@ -18,7 +18,7 @@ from rich.console import Console # Bittensor code and protocol version. -__version__ = '3.0.0' +__version__ = '3.3.3' version_split = __version__.split(".") __version_as_int__ = (100 * int(version_split[0])) + (10 * int(version_split[1])) + (1 * int(version_split[2])) diff --git a/bittensor/_axon/__init__.py b/bittensor/_axon/__init__.py index 0018ed26c7..afa1d0979f 100644 --- a/bittensor/_axon/__init__.py +++ b/bittensor/_axon/__init__.py @@ -55,8 +55,13 @@ def __new__( synapse_causal_lm: 'Callable' = None, synapse_causal_lm_next: 'Callable' = None, synapse_seq_2_seq: 'Callable' = None, + synapse_lasthidden_timeout: int = None, + synapse_causallm_timeout: int = None, + synapse_causallmnext_timeout: int = None, + synapse_seq2seq_timeout: int = None, synapse_checks: 'Callable' = None, thread_pool: 'futures.ThreadPoolExecutor' = None, + priority_threadpool: 'bittensor.prioritythreadpool' = None, server: 'grpc._Server' = None, port: int = None, ip: str = None, @@ -120,6 +125,10 @@ def __new__( config.axon.forward_timeout = forward_timeout if forward_timeout != None else config.axon.forward_timeout config.axon.backward_timeout = backward_timeout if backward_timeout != None else config.axon.backward_timeout config.axon.compression = compression if compression != None else config.axon.compression + config.axon.lasthidden_timeout = synapse_lasthidden_timeout if synapse_lasthidden_timeout != None else config.axon.lasthidden_timeout + config.axon.causallm_timeout = synapse_causallm_timeout if synapse_causallm_timeout != None else config.axon.causallm_timeout + config.axon.causallmnext_timeout = synapse_causallmnext_timeout if synapse_causallmnext_timeout is not None else config.axon.causallmnext_timeout + config.axon.seq2seq_timeout = synapse_seq2seq_timeout if synapse_seq2seq_timeout != None else config.axon.seq2seq_timeout axon.check_config( config ) # Determine the grpc compression algorithm @@ -147,15 +156,20 @@ def __new__( synapses[bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM] = synapse_causal_lm synapses[bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT] = synapse_causal_lm_next synapses[bittensor.proto.Synapse.SynapseType.TEXT_SEQ_2_SEQ] = synapse_seq_2_seq + + synapse_timeouts = { + bittensor.proto.Synapse.SynapseType.TEXT_LAST_HIDDEN_STATE: config.axon.lasthidden_timeout, + bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM: config.axon.causallm_timeout, + bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT: config.axon.causallmnext_timeout, + bittensor.proto.Synapse.SynapseType.TEXT_SEQ_2_SEQ: config.axon.seq2seq_timeout + } synapse_check_function = synapse_checks if synapse_checks != None else axon.default_synapse_check - if priority != None: + if priority != None and priority_threadpool == None: priority_threadpool = bittensor.prioritythreadpool(config=config) - else: - priority_threadpool = None - axon_instance = axon_impl.Axon( + axon_instance = axon_impl.Axon( wallet = wallet, server = server, ip = config.axon.ip, @@ -164,6 +178,7 @@ def __new__( backward = backward_text, synapses = synapses, synapse_checks = synapse_check_function, + synapse_timeouts = synapse_timeouts, priority = priority, priority_threadpool = priority_threadpool, forward_timeout = config.axon.forward_timeout, @@ -217,6 +232,14 @@ def add_args( cls, parser: argparse.ArgumentParser, prefix: str = None ): help='''maximum size of tasks in priority queue''', default = bittensor.defaults.axon.priority.maxsize) parser.add_argument('--' + prefix_str + 'axon.compression', type=str, help='''Which compression algorithm to use for compression (gzip, deflate, NoCompression) ''', default = bittensor.defaults.axon.compression) + parser.add_argument('--' + prefix_str + 'axon.lasthidden_timeout', type = int, + help='Timeout for last hidden synapse', default= bittensor.__blocktime__) + parser.add_argument('--' + prefix_str + 'axon.causallm_timeout', type = int, + help='Timeout for causallm synapse', default= bittensor.__blocktime__) + parser.add_argument('--' + prefix_str + 'axon.causallmnext_timeout', type = int, + help='Timeout for causallmnext synapse', default= bittensor.__blocktime__) + parser.add_argument('--' + prefix_str + 'axon.seq2seq_timeout', type = int, + help='Timeout for seq2seq synapse', default= 3*bittensor.__blocktime__) except argparse.ArgumentError: # re-parsing arguments. pass @@ -227,13 +250,13 @@ def add_args( cls, parser: argparse.ArgumentParser, prefix: str = None ): def add_defaults(cls, defaults): """ Adds parser defaults to object from enviroment variables. """ - defaults.axon = bittensor.config() + defaults.axon = bittensor.Config() defaults.axon.port = os.getenv('BT_AXON_PORT') if os.getenv('BT_AXON_PORT') != None else 8091 defaults.axon.ip = os.getenv('BT_AXON_IP') if os.getenv('BT_AXON_IP') != None else '[::]' defaults.axon.max_workers = os.getenv('BT_AXON_MAX_WORERS') if os.getenv('BT_AXON_MAX_WORERS') != None else 10 defaults.axon.maximum_concurrent_rpcs = os.getenv('BT_AXON_MAXIMUM_CONCURRENT_RPCS') if os.getenv('BT_AXON_MAXIMUM_CONCURRENT_RPCS') != None else 400 - defaults.axon.priority = bittensor.config() + defaults.axon.priority = bittensor.Config() defaults.axon.priority.max_workers = os.getenv('BT_AXON_PRIORITY_MAX_WORKERS') if os.getenv('BT_AXON_PRIORITY_MAX_WORKERS') != None else 10 defaults.axon.priority.maxsize = os.getenv('BT_AXON_PRIORITY_MAXSIZE') if os.getenv('BT_AXON_PRIORITY_MAXSIZE') != None else -1 @@ -325,7 +348,6 @@ def intercept_service(self, continuation, handler_call_details): self.message = str(e) return self._deny - def vertification(self,meta): r"""vertification of signature in metadata. Uses the pubkey and nounce """ diff --git a/bittensor/_axon/axon_impl.py b/bittensor/_axon/axon_impl.py index 2aafa4f9b7..8edc91cf98 100644 --- a/bittensor/_axon/axon_impl.py +++ b/bittensor/_axon/axon_impl.py @@ -32,6 +32,7 @@ import bittensor import bittensor.utils.stats as stat_utils +from datetime import datetime logger = logger.opt(colors=True) @@ -48,6 +49,7 @@ def __init__( backward: 'Callable', synapses: dict, synapse_checks: 'Callable', + synapse_timeouts: dict, priority: 'Callable' = None, priority_threadpool: 'bittensor.prioritythreadpool' = None, forward_timeout: int = None, @@ -81,6 +83,7 @@ def __init__( self.backward_timeout = backward_timeout self.synapse_callbacks = synapses self.synapse_checks = synapse_checks + self.synapse_timeouts = synapse_timeouts self.stats = self._init_stats() self.started = None self.optimizer_step = None @@ -184,6 +187,7 @@ def _forward(self, request): synapse_responses = [ synapse.empty() for synapse in synapses ] # We fill nones for non success. synapse_is_response = [ False for _ in synapses ] synapse_call_times = [ 0 for _ in synapses ] + synapse_timeout = min( [self.synapse_timeouts[s.synapse_type] for s in synapses] + [bittensor.__blocktime__] ) start_time = clock.time() # ================================================================== @@ -199,7 +203,7 @@ def check_if_should_return() -> bool: # ============================================================== # ==== Function which prints all log statements per synapse ==== # ============================================================== - def finalize_codes_stats_and_logs(): + def finalize_codes_stats_and_logs( message = None): for index, synapse in enumerate( synapses ): request.synapses [ index ].return_code = synapse_codes[ index ] # Set synapse wire proto codes. request.synapses [ index ].message = synapse_messages[ index ] # Set synapse wire proto message @@ -212,7 +216,7 @@ def finalize_codes_stats_and_logs(): pubkey = request.hotkey, inputs = synapse_inputs [index] , outputs = None if synapse_responses[index] == None else list( synapse_responses[index].shape ), - message = synapse_messages[ index ], + message = synapse_messages[ index ] if message == None else message, synapse = synapse.synapse_type ) @@ -280,9 +284,9 @@ def finalize_codes_stats_and_logs(): inputs_x = deserialized_forward_tensors, synapses = synapses, priority = priority, - hotkey= request.hotkey + hotkey = request.hotkey ) - forward_response_tensors, forward_codes, forward_messages = future.result( timeout= self.forward_timeout ) + forward_response_tensors, forward_codes, forward_messages = future.result( timeout = synapse_timeout - (clock.time() - start_time) ) else: forward_response_tensors, forward_codes, forward_messages = self.forward_callback( @@ -318,11 +322,10 @@ def finalize_codes_stats_and_logs(): except Exception as e: code = bittensor.proto.ReturnCode.UnknownException call_time = clock.time() - start_time - message = str ( e ) synapse_codes = [code for _ in synapses ] synapse_call_times = [call_time for _ in synapses ] - synapse_messages = [ message for _ in synapses ] - finalize_codes_stats_and_logs() + synapse_messages = [ 'Exception on Server' for _ in synapses ] + finalize_codes_stats_and_logs(message = str(e)) return [], bittensor.proto.ReturnCode.UnknownException, request.synapses # ================================================= diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index c9e5ecedfe..3b21794db4 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -3,6 +3,7 @@ """ # The MIT License (MIT) # Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -21,12 +22,11 @@ import argparse import os import sys -from typing import List +from typing import List, Optional import bittensor import torch from rich.prompt import Confirm, Prompt -from substrateinterface.utils.ss58 import ss58_decode, ss58_encode from . import cli_impl @@ -34,28 +34,34 @@ class cli: """ - Create and init the CLI class, which handles the coldkey, hotkey and tau transfer + Create and init the CLI class, which handles the coldkey, hotkey and tao transfer """ def __new__( - cls, - config: 'bittensor.Config' = None, + cls, + config: Optional['bittensor.Config'] = None, + args: Optional[List[str]] = None, ) -> 'bittensor.CLI': r""" Creates a new bittensor.cli from passed arguments. Args: config (:obj:`bittensor.Config`, `optional`): bittensor.cli.config() + args (`List[str]`, `optional`): + The arguments to parse from the command line. """ if config == None: - config = cli.config() + config = cli.config(args) cli.check_config( config ) return cli_impl.CLI( config = config) @staticmethod - def config() -> 'bittensor.config': + def config(args: List[str]) -> 'bittensor.config': """ From the argument parser, add config to bittensor.executor and local config Return: bittensor.config object """ - parser = argparse.ArgumentParser(description="Bittensor cli", usage="btcli ", add_help=True) + parser = argparse.ArgumentParser( + description=f"bittensor cli v{bittensor.__version__}", + usage="btcli ", + add_help=True) cmd_parsers = parser.add_subparsers(dest='command') overview_parser = cmd_parsers.add_parser( @@ -611,7 +617,7 @@ def config() -> 'bittensor.config': required=False ) - return bittensor.config( parser ) + return bittensor.config( parser, args=args ) @staticmethod def check_config (config: 'bittensor.Config'): diff --git a/bittensor/_cli/cli_impl.py b/bittensor/_cli/cli_impl.py index bdc4744e1a..bdce4358dc 100644 --- a/bittensor/_cli/cli_impl.py +++ b/bittensor/_cli/cli_impl.py @@ -309,7 +309,7 @@ def unstake( self ): if not self.config.no_prompt: if not Confirm.ask("Do you want to unstake from the following keys:\n" + \ "".join([ - f" [bold white]- {wallet.hotkey_str}: {amount}𝜏[/bold white]\n" for wallet, amount in zip(final_wallets, final_amounts) + f" [bold white]- {wallet.hotkey_str}: {amount.tao}𝜏[/bold white]\n" for wallet, amount in zip(final_wallets, final_amounts) ]) ): return None diff --git a/bittensor/_config/__init__.py b/bittensor/_config/__init__.py index a1e984dab6..0a18c7c8f5 100644 --- a/bittensor/_config/__init__.py +++ b/bittensor/_config/__init__.py @@ -3,6 +3,7 @@ """ # The MIT License (MIT) # Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -19,12 +20,14 @@ # DEALINGS IN THE SOFTWARE. import os -from argparse import ArgumentParser +import sys +from argparse import ArgumentParser, Namespace +from typing import List, Optional +import bittensor import yaml from loguru import logger -import bittensor from . import config_impl logger = logger.opt(colors=True) @@ -37,13 +40,15 @@ class InvalidConfigFile(Exception): """ In place of YAMLError """ - def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): + def __new__( cls, parser: ArgumentParser = None, strict: bool = False, args: Optional[List[str]] = None ): r""" Translates the passed parser into a nested Bittensor config. Args: parser (argparse.Parser): Command line parser object. strict (bool): If true, the command line arguments are strictly parsed. + args (list of str): + Command line arguments. Returns: config (bittensor.Config): Nested config object created from parser arguments. @@ -69,8 +74,15 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): except Exception as e: config_file_path = None + # Get args from argv if not passed in. + if args == None: + args = sys.argv[1:] + + # Parse args not strict + params = cls.__parse_args__(args=args, parser=parser, strict=False) + # 2. Optionally check for --strict, if stict we will parse the args strictly. - strict = parser.parse_known_args()[0].strict + strict = params.strict if config_file_path != None: config_file_path = os.path.expanduser(config_file_path) @@ -83,10 +95,8 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): print('Error in loading: {} using default parser settings'.format(e)) # 2. Continue with loading in params. - if not strict: - params = parser.parse_known_args()[0] - else: - params = parser.parse_args() + params = cls.__parse_args__(args=args, parser=parser, strict=strict) + _config = config_impl.Config() # Splits params on dot syntax i.e neuron.axon_port @@ -107,6 +117,27 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): return _config + @staticmethod + def __parse_args__( args: List[str], parser: ArgumentParser = None, strict: bool = False) -> Namespace: + """Parses the passed args use the passed parser. + Args: + args (List[str]): + List of arguments to parse. + parser (argparse.ArgumentParser): + Command line parser object. + strict (bool): + If true, the command line arguments are strictly parsed. + Returns: + Namespace: + Namespace object created from parser arguments. + """ + if not strict: + params = parser.parse_known_args(args=args)[0] + else: + params = parser.parse_args(args=args) + + return params + @staticmethod def full(): """ From the parser, add arguments to multiple bittensor sub-modules diff --git a/bittensor/_dendrite/dendrite_impl.py b/bittensor/_dendrite/dendrite_impl.py index de117f0fe9..2b973ae925 100644 --- a/bittensor/_dendrite/dendrite_impl.py +++ b/bittensor/_dendrite/dendrite_impl.py @@ -860,4 +860,3 @@ def to_wandb( self ): except Exception as e: bittensor.logging.error( prefix='failed dendrite.to_wandb()', sufix = str(e)) return {} - diff --git a/bittensor/_neuron/text/core_server/__init__.py b/bittensor/_neuron/text/core_server/__init__.py index 2fefb88748..e1ee4c17fb 100644 --- a/bittensor/_neuron/text/core_server/__init__.py +++ b/bittensor/_neuron/text/core_server/__init__.py @@ -73,7 +73,6 @@ def __init__( causallmnext = None, seq2seq = None, synapse_list = None, - ): if config == None: config = server.config() config = config; diff --git a/bittensor/_neuron/text/core_server/nucleus_impl.py b/bittensor/_neuron/text/core_server/nucleus_impl.py index 661e0cf38e..f2d7d2c19b 100644 --- a/bittensor/_neuron/text/core_server/nucleus_impl.py +++ b/bittensor/_neuron/text/core_server/nucleus_impl.py @@ -123,7 +123,6 @@ def __init__(self, # -- keeps track of gradients applied self.backward_gradients_count = 0 - def set_fine_tuning_params(self) -> Tuple[bool, str]: r''' Set to tune only the parameter of the last layer Returns: @@ -538,7 +537,7 @@ def config (): parser.add_argument('--neuron.blacklist.stake', type=float, help='Amount of stake (tao) in order not to get blacklisted', default=10) parser.add_argument('--neuron.blocks_per_epoch', type=int, help='Blocks per epoch', default=10) parser.add_argument('--neuron.blacklist.time', type=int, help='how often a peer can query you (seconds) ', default=1) - parser.add_argument('--neuron.blocks_per_set_weights', type=float, help='how often to set weights', default=100) + parser.add_argument('--neuron.blocks_per_set_weights', type=float, help='how often to set weights', default=-1) parser.add_argument('--neuron.metagraph_sync', type=float, help='how often to sync the metagraph', default=100000) parser.add_argument('--neuron.blacklist_allow_non_registered', action='store_true', help='''If true, allow non-registered peers''', default=False) parser.add_argument('--neuron.disable_blacklist', action='store_true', help='Turns off blacklisting', default=False) @@ -564,4 +563,3 @@ def config (): bittensor.dataset.add_args( parser ) bittensor.metagraph.add_args( parser ) return bittensor.config( parser ) - diff --git a/bittensor/_neuron/text/core_server/run.py b/bittensor/_neuron/text/core_server/run.py index acc095bef3..9ef5a65630 100644 --- a/bittensor/_neuron/text/core_server/run.py +++ b/bittensor/_neuron/text/core_server/run.py @@ -331,7 +331,8 @@ def backward_callback(inputs_x:torch.FloatTensor, grads_dy:torch.FloatTensor, sy ) last_set_block = subtensor.get_current_block() - + blocks_per_epoch = subtensor.blocks_per_epoch if config.neuron.blocks_per_epoch == -1 else config.neuron.blocks_per_epoch + blocks_per_set_weights = subtensor.blocks_per_epoch if config.neuron.blocks_per_set_weights == -1 else config.neuron.blocks_per_set_weights # --- Run Forever. while True: @@ -404,7 +405,7 @@ def backward_callback(inputs_x:torch.FloatTensor, grads_dy:torch.FloatTensor, sy wandb.log( { **wandb_data, **wandb_info_axon, **local_data }, step = current_block ) wandb.log( { 'stats': wandb.Table( dataframe = df ) }, step = current_block ) - if current_block - last_set_block > config.neuron.blocks_per_set_weights: + if current_block - last_set_block > blocks_per_set_weights: try: bittensor.__console__.print('[green]Current Status:[/green]', {**wandb_data, **local_data}) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 4329f9265b..57e1f840be 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -211,6 +211,7 @@ def add_args( cls, parser ): parser.add_argument('--neuron.wait_for_finalization', action='store_true', help='''when setting weights the miner waits for trnasaction finalization.''', default=False) parser.add_argument('--neuron.forward_num', type=int, help='''How much forward request before a backward call.''', default=3) parser.add_argument('--neuron.validation_synapse', type=str, help='''Synapse used for validation.''', default='TextCausalLMNext', choices = ['TextCausalLMNext', 'TextCausalLM']) + parser.add_argument('--neuron.exclude_quantile', type=float, help='Exclude the lowest quantile from weight setting.', default=0.1) @classmethod def config ( cls ): @@ -365,14 +366,23 @@ def run_epoch( self ): # Each block length lasts blocks_per_epoch blocks. # This gives us a consistent network wide timer. # Here we run until blocks_per_epochs have progressed. - self.metagraph_sync() # Reset metagraph. + if self.epoch > 0: # skip first epoch: already synced at start of run + self.metagraph_sync() # Reset metagraph. + + self.nucleus.permute_uids = [] # clear nucleus permutation before epoch epoch_steps = 0 epoch_responsive_uids = set() epoch_queried_uids = set() + epoch_start_time = time.time() start_block = self.subtensor.block - while self.subtensor.block < start_block + blocks_per_epoch: + # normal epoch duration is blocks_per_epoch if all UIDs have been queried + # try to query each UID at least once - assumes nucleus samples without replacement + # but keep maximum epoch duration at 2 * blocks_per_epoch + while (self.subtensor.block < start_block + blocks_per_epoch or + (self.subtensor.block < start_block + 2 * blocks_per_epoch and + len(epoch_queried_uids) < self.metagraph.n)): start_time = time.time() # === Forward === @@ -434,8 +444,9 @@ def run_epoch( self ): f'[dim] Epoch {self.epoch}[/dim] | ' f'[bright_green not bold]{len(responsive_uids)}[/bright_green not bold]/' f'[white]{len(queried_uids)}[/white] ' - f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' - f'[[yellow]{step_time:.3g}[/yellow]s]') + f'[[yellow]{step_time:.3g}[/yellow]s] ' + f'[dim white not bold][green]{len(epoch_responsive_uids)}[/green]/' + f'{len(epoch_queried_uids)}[/dim white not bold]') if self.config.logging.debug or self.config.logging.trace: # === Print stats update (table) === @@ -485,6 +496,20 @@ def run_epoch( self ): if self.config.logging.debug or self.config.logging.trace: self.weights_table(sample_uids, sample_weights) # print weights table + # set weights console message (every epoch) + print(f"[white not bold]{datetime.datetime.now():%Y-%m-%d %H:%M:%S}[/white not bold]{' ' * 4} | " + f"{f'[bright_white]Set weights[/bright_white]'.center(16 + len('[bright_white][/bright_white]'))} | " + f'[bright_green not bold]{len(sample_weights)}[/bright_green not bold] [dim]weights set[/dim] | ' + f'[bright_green not bold]{len(epoch_responsive_uids)}[/bright_green not bold]/' + f'[white]{len(epoch_queried_uids)}[/white] ' + f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' + f'[[yellow]{time.time() - epoch_start_time:.0f}[/yellow]s] | ' + f'[dim]weights[/dim] sum:{sample_weights.sum().item():.2g} ' + f'[white] max:[bold]{sample_weights.max().item():.4g}[/bold] / ' + f'min:[bold]{sample_weights.min().item():.4g}[/bold] [/white] ' + f'\[{sample_weights.max().item() / sample_weights.min().item():.1f}:1] ' + f'({max_allowed_ratio} allowed)') + self.subtensor.set_weights( uids=sample_uids.detach().to('cpu'), weights=sample_weights.detach().to('cpu'), @@ -543,6 +568,7 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): zkey = key + '!' # zeroing key stats.setdefault(zkey, 0.) # initialize zkey val to zero to gradually increase with observations if key in _stats and not math.isnan(_stats[key]): + responsive_uids += [_uid] stats[zkey] = (1 - self.alpha) * stats[zkey] + self.alpha * _stats[key] else: stats[zkey] = (1 - self.alpha) * stats[zkey] # + self.alpha * 0 @@ -555,7 +581,6 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): updates = 'updates_' + key if updates in stats: stats[updates] += 1 # increment number of normal EMA updates made - responsive_uids += [_uid] else: stats.setdefault(updates, 1) # add updates fields for new uid entries @@ -608,9 +633,25 @@ def calculate_weights(self, responsive_uids: Set, queried_uids: Set): sample_uids = preferred_uids[:weights_to_set] # slice to weights_to_set sample_weights = neuron_weights[:weights_to_set] # slice to weights_to_set + logger.info(f'{len(sample_weights)} Shapley values | min:{sample_weights.min()} max:{sample_weights.max()}') + + # === Exclude lowest quantile from weight setting === + max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weight quantile + if 0 < max_exclude: + exclude_quantile = min([self.config.neuron.exclude_quantile, max_exclude]) # reduce quantile to meet min_allowed_weights + lowest_quantile = sample_weights.quantile(exclude_quantile) # find lowest quantile threshold + sample_uids = sample_uids[lowest_quantile <= sample_weights] # exclude uids with weights below quantile + sample_weights = sample_weights[lowest_quantile <= sample_weights] # exclude weights below quantile + + logger.info(f'Exclude {exclude_quantile} quantile ({lowest_quantile}) | ' + f'{len(sample_weights)} Shapley values | min:{sample_weights.min()} max:{sample_weights.max()}') + # === Normalize and apply max_allowed_ratio === sample_weights = bittensor.utils.weight_utils.normalize_max_multiple(x=sample_weights, multiple=max_allowed_ratio) + logger.info(f'{len(sample_weights)} normalize_max_multiple | ' + f'min:{sample_weights.min()} max:{sample_weights.max()}') + return sample_uids, sample_weights def weights_table(self, sample_uids, sample_weights, include_uids=None, num_rows: int = None): @@ -660,6 +701,7 @@ def __init__( self, config, device, subtensor ): self.config = config self.device = device self.max_n = subtensor.max_n + self.permute_uids = [] # iterable of next UIDs to query, reset to permuted UIDs when empty tokenizer = bittensor.tokenizer() self.pad_token = tokenizer(tokenizer.pad_token)['input_ids'][0] @@ -702,6 +744,7 @@ def add_args( cls, parser ): parser.add_argument('--nucleus.noise_multiplier', type=float, help='Standard deviation multipler on weights', default=2 ) parser.add_argument('--nucleus.dendrite_backward', action='store_true', help='Pass backward request to the server side or not', default=False ) parser.add_argument('--nucleus.scaling_law_power', type=float, help='Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) + parser.add_argument('--nucleus.synergy_scaling_law_power', type=float, help='Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.6) @classmethod def config ( cls ): @@ -794,18 +837,26 @@ def forward( # Ensure number of queried neurons does not exceed metagraph.n num_endpoints = min([self.config.nucleus.topk, metagraph.n]) - logger.info(f'Forward \t| Routing forward [{time.time() - start_time:.3g}s]') - logger.info(f'Dendrite \t| Request {num_endpoints} x {list(inputs_seq.shape)}') - request_start_time = time.time() + # === Ensure each UID is queried once === + # Persist object variable self.permute_uids across forward calls. + # Reset to new permutation of all UIDs once empty. + if len(self.permute_uids) == 0: # no more UIDs to query + self.permute_uids = torch.randperm(metagraph.n) # reset to new permutation of all UIDs # === Randomly select num_endpoints UIDs === - random_uids = torch.randperm(metagraph.n)[:num_endpoints] + random_uids = self.permute_uids[:num_endpoints] # newest selection of UIDs to query + self.permute_uids = self.permute_uids[num_endpoints:] # slice out remaining selection # === Get endpoint information for the selected UIDs === # We index into the metagraph's endpoints and return a list of the filtered set of endpoints we wish to query. # random_endpoints: List[bittensor.endpoints]: endpoint information for filtered uids. # len(neurons) == self.config.nucleus.topk random_endpoints = [metagraph.endpoints[uid] for uid in random_uids] + num_endpoints = len(random_endpoints) # in case len(self.permute_uids) < num_endpoints during random_uids select + + logger.info(f'Forward \t| Routing forward [{time.time() - start_time:.3g}s]') + logger.info(f'Dendrite \t| Request {num_endpoints} x {list(inputs_seq.shape)}') + request_start_time = time.time() # === Define which synapse we want to use === # The synapse defines the task we are sending to the neurons @@ -847,8 +898,9 @@ def forward( # === Prepare validation parameter set === console_width = self.config.get('width', None) # console width for rich table displays of synapse measures validation_params = (random_uids, query_responses, return_ops, times, routing_score, - inputs, val_len, self.loss_fct, self.config.nucleus.scaling_law_power, console_width, - self.config.logging.debug or self.config.logging.trace) + inputs, val_len, self.loss_fct, + self.config.nucleus.scaling_law_power, self.config.nucleus.synergy_scaling_law_power, + console_width, self.config.logging.debug or self.config.logging.trace) loss = torch.tensor(0.).to(self.device) # to accumulate neuron_loss and routing_loss over synapses neuron_stats = {} # to gather neuron synapse validation measures and statistics @@ -876,7 +928,8 @@ def scaling_law_loss_to_params(loss): def textcausallm(uids: torch.Tensor, query_responses: List[List[torch.FloatTensor]], return_ops: List[torch.LongTensor], times: List[torch.FloatTensor], routing_score: torch.FloatTensor, - inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, scaling_law_power: float, + inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, + scaling_law_power: float, synergy_scaling_law_power: float, console_width: int, logging, synapse: 'bittensor.TextCausalLM' = None, index_s: int = 0 ) -> Tuple[torch.FloatTensor, Dict]: r""" @@ -901,6 +954,8 @@ def textcausallm(uids: torch.Tensor, query_responses: List[List[torch.FloatTenso CrossEntropy loss function to use. scaling_law_power (:obj:`float`, `required`): Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. + synergy_scaling_law_power (:obj:`float`, `required`): + Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. console_width (:obj:`int`, `required`): Config console width for table print. logging (:obj:`bool`, `required`): @@ -957,9 +1012,9 @@ def _synergy(first, second, target, _ext): synergy_start_time = time.time() syn_loss_diff = shapley_synergy(stats, _synergy, ext='', target=inputs_seq[:, 1:], - scaling_law_power=scaling_law_power) + scaling_law_power=synergy_scaling_law_power) syn_loss_diff_val = shapley_synergy(stats, _synergy, ext='_val', target=inputs_val, - scaling_law_power=scaling_law_power) + scaling_law_power=synergy_scaling_law_power) # === Shapley value combination === # Combine base values with synergy approximation to get final Shapley values. @@ -998,7 +1053,8 @@ def _synergy(first, second, target, _ext): def textcausallmnext(uids: torch.Tensor, query_responses: List[List[torch.FloatTensor]], return_ops: List[torch.LongTensor], times: List[torch.FloatTensor], routing_score: torch.FloatTensor, - inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, scaling_law_power: float, + inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, + scaling_law_power: float, synergy_scaling_law_power: float, console_width: int, logging, synapse: 'bittensor.TextCausalLMNext' = None, index_s: int = 0 ) -> Tuple[torch.FloatTensor, Dict]: r""" @@ -1023,6 +1079,8 @@ def textcausallmnext(uids: torch.Tensor, query_responses: List[List[torch.FloatT CrossEntropy loss function to use. scaling_law_power (:obj:`float`, `required`): Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. + synergy_scaling_law_power (:obj:`float`, `required`): + Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. console_width (:obj:`int`, `required`): Config console width for table print. logging (:obj:`bool`, `required`): @@ -1075,7 +1133,7 @@ def _synergy(first, second, target, ext): synergy_start_time = time.time() - syn_loss_diff = shapley_synergy(stats, _synergy, '_nxt', scaling_law_power=scaling_law_power) + syn_loss_diff = shapley_synergy(stats, _synergy, '_nxt', scaling_law_power=synergy_scaling_law_power) # === Shapley value combination === # Combine base values with synergy approximation to get final Shapley values. @@ -1212,6 +1270,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens # Synergy = measured performance above expected performance # Measured in effective number of model parameters, just like base Shapley values. syn_loss_diff = {} # expected_loss - measured_loss (where > 0) + responsives = [uid for uid, stat in stats.items() if 'loss' + ext in stat] for _first, first in stats.items(): if 'loss' + ext not in first: continue @@ -1229,6 +1288,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens measured_loss = synergy(first, second, target, ext) loss_diff_share = torch.clamp(expected_loss - measured_loss, 0) / 2 # record direct loss diff + loss_diff_share /= len(responsives) # average over responsives first['synergy_loss_diff' + ext] += loss_diff_share second['synergy_loss_diff' + ext] += loss_diff_share @@ -1244,6 +1304,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens pow_expected_params = torch.pow(expected_params, scaling_law_power) synergy_share = torch.clamp(pow_measured_params - pow_expected_params, 0) / 2 + synergy_share /= len(responsives) # average over responsives first['synergy' + ext] += synergy_share # share synergy amongst coalition members second['synergy' + ext] += synergy_share diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index a994c1e79a..8c0be7c88f 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -137,11 +137,16 @@ def __new__( config.subtensor.chain_endpoint = subtensor.determine_chain_endpoint( bittensor.defaults.subtensor.network ) config.subtensor.network = bittensor.defaults.subtensor.network + # make sure it's wss:// or ws:// + endpoint_url: str = config.subtensor.chain_endpoint + if endpoint_url[0:6] != "wss://" and endpoint_url[0:5] != "ws://": + endpoint_url = "ws://{}".format(endpoint_url) + substrate = SubstrateInterface( ss58_format = bittensor.__ss58_format__, type_registry_preset='substrate-node-template', type_registry = __type_registery__, - url = "ws://{}".format(config.subtensor.chain_endpoint), + url = endpoint_url, use_remote_preset=True ) diff --git a/bittensor/_subtensor/subtensor_impl.py b/bittensor/_subtensor/subtensor_impl.py index c05f3a68da..48d67d07e7 100644 --- a/bittensor/_subtensor/subtensor_impl.py +++ b/bittensor/_subtensor/subtensor_impl.py @@ -501,74 +501,73 @@ def register ( else: pow_result = bittensor.utils.create_pow( self, wallet, num_processes=num_processes, update_interval=update_interval) - # pow failed - if not pow_result: - # might be registered already - if (wallet.is_registered( self )): - bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") - return True - - # pow successful, proceed to submit pow to chain for registration - else: - with bittensor.__console__.status(":satellite: Submitting POW..."): - # check if pow result is still valid - while bittensor.utils.POWNotStale(self, pow_result): - with self.substrate as substrate: - # create extrinsic call - call = substrate.compose_call( - call_module='SubtensorModule', - call_function='register', - call_params={ - 'block_number': pow_result['block_number'], - 'nonce': pow_result['nonce'], - 'work': bittensor.utils.hex_bytes_to_u8_list( pow_result['work'] ), - 'hotkey': wallet.hotkey.ss58_address, - 'coldkey': wallet.coldkeypub.ss58_address - } - ) - extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.hotkey ) - response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization ) - - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") - return True - - # process if registration successful, try again if pow is still valid - response.process_events() - if not response.is_success: - if 'key is already registered' in response.error_message: - # Error meant that the key is already registered. - bittensor.__console__.print(":white_heavy_check_mark: [green]Already Registered[/green]") - return True - - bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) - time.sleep(0.5) - - # Successful registration, final check for neuron and pubkey - else: - bittensor.__console__.print(":satellite: Checking Balance...") - neuron = self.neuron_for_pubkey( wallet.hotkey.ss58_address ) - if not neuron.is_null: - bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") + # pow failed + if not pow_result: + # might be registered already + if (wallet.is_registered( self )): + bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") + return True + + # pow successful, proceed to submit pow to chain for registration + else: + with bittensor.__console__.status(":satellite: Submitting POW..."): + # check if pow result is still valid + while bittensor.utils.POWNotStale(self, pow_result): + with self.substrate as substrate: + # create extrinsic call + call = substrate.compose_call( + call_module='SubtensorModule', + call_function='register', + call_params={ + 'block_number': pow_result['block_number'], + 'nonce': pow_result['nonce'], + 'work': bittensor.utils.hex_bytes_to_u8_list( pow_result['work'] ), + 'hotkey': wallet.hotkey.ss58_address, + 'coldkey': wallet.coldkeypub.ss58_address + } + ) + extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.hotkey ) + response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization ) + + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") return True + + # process if registration successful, try again if pow is still valid + response.process_events() + if not response.is_success: + if 'key is already registered' in response.error_message: + # Error meant that the key is already registered. + bittensor.__console__.print(":white_heavy_check_mark: [green]Already Registered[/green]") + return True + + bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) + time.sleep(0.5) + + # Successful registration, final check for neuron and pubkey else: - # neuron not found, try again - bittensor.__console__.print(":cross_mark: [red]Unknown error. Neuron not found.[/red]") - continue - else: - # Exited loop because pow is no longer valid. - bittensor.__console__.print( "[red]POW is stale.[/red]" ) - return False - - if attempts < max_allowed_attempts: - #Failed registration, retry pow - attempts += 1 - bittensor.__console__.print( ":satellite: Failed registration, retrying pow ...({}/{})".format(attempts, max_allowed_attempts)) - else: - # Failed to register after max attempts. - bittensor.__console__.print( "[red]No more attempts.[/red]" ) - return False + bittensor.__console__.print(":satellite: Checking Balance...") + neuron = self.neuron_for_pubkey( wallet.hotkey.ss58_address ) + if not neuron.is_null: + bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") + return True + else: + # neuron not found, try again + bittensor.__console__.print(":cross_mark: [red]Unknown error. Neuron not found.[/red]") + continue + else: + # Exited loop because pow is no longer valid. + bittensor.__console__.print( "[red]POW is stale.[/red]" ) + return False + if attempts < max_allowed_attempts: + #Failed registration, retry pow + attempts += 1 + bittensor.__console__.print( ":satellite: Failed registration, retrying pow ...({}/{})".format(attempts, max_allowed_attempts)) + else: + # Failed to register after max attempts. + bittensor.__console__.print( "[red]No more attempts.[/red]" ) + return False def serve ( self, diff --git a/bittensor/_threadpool/__init__.py b/bittensor/_threadpool/__init__.py index 63cc8d4a24..d04e0f4c14 100644 --- a/bittensor/_threadpool/__init__.py +++ b/bittensor/_threadpool/__init__.py @@ -48,7 +48,6 @@ def __new__( config.axon.priority.maxsize = maxsize if maxsize != None else config.axon.priority.maxsize prioritythreadpool.check_config( config ) - return priority_thread_pool_impl.PriorityThreadPoolExecutor(maxsize = config.axon.priority.maxsize, max_workers = config.axon.priority.max_workers) @classmethod diff --git a/bittensor/_threadpool/priority_thread_pool_impl.py b/bittensor/_threadpool/priority_thread_pool_impl.py index a162d4f680..adcabbe8f2 100644 --- a/bittensor/_threadpool/priority_thread_pool_impl.py +++ b/bittensor/_threadpool/priority_thread_pool_impl.py @@ -166,10 +166,10 @@ def submit(self, fn, *args, **kwargs): start_time = time.time() if 'priority' in kwargs: del kwargs['priority'] + f = _base.Future() w = _WorkItem(f, fn, start_time, args, kwargs) - self._work_queue.put((-float(priority + eplison), w), block=False) self._adjust_thread_count() return f diff --git a/requirements.txt b/requirements.txt index 1c7f9b8fd6..c96f9a27e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ pytest-rerunfailures coveralls pytest-cov pyyaml -rich +rich>=12.5.1 retry requests>=2.25.0 scalecodec>=1.0.35 diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index d8276422e0..65bdd8f67a 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -1,5 +1,6 @@ # The MIT License (MIT) # Copyright © 2022 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -16,18 +17,16 @@ # DEALINGS IN THE SOFTWARE. -import sys import unittest from types import SimpleNamespace from typing import Dict from unittest.mock import ANY, MagicMock, call, patch +import pytest import bittensor from bittensor._subtensor.subtensor_mock import mock_subtensor from bittensor.utils.balance import Balance from substrateinterface.base import Keypair -from substrateinterface.exceptions import SubstrateRequestException - from tests.helpers import CLOSE_IN_VALUE @@ -1327,6 +1326,33 @@ def test_list_no_wallet( self ): # This shouldn't raise an error anymore cli.run() +def test_btcli_help(): + """ + Verify the correct help text is output when the --help flag is passed + """ + with pytest.raises(SystemExit) as pytest_wrapped_e: + with patch('argparse.ArgumentParser._print_message', return_value=None) as mock_print_message: + args = [ + '--help' + ] + bittensor.cli(args=args).run() + + # Should try to print help + mock_print_message.assert_called_once() + + call_args = mock_print_message.call_args + args, _ = call_args + help_out = args[0] + + # Expected help output even if parser isn't working well + ## py3.6-3.9 or py3.10+ + assert 'optional arguments' in help_out or 'options' in help_out + # Expected help output if all commands are listed + assert 'positional arguments' in help_out + # Verify that cli is printing the help message for + assert 'overview' in help_out + assert 'run' in help_out + if __name__ == "__main__": cli = TestCli() diff --git a/tests/integration_tests/test_dendrite.py b/tests/integration_tests/test_dendrite.py index df0920441b..b2a243eb14 100644 --- a/tests/integration_tests/test_dendrite.py +++ b/tests/integration_tests/test_dendrite.py @@ -15,6 +15,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import time import torch import pytest import bittensor @@ -260,9 +261,6 @@ def test_dendrite_multiple(): def test_dendrite_to_df(): dendrite.to_dataframe(bittensor.metagraph(_mock=True).sync()) -def test_dend_del(): - dendrite.__del__() - def test_successful_synapse(): wallet = bittensor.wallet() def forward_generate( inputs_x, synapse, model_output = None): @@ -310,7 +308,7 @@ def forward_casual_lm_next(inputs_x, synapse, model_output=None): print(codes) assert list(codes[0]) == [bittensor.proto.ReturnCode.Success] * len(synapses) - + def test_failing_synapse(): wallet = bittensor.wallet() def faulty( inputs_x, synapse, model_output = None): @@ -370,6 +368,8 @@ def forward_casual_lm_next(inputs_x, synapse, model_output=None): axon.attach_synapse_callback(faulty, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT) return_tensors, codes, times = dendrite.text(endpoints=endpoint, inputs=inputs, synapses=synapses) assert list(codes[0]) == [bittensor.proto.ReturnCode.UnknownException] * len(synapses) + + axon.stop() def test_missing_synapse(): wallet = bittensor.wallet() @@ -424,9 +424,62 @@ def forward_casual_lm_next(inputs_x, synapse, model_output=None): assert list(codes[0]) == [bittensor.proto.ReturnCode.Success, bittensor.proto.ReturnCode.Success, bittensor.proto.ReturnCode.Success, bittensor.proto.ReturnCode.NotImplemented] + axon.stop() + +def test_dendrite_timeout(): + wallet = bittensor.wallet() + def forward_hidden_state( inputs_x, synapse, model_output = None): + time.sleep(3) + return None, None, torch.rand(inputs_x.shape[0], inputs_x.shape[1], bittensor.__network_dim__) + + def forward_casual_lm(inputs_x, synapse, model_output = None): + time.sleep(3) + return None, None, torch.rand(inputs_x.shape[0], inputs_x.shape[1], bittensor.__vocab_size__) + + def forward_casual_lm_next(inputs_x, synapse, model_output=None): + time.sleep(3) + return None, None, synapse.nill_forward_response_tensor(inputs_x) + + axon = bittensor.axon ( + port = 8098, + ip = '0.0.0.0', + wallet = wallet, + ) + + axon.start() + + endpoint = bittensor.endpoint( + version = bittensor.__version_as_int__, + uid = 0, + hotkey = wallet.hotkey.ss58_address, + ip = '0.0.0.0', + ip_type = 4, + port = 8098, + modality = 0, + coldkey = wallet.coldkeypub.ss58_address + ) + + dendrite = bittensor.dendrite() + inputs = next(dataset) + synapses = [bittensor.synapse.TextLastHiddenState(), bittensor.synapse.TextCausalLM(), + bittensor.synapse.TextCausalLMNext()] + + axon.attach_synapse_callback( forward_hidden_state, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_LAST_HIDDEN_STATE ) + axon.attach_synapse_callback( forward_casual_lm, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM ) + axon.attach_synapse_callback(forward_casual_lm_next, synapse_type=bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT) + + return_tensors, codes, times = dendrite.text( endpoints=endpoint, inputs=inputs, synapses=synapses, timeout = 2) + assert list(codes[0]) == [bittensor.proto.ReturnCode.Timeout, bittensor.proto.ReturnCode.Timeout, + bittensor.proto.ReturnCode.Timeout] + + axon.stop() + +def test_dend_del(): + dendrite.__del__() + def test_clear(): dataset.close() if __name__ == "__main__": bittensor.logging(debug = True) - test_dendrite_forward_tensor() \ No newline at end of file + test_dendrite_timeout() \ No newline at end of file diff --git a/tests/integration_tests/test_wallet.py b/tests/integration_tests/test_wallet.py index 505b1cb631..ad86abff3b 100644 --- a/tests/integration_tests/test_wallet.py +++ b/tests/integration_tests/test_wallet.py @@ -125,7 +125,6 @@ def test_wallet_coldkeypub_create(): check_keys_exists(the_wallet, coldkey_exists=False, hotkey_exists=False) # Don't check the coldkey or hotkey assert the_wallet.coldkeypub.ss58_address == "5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" - def test_wallet_add_stake(): subtensor = bittensor.subtensor(network = 'nobunaga') the_wallet = init_wallet().create(coldkey_use_password = False, hotkey_use_password = False) @@ -167,7 +166,6 @@ def test_wallet_transfer(): the_wallet.is_registered = MagicMock(return_value = False) the_wallet.remove_stake(subtensor = subtensor) - def test_wallet_mock(): wallet = bittensor.wallet(_mock=True) assert wallet.hotkey_file.exists_on_device() @@ -177,7 +175,6 @@ def test_wallet_mock(): assert wallet.coldkey assert wallet.coldkeypub - def test_wallet_mock_from_config(): config = bittensor.wallet.config() config.wallet.name = 'mock' @@ -189,7 +186,6 @@ def test_wallet_mock_from_config(): assert wallet.coldkey assert wallet.coldkeypub - def test_wallet_mock_from_name(): wallet = bittensor.wallet(name = 'mock') assert wallet.hotkey_file.exists_on_device() @@ -199,7 +195,6 @@ def test_wallet_mock_from_name(): assert wallet.coldkey assert wallet.coldkeypub - def test_wallet_mock_from_func(): wallet = bittensor.wallet.mock() assert wallet.hotkey_file.exists_on_device() diff --git a/tests/unit_tests/bittensor_tests/test_axon.py b/tests/unit_tests/bittensor_tests/test_axon.py index 20040e0b3f..8f2bfd22fa 100644 --- a/tests/unit_tests/bittensor_tests/test_axon.py +++ b/tests/unit_tests/bittensor_tests/test_axon.py @@ -27,6 +27,8 @@ from bittensor.utils.test_utils import get_random_unused_port import concurrent +from concurrent.futures import ThreadPoolExecutor + wallet = bittensor.wallet.mock() axon = bittensor.axon(wallet = wallet) bittensor.logging(debug = True) @@ -812,6 +814,70 @@ def forward( inputs_x: torch.FloatTensor, synapses , model_output = None): response, code, synapses = axon._forward( request ) assert code == bittensor.proto.ReturnCode.Success + +def test_forward_priority_timeout(): + def priority(pubkey:str, request_type:str, inputs_x): + return 100 + + def forward( inputs_x: torch.FloatTensor, synapses, hotkey): + time.sleep(15) + + axon = bittensor.axon(wallet = wallet, priority= priority, forward_timeout = 5) + axon.attach_forward_callback(forward) + + inputs_raw = torch.rand(1,1) + synapses = [bittensor.synapse.TextLastHiddenState()] + serializer = bittensor.serializer( serializer_type = bittensor.proto.Serializer.MSGPACK ) + inputs_serialized = serializer.serialize(inputs_raw, from_type = bittensor.proto.TensorType.TORCH) + request = bittensor.proto.TensorMessage( + version = bittensor.__version_as_int__, + tensors=[inputs_serialized], + hotkey = axon.wallet.hotkey.ss58_address, + synapses= [ syn.serialize_to_wire_proto() for syn in synapses ] + ) + + response, code, synapses = axon._forward( request ) + assert code == bittensor.proto.ReturnCode.Timeout + + axon.stop() + +def test_forward_priority_2nd_request_timeout(): + def priority(pubkey:str, request_type:str, inputs_x): + return 100 + + axon = bittensor.axon(wallet = wallet, priority= priority, priority_threadpool = bittensor.prioritythreadpool(max_workers = 1)) + + def forward( inputs_x: torch.FloatTensor, synapses , model_output = None): + time.sleep(1) + return None, dict(), torch.zeros( [inputs_x.shape[0], inputs_x.shape[1], bittensor.__network_dim__]) + + axon.attach_synapse_callback( forward, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_LAST_HIDDEN_STATE) + inputs_raw = torch.rand(3, 3) + synapses = [bittensor.synapse.TextLastHiddenState()] + serializer = bittensor.serializer( serializer_type = bittensor.proto.Serializer.MSGPACK ) + inputs_serialized = synapses[0].serialize_forward_request_tensor(inputs_raw) + request = bittensor.proto.TensorMessage( + version = bittensor.__version_as_int__, + tensors=[inputs_serialized], + synapses= [ syn.serialize_to_wire_proto() for syn in synapses ], + hotkey = axon.wallet.hotkey.ss58_address, + ) + start_time = time.time() + executor = ThreadPoolExecutor(2) + future = executor.submit(axon._forward, (request)) + future2 = executor.submit(axon._forward, (request)) + response, code, synapses = future.result() + assert code == bittensor.proto.ReturnCode.Success + + try: + response2, code2, synapses2 = future2.result(timeout = 1 - (time.time() - start_time)) + except concurrent.futures.TimeoutError: + pass + else: + raise AssertionError('Expected to Timeout') + + axon.stop() + def test_backward_response_success_text_priority(): def priority(pubkey:str, request_type:str, inputs_x): @@ -1044,5 +1110,7 @@ def test_axon_is_destroyed(): if __name__ == "__main__": # test_forward_joint_success() - test_forward_joint_missing_synapse() + # test_forward_joint_missing_synapse() + # test_forward_priority_timeout() + test_forward_priority_2nd_request_timeout() # test_forward_joint_faulty_synapse() \ No newline at end of file diff --git a/tests/unit_tests/bittensor_tests/test_config.py b/tests/unit_tests/bittensor_tests/test_config.py index 6713a5f5db..981bc3beab 100644 --- a/tests/unit_tests/bittensor_tests/test_config.py +++ b/tests/unit_tests/bittensor_tests/test_config.py @@ -21,6 +21,8 @@ import argparse import pytest +bittensor.logging(debug = True) + def test_loaded_config(): with pytest.raises(NotImplementedError): bittensor.Config(loaded_config=True) @@ -77,31 +79,21 @@ def test_prefix(): config_non_strict = bittensor.config( parser, strict=False) config_strict = bittensor.config( parser, strict=True) - bittensor.dendrite( config_strict ) - bittensor.dendrite( config_non_strict ) - bittensor.dendrite( config_strict.second ) - bittensor.dendrite( config_non_strict.second ) - - bittensor.axon( config_strict ) - bittensor.axon( config_non_strict ) - bittensor.axon( config_strict.second ) - bittensor.axon( config_non_strict.second ) - - bittensor.dataset( config_strict ) - bittensor.dataset( config_non_strict ) - bittensor.dataset( config_strict.second ) - bittensor.dataset( config_non_strict.second ) + bittensor.dendrite( config_strict ).__del__() + bittensor.dendrite( config_non_strict ).__del__() + bittensor.dendrite( config_strict.second ).__del__() + bittensor.dendrite( config_non_strict.second ).__del__() - bittensor.axon( config_strict ) - bittensor.axon( config_non_strict ) - bittensor.axon( config_strict.second ) - bittensor.axon( config_non_strict.second ) + bittensor.axon( config_strict ).stop() + bittensor.axon( config_non_strict ).stop() + bittensor.axon( config_strict.second ).stop() + bittensor.axon( config_non_strict.second ).stop() bittensor.metagraph( config_strict ) bittensor.metagraph( config_non_strict ) bittensor.metagraph( config_strict.second ) bittensor.metagraph( config_non_strict.second ) - + bittensor.wallet( config_strict ) bittensor.wallet( config_non_strict ) bittensor.wallet( config_strict.second ) @@ -138,6 +130,7 @@ def test_to_defaults(): config.to_defaults() if __name__ == "__main__": - test_loaded_config() - test_strict() - test_to_defaults() \ No newline at end of file + # test_loaded_config() + # test_strict() + # test_to_defaults() + test_prefix() \ No newline at end of file diff --git a/tests/unit_tests/bittensor_tests/test_forward_backward.py b/tests/unit_tests/bittensor_tests/test_forward_backward.py index 48296f2cc0..ee5fcb47ea 100644 --- a/tests/unit_tests/bittensor_tests/test_forward_backward.py +++ b/tests/unit_tests/bittensor_tests/test_forward_backward.py @@ -292,5 +292,5 @@ def test_dendrite_del(): del dendrite_mock if __name__ == "__main__": - test_dendrite_backward_multiple() + pass diff --git a/tests/unit_tests/bittensor_tests/test_neuron.py b/tests/unit_tests/bittensor_tests/test_neuron.py index 9fa79e7768..4d5195cb1f 100644 --- a/tests/unit_tests/bittensor_tests/test_neuron.py +++ b/tests/unit_tests/bittensor_tests/test_neuron.py @@ -260,7 +260,5 @@ def exit_early(*args, **kwargs): # Should try to register the neuron mock_register.assert_called_once() - - if __name__ == '__main__': pass diff --git a/tests/unit_tests/bittensor_tests/test_receptor.py b/tests/unit_tests/bittensor_tests/test_receptor.py index d1d3458be2..c1b2dc82c9 100644 --- a/tests/unit_tests/bittensor_tests/test_receptor.py +++ b/tests/unit_tests/bittensor_tests/test_receptor.py @@ -613,7 +613,7 @@ def forward_casual_lm_next(input, synapse): def test_axon_receptor_connection_forward_unimplemented(): axon = bittensor.axon ( - port = 8081, + port = 8091, ip = '127.0.0.1', wallet = wallet, ) @@ -624,7 +624,7 @@ def test_axon_receptor_connection_forward_unimplemented(): uid = 0, ip = '127.0.0.1', ip_type = 4, - port = 8081, + port = 8091, hotkey = wallet.hotkey.ss58_address, coldkey = wallet.coldkey.ss58_address, modality = 2 diff --git a/tests/unit_tests/bittensor_tests/test_receptor_pool.py b/tests/unit_tests/bittensor_tests/test_receptor_pool.py index 337be0923d..165e787198 100644 --- a/tests/unit_tests/bittensor_tests/test_receptor_pool.py +++ b/tests/unit_tests/bittensor_tests/test_receptor_pool.py @@ -72,14 +72,13 @@ def test_receptor_pool_backward(): torch.tensor([])]] receptor_pool.backward( endpoints, synapses, x, grads, timeout=1) - def test_receptor_pool_max_workers_forward(): neuron_obj2 = bittensor.endpoint( version = bittensor.__version_as_int__, uid = 0, ip = '0.0.0.1', ip_type = 4, - port = 12345, + port = 12346, hotkey = wallet2.hotkey.ss58_address, coldkey = wallet2.coldkey.ss58_address, modality = 0 @@ -116,6 +115,7 @@ def test_receptor_pool_forward_success(): tensors = [y_hidden_serialized, y_causallm_serialized, y_causallmnext_serialized, y_seq_2_seq_serialized] ) + receptor_pool = bittensor.receptor_pool(wallet=wallet,max_active_receptors=1) receptor_pool._get_or_create_receptor_for_endpoint(neuron_obj) receptor_pool.receptors[neuron_obj.hotkey].stub.Forward = MagicMock( return_value = mock_return_val ) resp1, codes, _ = receptor_pool.forward( endpoints, synapses, x, timeout=1) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index d3bad0f6f6..5d0643bc08 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -10,7 +10,6 @@ import random import torch import multiprocessing -from types import SimpleNamespace from sys import platform from substrateinterface.base import Keypair @@ -347,57 +346,6 @@ def test_pow_not_stale_diff_block_number_too_old(self): assert not bittensor.utils.POWNotStale(mock_subtensor, mock_solution) -def test_pow_called_for_cuda(): - class MockException(Exception): - pass - mock_compose_call = MagicMock(side_effect=MockException) - - mock_subtensor = bittensor.subtensor(_mock=True) - mock_subtensor.neuron_for_pubkey=MagicMock(is_null=True) - mock_subtensor.substrate = MagicMock( - __enter__= MagicMock(return_value=MagicMock( - compose_call=mock_compose_call - )), - __exit__ = MagicMock(return_value=None), - ) - - mock_wallet = SimpleNamespace( - hotkey=SimpleNamespace( - ss58_address='' - ), - coldkeypub=SimpleNamespace( - ss58_address='' - ) - ) - - mock_result = { - "block_number": 1, - 'nonce': random.randint(0, pow(2, 32)), - 'work': b'\x00' * 64, - } - - with patch('bittensor.utils.POWNotStale', return_value=True) as mock_pow_not_stale: - with patch('torch.cuda.is_available', return_value=True) as mock_cuda_available: - with patch('bittensor.utils.create_pow', return_value=mock_result) as mock_create_pow: - with patch('bittensor.utils.hex_bytes_to_u8_list', return_value=b''): - - # Should exit early - with pytest.raises(MockException): - mock_subtensor.register(mock_wallet, cuda=True, prompt=False) - - mock_pow_not_stale.assert_called_once() - mock_create_pow.assert_called_once() - mock_cuda_available.assert_called_once() - - call0 = mock_pow_not_stale.call_args - assert call0[0][0] == mock_subtensor - assert call0[0][1] == mock_result - - mock_compose_call.assert_called_once() - call1 = mock_compose_call.call_args - assert call1[1]['call_function'] == 'register' - call_params = call1[1]['call_params'] - assert call_params['nonce'] == mock_result['nonce'] if __name__ == "__main__": test_solve_for_difficulty_fast_registered_already() \ No newline at end of file