From 4288c3af92927fc60bc93379d562d684e2eaf9ce Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 11 Aug 2022 14:56:05 -0400 Subject: [PATCH 01/28] [feature] cpu register faster (#854) * add update interval and num proc flag * add better number output * optimize multiproc cpu reg keeping proc until solution * fix test * make sure to exit properly if registered during * fix tests * change import to use tests * add optional type hints and None default * change to count using allowed processes * add documentation. Fix random start --- bittensor/_cli/__init__.py | 18 + bittensor/_cli/cli_impl.py | 2 +- bittensor/_subtensor/subtensor_impl.py | 9 +- bittensor/utils/__init__.py | 406 +++++++++++++----- tests/integration_tests/test_cli.py | 4 +- .../bittensor_tests/test_balance.py | 2 +- .../bittensor_tests/utils/test_utils.py | 33 +- 7 files changed, 346 insertions(+), 128 deletions(-) diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index e74946b58e..83fe2b8f26 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -571,6 +571,24 @@ def config() -> 'bittensor.config': help='''Set true to avoid prompting the user.''', default=False, ) + register_parser.add_argument( + '--num_processes', + '--num', + '-n', + dest='num_processes', + help="Number of processors to use for registration", + type=int, + default=None, + ) + register_parser.add_argument( + '--update_interval', + '-u', + dest='update_interval', + help="The number of nonces to process before checking for next block during registration", + type=int, + default=None, + ) + bittensor.wallet.add_args( register_parser ) bittensor.subtensor.add_args( register_parser ) diff --git a/bittensor/_cli/cli_impl.py b/bittensor/_cli/cli_impl.py index cec4734fd4..245b943885 100644 --- a/bittensor/_cli/cli_impl.py +++ b/bittensor/_cli/cli_impl.py @@ -239,7 +239,7 @@ def register( self ): """ wallet = bittensor.wallet( config = self.config ) subtensor = bittensor.subtensor( config = self.config ) - subtensor.register( wallet = wallet, prompt = not self.config.no_prompt) + subtensor.register( wallet = wallet, prompt = not self.config.no_prompt, num_processes = self.config.num_processes, update_interval = self.config.update_interval ) def transfer( self ): r""" Transfer token of amount to destination. diff --git a/bittensor/_subtensor/subtensor_impl.py b/bittensor/_subtensor/subtensor_impl.py index e1f3dc1b33..e71fcca7fa 100644 --- a/bittensor/_subtensor/subtensor_impl.py +++ b/bittensor/_subtensor/subtensor_impl.py @@ -16,8 +16,7 @@ # DEALINGS IN THE SOFTWARE. import torch from rich.prompt import Confirm -from typing import List, Dict, Union -from multiprocessing import Process +from typing import List, Dict, Union, Optional import bittensor from tqdm import tqdm @@ -440,7 +439,9 @@ def register ( wait_for_inclusion: bool = False, wait_for_finalization: bool = True, prompt: bool = False, - max_allowed_attempts: int = 3 + max_allowed_attempts: int = 3, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, ) -> bool: r""" Registers the wallet to chain. Args: @@ -474,7 +475,7 @@ def register ( attempts = 1 while True: # Solve latest POW. - pow_result = bittensor.utils.create_pow( self, wallet ) + pow_result = bittensor.utils.create_pow( self, wallet, num_processes=num_processes, update_interval=update_interval ) with bittensor.__console__.status(":satellite: Registering...({}/{})".format(attempts,max_allowed_attempts)) as status: # pow failed diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 8ebdb9ef95..a5379e5428 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -1,21 +1,22 @@ import binascii -import multiprocessing -import ctypes -import struct import hashlib -from Crypto.Hash import keccak import math -import bittensor +import multiprocessing +import numbers +import os import random -import rich import time -import torch -import numbers +from dataclasses import dataclass +from queue import Empty +from typing import Any, Dict, Optional, Tuple, Union + +import bittensor import pandas import requests +import torch +from Crypto.Hash import keccak +from substrateinterface import Keypair from substrateinterface.utils import ss58 -from substrateinterface import Keypair, KeypairType -from typing import Any, Tuple, List, Union, Optional def indexed_values_to_dataframe ( @@ -76,10 +77,12 @@ def u8_list_to_hex( values: list ): return total def create_seal_hash( block_hash:bytes, nonce:int ) -> bytes: - nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) block_bytes = block_hash.encode('utf-8')[2:] + nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) pre_seal = nonce_bytes + block_bytes - seal = hashlib.sha256( bytearray(hex_bytes_to_u8_list(pre_seal)) ).digest() + seal_sh256 = hashlib.sha256( bytearray(hex_bytes_to_u8_list(pre_seal)) ).digest() + kec = keccak.new(digest_bits=256) + seal = kec.update( seal_sh256 ).digest() return seal def seal_meets_difficulty( seal:bytes, difficulty:int ): @@ -102,7 +105,172 @@ def solve_for_difficulty( block_hash, difficulty ): break return nonce, seal -def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = None, update_interval: int = 500000 ) -> Tuple[int, int, Any, int, Any]: + +def get_human_readable(num, suffix="H"): + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(num) < 1000.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1000.0 + return f"{num:.1f}Y{suffix}" + + +def millify(n: int): + millnames = ['',' K',' M',' B',' T'] + n = float(n) + millidx = max(0,min(len(millnames)-1, + int(math.floor(0 if n == 0 else math.log10(abs(n))/3)))) + + return '{:.0f}{}'.format(n / 10**(3 * millidx), millnames[millidx]) + +@dataclass +class POWSolution: + """A solution to the registration PoW problem.""" + nonce: int + block_number: int + difficulty: int + seal: bytes + +class Solver(multiprocessing.Process): + """ + A process that solves the registration PoW problem. + + Args: + proc_num: int + The number of the process being created. + num_proc: int + The total number of processes running. + update_interval: int + The number of nonces to try to solve before checking for a new block. + best_queue: multiprocessing.Queue + The queue to put the best nonce the process has found during the pow solve. + New nonces are added each update_interval. + time_queue: multiprocessing.Queue + The queue to put the time the process took to finish each update_interval. + Used for calculating the average time per update_interval across all processes. + solution_queue: multiprocessing.Queue + The queue to put the solution the process has found during the pow solve. + newBlockEvent: multiprocessing.Event + The event to set by the main process when a new block is finalized in the network. + The solver process will check for the event after each update_interval. + The solver process will get the new block hash and difficulty and start solving for a new nonce. + stopEvent: multiprocessing.Event + The event to set by the main process when all the solver processes should stop. + The solver process will check for the event after each update_interval. + The solver process will stop when the event is set. + Used to stop the solver processes when a solution is found. + curr_block: multiprocessing.Array + The array containing this process's current block hash. + The main process will set the array to the new block hash when a new block is finalized in the network. + The solver process will get the new block hash from this array when newBlockEvent is set. + curr_block_num: multiprocessing.Value + The value containing this process's current block number. + The main process will set the value to the new block number when a new block is finalized in the network. + The solver process will get the new block number from this value when newBlockEvent is set. + curr_diff: multiprocessing.Array + The array containing this process's current difficulty. + The main process will set the array to the new difficulty when a new block is finalized in the network. + The solver process will get the new difficulty from this array when newBlockEvent is set. + check_block: multiprocessing.Lock + The lock to prevent this process from getting the new block data while the main process is updating the data. + limit: int + The limit of the pow solve for a valid solution. + """ + proc_num: int + num_proc: int + update_interval: int + best_queue: multiprocessing.Queue + time_queue: multiprocessing.Queue + solution_queue: multiprocessing.Queue + newBlockEvent: multiprocessing.Event + stopEvent: multiprocessing.Event + curr_block: multiprocessing.Array + curr_block_num: multiprocessing.Value + curr_diff: multiprocessing.Array + check_block: multiprocessing.Lock + limit: int + + def __init__(self, proc_num, num_proc, update_interval, best_queue, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit): + multiprocessing.Process.__init__(self) + self.proc_num = proc_num + self.num_proc = num_proc + self.update_interval = update_interval + self.best_queue = best_queue + self.time_queue = time_queue + self.solution_queue = solution_queue + self.newBlockEvent = multiprocessing.Event() + self.newBlockEvent.clear() + self.curr_block = curr_block + self.curr_block_num = curr_block_num + self.curr_diff = curr_diff + self.check_block = check_block + self.stopEvent = stopEvent + self.limit = limit + + def run(self): + block_number: int + block_bytes: bytes + block_difficulty: int + nonce_limit = int(math.pow(2,64)) - 1 + + # Start at random nonce + nonce_start = random.randint( 0, nonce_limit ) + nonce_end = nonce_start + self.update_interval + while not self.stopEvent.is_set(): + if self.newBlockEvent.is_set(): + with self.check_block: + block_number = self.curr_block_num.value + block_bytes = bytes(self.curr_block) + block_difficulty = int(self.curr_diff[0] >> 32 | self.curr_diff[1]) + + self.newBlockEvent.clear() + # reset nonces to start from random point + # prevents the same nonces (for each block) from being tried by multiple processes + # also prevents the same nonces from being tried by multiple peers + nonce_start = random.randint( 0, nonce_limit ) + nonce_end = nonce_start + self.update_interval + + # Do a block of nonces + solution, time = solve_for_nonce_block(self, nonce_start, nonce_end, block_bytes, block_difficulty, self.limit, block_number) + if solution is not None: + self.solution_queue.put(solution) + + # Send time + self.time_queue.put_nowait(time) + + nonce_start += self.update_interval * self.num_proc + nonce_end += self.update_interval * self.num_proc + + +def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, block_bytes: bytes, difficulty: int, limit: int, block_number: int) -> Tuple[Optional[POWSolution], int]: + best_local = float('inf') + best_seal_local = [0]*32 + start = time.time() + for nonce in range(nonce_start, nonce_end): + # Create seal. + nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) + pre_seal = nonce_bytes + block_bytes + seal_sh256 = hashlib.sha256( bytearray(hex_bytes_to_u8_list(pre_seal)) ).digest() + kec = keccak.new(digest_bits=256) + seal = kec.update( seal_sh256 ).digest() + seal_number = int.from_bytes(seal, "big") + + # Check if seal meets difficulty + product = seal_number * difficulty + if product < limit: + print(f"{solver.proc_num} found a solution: {nonce}, {block_number}, {str(block_bytes)}, {str(seal)}, {difficulty}") + # Found a solution, save it. + return POWSolution(nonce, block_number, difficulty, seal), time.time() - start + + if (product - limit) < best_local: + best_local = product - limit + best_seal_local = seal + + # Send best solution to best queue. + solver.best_queue.put((best_local, best_seal_local)) + return None, time.time() - start + + +def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = None, update_interval: Optional[int] = None ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. Args: @@ -115,126 +283,152 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = None, upd update_interval: int Number of nonces to solve before updating block information. Note: - - We should modify the number of processes based on user input. - We can also modify the update interval to do smaller blocks of work, while still updating the block information after a different number of nonces, to increase the transparency of the process while still keeping the speed. """ if num_processes == None: - num_processes = multiprocessing.cpu_count() + # get the number of allowed processes for this process + num_processes = len(os.sched_getaffinity(0)) + + if update_interval is None: + update_interval = 50_000 + limit = int(math.pow(2,256)) - 1 + + console = bittensor.__console__ + status = console.status("Solving") + + best_seal: bytes + best_number: int + best_number = float('inf') + + curr_block = multiprocessing.Array('h', 64, lock=True) # byte array + curr_block_num = multiprocessing.Value('i', 0, lock=True) # int + curr_diff = multiprocessing.Array('Q', [0, 0], lock=True) # [high, low] + + def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: multiprocessing.Lock): + with lock: + curr_block_num.value = block_number + for i in range(64): + curr_block[i] = block_bytes[i] + curr_diff[0] = diff >> 32 + curr_diff[1] = diff & 0xFFFFFFFF # low 32 bits + + status.start() + + # Establish communication queues + ## See the Solver class for more information on the queues. + stopEvent = multiprocessing.Event() + stopEvent.clear() + best_queue = multiprocessing.Queue() + solution_queue = multiprocessing.Queue() + time_queue = multiprocessing.Queue() + check_block = multiprocessing.Lock() + + # Start consumers + solvers = [ Solver(i, num_processes, update_interval, best_queue, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) + for i in range(num_processes) ] + + # Get first block block_number = subtensor.get_current_block() difficulty = subtensor.difficulty block_hash = subtensor.substrate.get_block_hash( block_number ) while block_hash == None: block_hash = subtensor.substrate.get_block_hash( block_number ) block_bytes = block_hash.encode('utf-8')[2:] - - limit = int(math.pow(2,256)) - 1 - nonce_limit = int(math.pow(2,64)) - 1 - nonce = random.randint( 0, nonce_limit ) - start_time = time.time() - - console = bittensor.__console__ - status = console.status("Solving") + old_block_number = block_number + # Set to current block + update_curr_block(block_number, block_bytes, difficulty, check_block) - #found_solution = multiprocessing.Value('q', -1, lock=False) # int - found_solution = multiprocessing.Array('Q', [0, 0, 0], lock=True) # [valid, nonce_high, nonce_low] - best_raw = struct.pack("d", float('inf')) - best = multiprocessing.Array(ctypes.c_char, best_raw, lock=True) # byte array to get around int size of ctypes - best_seal = multiprocessing.Array('h', 32, lock=True) # short array should hold bytes (0, 256) + # Set new block events for each solver to start + for w in solvers: + w.newBlockEvent.set() - with multiprocessing.Pool(processes=num_processes, initializer=initProcess_, initargs=(solve_, found_solution, best, best_seal)) as pool: - status.start() - while found_solution[0] == 0 and not wallet.is_registered(subtensor): - iterable = [( nonce_start, - nonce_start + update_interval , - block_bytes, - difficulty, - block_hash, - block_number, - limit) for nonce_start in list(range(nonce, nonce + update_interval*num_processes, update_interval))] - result = pool.starmap(solve_, iterable=iterable) - old_nonce = nonce - nonce += update_interval*num_processes - nonce = nonce % nonce_limit - itrs_per_sec = update_interval*num_processes / (time.time() - start_time) - start_time = time.time() - difficulty = subtensor.difficulty - block_number = subtensor.get_current_block() + for w in solvers: + w.start() # start the solver processes + + start_time = time.time() + solution = None + best_seal = None + itrs_per_sec = 0 + while not wallet.is_registered(subtensor): + # Wait until a solver finds a solution + try: + solution = solution_queue.get(block=True, timeout=0.25) + if solution is not None: + break + except Empty: + # No solution found, try again + pass + + # check for new block + block_number = subtensor.get_current_block() + if block_number != old_block_number: + old_block_number = block_number + # update block information block_hash = subtensor.substrate.get_block_hash( block_number) while block_hash == None: block_hash = subtensor.substrate.get_block_hash( block_number) block_bytes = block_hash.encode('utf-8')[2:] - with best_seal.get_lock(): - message = f"""Solving - time spent: {time.time() - start_time} - Nonce: [bold white]{nonce}[/bold white] - Difficulty: [bold white]{difficulty}[/bold white] - Iters: [bold white]{int(itrs_per_sec)}/s[/bold white] - Block: [bold white]{block_number}[/bold white] - Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white] - Best: [bold white]{binascii.hexlify(bytes(best_seal) or bytes(0))}[/bold white]""" - status.update(message.replace(" ", "")) + difficulty = subtensor.difficulty + + update_curr_block(block_number, block_bytes, difficulty, check_block) + # Set new block events for each solver + for w in solvers: + w.newBlockEvent.set() + + # Get times for each solver + time_total = 0 + num_time = 0 + while time_queue.qsize() > 0: + try: + time_ = time_queue.get_nowait() + time_total += time_ + num_time += 1 + + except Empty: + break - # exited while, found_solution contains the nonce or wallet is registered - if found_solution[0] == 0: # didn't find solution - status.stop() - return None, None, None, None, None + # Calculate average time per solver for the update_interval + if num_time > 0: + time_avg = time_total / num_time + itrs_per_sec = update_interval*num_processes / time_avg + + # get best solution from each solver using the best_queue + while best_queue.qsize() > 0: + try: + num, seal = best_queue.get_nowait() + if num < best_number: + best_number = num + best_seal = seal + + except Empty: + break - found_unpacked: int = found_solution[1] << 32 | found_solution[2] - nonce, block_number, block_hash, difficulty, seal = result[ math.floor( (found_unpacked-old_nonce) / update_interval) ] - status.stop() - return nonce, block_number, block_hash, difficulty, seal - -def initProcess_(f, found_solution, best, best_seal): - f.found = found_solution - f.best = best - f.best_seal = best_seal - -def solve_(nonce_start, nonce_end, block_bytes, difficulty, block_hash, block_number, limit): - best_local = float('inf') - best_seal_local = [0]*32 - start = time.time() - for nonce in range(nonce_start, nonce_end): - # Create seal. - nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) - pre_seal = nonce_bytes + block_bytes - seal_sh256 = hashlib.sha256( bytearray(hex_bytes_to_u8_list(pre_seal)) ).digest() - kec = keccak.new(digest_bits=256) - seal = kec.update( seal_sh256 ).digest() - seal_number = int.from_bytes(seal, "big") - product = seal_number * difficulty - - if product < limit: - with solve_.found.get_lock(): - solve_.found[0] = 1; - solve_.found[1] = nonce >> 32 - solve_.found[2] = nonce & 0xFFFFFFFF # low 32 bits - return (nonce, block_number, block_hash, difficulty, seal) - - if (product - limit) < best_local: - best_local = product - limit - best_seal_local = seal + message = f"""Solving + time spent: {time.time() - start_time} + Difficulty: [bold white]{millify(difficulty)}[/bold white] + Iters: [bold white]{get_human_readable(int(itrs_per_sec), 'H')}/s[/bold white] + Block: [bold white]{block_number}[/bold white] + Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white] + Best: [bold white]{binascii.hexlify(bytes(best_seal) if best_seal else bytes(0))}[/bold white]""" + status.update(message.replace(" ", "")) + + # exited while, solution contains the nonce or wallet is registered + stopEvent.set() # stop all other processes + status.stop() - with solve_.best.get_lock(): - best_value_as_d = struct.unpack('d', solve_.best.raw)[0] - - if best_local < best_value_as_d: - with solve_.best_seal.get_lock(): - solve_.best.raw = struct.pack('d', best_local) - for i in range(32): - solve_.best_seal[i] = best_seal_local[i] + return solution - return None -def create_pow( subtensor, wallet ): - nonce, block_number, block_hash, difficulty, seal = solve_for_difficulty_fast( subtensor, wallet ) +def create_pow( subtensor, wallet, num_processes: Optional[int] = None, update_interval: Optional[int] = None ) -> Optional[Dict[str, Any]]: + solution: POWSolution = solve_for_difficulty_fast( subtensor, wallet, num_processes=num_processes, update_interval=update_interval ) + nonce, block_number, difficulty, seal = solution.nonce, solution.block_number, solution.difficulty, solution.seal return None if nonce is None else { 'nonce': nonce, 'difficulty': difficulty, 'block_number': block_number, - 'block_hash': block_hash, 'work': binascii.hexlify(seal) } diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index 6c147338b3..77269d366b 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -28,7 +28,7 @@ from substrateinterface.base import Keypair from substrateinterface.exceptions import SubstrateRequestException -from ..helpers import CLOSE_IN_VALUE +from tests.helpers import CLOSE_IN_VALUE class TestCli(unittest.TestCase): @@ -1082,6 +1082,8 @@ def test_register( self ): config = self.config config.subtensor._mock = True config.command = "register" + config.num_processes = 1 + config.update_interval = 50_000 config.subtensor.network = "mock" config.no_prompt = True diff --git a/tests/unit_tests/bittensor_tests/test_balance.py b/tests/unit_tests/bittensor_tests/test_balance.py index 0349eb678f..60f61fac67 100644 --- a/tests/unit_tests/bittensor_tests/test_balance.py +++ b/tests/unit_tests/bittensor_tests/test_balance.py @@ -19,7 +19,7 @@ from typing import Union from bittensor import Balance -from ...helpers import CLOSE_IN_VALUE +from tests.helpers import CLOSE_IN_VALUE from hypothesis import given from hypothesis import strategies as st diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 677cb87ddc..03065ccdca 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -111,7 +111,8 @@ def test_u8_list_to_hex(): def test_create_seal_hash(): block_hash = '0xba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279' nonce = 10 - assert bittensor.utils.create_seal_hash(block_hash, nonce) == b'\xf5\xad\xd3\xff\x9d\xd0=F\x1c\xe0}\n9Szs[kb\xb7^@\xe94\x99hw\xa7Q\xde\x8d3' + seal_hash = bittensor.utils.create_seal_hash(block_hash, nonce) + assert seal_hash == b'\xc5\x01B6"\xa8\xa5FDPK\xe49\xad\xdat\xbb:\x87d\x13/\x86\xc6:I8\x9b\x88\xf0\xc20' def test_seal_meets_difficulty(): block_hash = '0xba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279' @@ -134,11 +135,11 @@ def test_solve_for_difficulty(): nonce, seal = bittensor.utils.solve_for_difficulty(block_hash, 1) assert nonce == 0 - assert seal == b'\xd6mKj\xd0\x00=?2<\xaa\xc6\xcf;\xfc1\xe9\x02\xaa\x8e\xa0Q\x16\x16U]\xfe\xaa\x18jU\xcd' + assert seal == b'\xe2d\xbc\x10Tu|\xd0nQ\x1f\x15wTd\xb0\x18\x8f\xc7\xe7:\x12\xc6>\\\xbe\xac\xc5/v\xa7\xce' nonce, seal = bittensor.utils.solve_for_difficulty(block_hash, 10) assert nonce == 2 - assert seal == b'\x8a\xa5\x0fA\xb1\n\xd0\xdea\x7f\x86rWq1\xa5|\x18\xd0\xc7\x81\xf4\x81\x03\xf9P\xc8\x19\xb9\x1f-\xcf' + assert seal == b'\x19\xf2H1mB3\xa3y\xda\xe7)\xc7P\x93t\xe5o\xbc$\x14sQ\x10\xc3M\xc6\x90M8vq' def test_solve_for_difficulty_fast(): block_hash = '0xba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279' @@ -149,19 +150,22 @@ def test_solve_for_difficulty_fast(): subtensor.substrate.get_block_hash = MagicMock( return_value=block_hash ) wallet = MagicMock() wallet.is_registered = MagicMock( return_value=False ) + num_proc: int = 1 - _, _, _, _, seal = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet ) + solution = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet, num_processes=num_proc ) + seal = solution.seal assert bittensor.utils.seal_meets_difficulty(seal, 1) subtensor.difficulty = 10 - _, _, _, _, seal = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet ) + solution = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet, num_processes=num_proc ) + seal = solution.seal assert bittensor.utils.seal_meets_difficulty(seal, 10) def test_solve_for_difficulty_fast_registered_already(): # tests if the registration stops after the first block of nonces for _ in range(10): - workblocks_before_is_registered = random.randint(2, 10) + workblocks_before_is_registered = random.randint(1, 4) # return False each work block but return True after a random number of blocks is_registered_return_values = [False for _ in range(workblocks_before_is_registered)] + [True] + [False, False] @@ -175,12 +179,9 @@ def test_solve_for_difficulty_fast_registered_already(): wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) # all arugments should return None to indicate an early return - a, b, c, d, e = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet, num_processes = 1, update_interval = 1000) - assert a is None - assert b is None - assert c is None - assert d is None - assert e is None + solution = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet, num_processes = 1, update_interval = 1000) + + assert solution is None # called every time until True assert wallet.is_registered.call_count == workblocks_before_is_registered + 1 @@ -193,13 +194,15 @@ def test_solve_for_difficulty_fast_missing_hash(): subtensor.substrate.get_block_hash = MagicMock( side_effect= [None, None] + [block_hash]*20) wallet = MagicMock() wallet.is_registered = MagicMock( return_value=False ) + num_proc: int = 1 - _, _, _, _, seal = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet ) - + solution = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet, num_processes=num_proc ) + seal = solution.seal assert bittensor.utils.seal_meets_difficulty(seal, 1) subtensor.difficulty = 10 - _, _, _, _, seal = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet ) + solution = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet, num_processes=num_proc ) + seal = solution.seal assert bittensor.utils.seal_meets_difficulty(seal, 10) def test_is_valid_ss58_address(): From 4629b828ca797082f9071eccb433fa935e6fed55 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Fri, 12 Aug 2022 13:55:19 -0400 Subject: [PATCH 02/28] [hotfix] fix flags for multiproc register limit (#876) * add dot get * add to subtensor args and defaults * remove dot get because in subtensor args * typo * fix test --- bittensor/_cli/__init__.py | 17 ----------------- bittensor/_cli/cli_impl.py | 2 +- bittensor/_subtensor/__init__.py | 7 +++++++ tests/integration_tests/test_cli.py | 4 ++-- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index 83fe2b8f26..765df518cb 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -571,23 +571,6 @@ def config() -> 'bittensor.config': help='''Set true to avoid prompting the user.''', default=False, ) - register_parser.add_argument( - '--num_processes', - '--num', - '-n', - dest='num_processes', - help="Number of processors to use for registration", - type=int, - default=None, - ) - register_parser.add_argument( - '--update_interval', - '-u', - dest='update_interval', - help="The number of nonces to process before checking for next block during registration", - type=int, - default=None, - ) bittensor.wallet.add_args( register_parser ) bittensor.subtensor.add_args( register_parser ) diff --git a/bittensor/_cli/cli_impl.py b/bittensor/_cli/cli_impl.py index 245b943885..54682b1b8f 100644 --- a/bittensor/_cli/cli_impl.py +++ b/bittensor/_cli/cli_impl.py @@ -239,7 +239,7 @@ def register( self ): """ wallet = bittensor.wallet( config = self.config ) subtensor = bittensor.subtensor( config = self.config ) - subtensor.register( wallet = wallet, prompt = not self.config.no_prompt, num_processes = self.config.num_processes, update_interval = self.config.update_interval ) + subtensor.register( wallet = wallet, prompt = not self.config.no_prompt, num_processes = self.config.subtensor.register.num_processes, update_interval = self.config.subtensor.register.update_interval ) def transfer( self ): r""" Transfer token of amount to destination. diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index 9819c897ed..88a88c7cdf 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -184,6 +184,9 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): help='''The subtensor endpoint flag. If set, overrides the --network flag. ''') parser.add_argument('--' + prefix_str + 'subtensor._mock', action='store_true', help='To turn on subtensor mocking for testing purposes.', default=bittensor.defaults.subtensor._mock) + + parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest='subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) + parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '-u', dest='subtensor.register.update_interval', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) except argparse.ArgumentError: # re-parsing arguments. pass @@ -197,6 +200,10 @@ def add_defaults(cls, defaults ): defaults.subtensor.chain_endpoint = os.getenv('BT_SUBTENSOR_CHAIN_ENDPOINT') if os.getenv('BT_SUBTENSOR_CHAIN_ENDPOINT') != None else None defaults.subtensor._mock = os.getenv('BT_SUBTENSOR_MOCK') if os.getenv('BT_SUBTENSOR_MOCK') != None else False + defaults.subtensor.register = bittensor.Config() + defaults.subtensor.register.num_processes = os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') if os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') != None else None # uses processor count by default within the function + defaults.subtensor.register.update_interval = os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') if os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') != None else 50_000 + @staticmethod def check_config( config: 'bittensor.Config' ): assert config.subtensor diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index 77269d366b..ef76720630 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -1082,8 +1082,8 @@ def test_register( self ): config = self.config config.subtensor._mock = True config.command = "register" - config.num_processes = 1 - config.update_interval = 50_000 + config.subtensor.register.num_processes = 1 + config.subtensor.register.update_interval = 50_000 config.subtensor.network = "mock" config.no_prompt = True From 224719caa1f105bc6b8d8896365d43c17416095f Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 15 Aug 2022 09:34:01 -0400 Subject: [PATCH 03/28] Fix/diff unpack bit shift (#878) * fix incorrect bit shift * move inner function out and add test for diff pack * fix test * fix call arg check in test * add assert * fix test for py37 * refactor the diff pack into two functions move the test to a unit test * fix test --- bittensor/utils/__init__.py | 35 ++++++++++++------- tests/integration_tests/test_subtensor.py | 13 ++++--- .../bittensor_tests/utils/test_utils.py | 14 ++++++++ 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index a5379e5428..29fe81f878 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -220,7 +220,7 @@ def run(self): with self.check_block: block_number = self.curr_block_num.value block_bytes = bytes(self.curr_block) - block_difficulty = int(self.curr_diff[0] >> 32 | self.curr_diff[1]) + block_difficulty = registration_diff_unpack(self.curr_diff) self.newBlockEvent.clear() # reset nonces to start from random point @@ -270,6 +270,25 @@ def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, bloc return None, time.time() - start +def registration_diff_unpack(packed_diff: multiprocessing.Array) -> int: + """Unpacks the packed two 32-bit integers into one 64-bit integer. Little endian.""" + return int(packed_diff[0] << 32 | packed_diff[1]) + + +def registration_diff_pack(diff: int, packed_diff: multiprocessing.Array): + """Packs the difficulty into two 32-bit integers. Little endian.""" + packed_diff[0] = diff >> 32 + packed_diff[1] = diff & 0xFFFFFFFF # low 32 bits + + +def update_curr_block(curr_diff: multiprocessing.Array, curr_block: multiprocessing.Array, curr_block_num: multiprocessing.Value, block_number: int, block_bytes: bytes, diff: int, lock: multiprocessing.Lock): + with lock: + curr_block_num.value = block_number + for i in range(64): + curr_block[i] = block_bytes[i] + registration_diff_pack(diff, curr_diff) + + def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = None, update_interval: Optional[int] = None ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. @@ -306,15 +325,7 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = curr_block = multiprocessing.Array('h', 64, lock=True) # byte array curr_block_num = multiprocessing.Value('i', 0, lock=True) # int curr_diff = multiprocessing.Array('Q', [0, 0], lock=True) # [high, low] - - def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: multiprocessing.Lock): - with lock: - curr_block_num.value = block_number - for i in range(64): - curr_block[i] = block_bytes[i] - curr_diff[0] = diff >> 32 - curr_diff[1] = diff & 0xFFFFFFFF # low 32 bits - + status.start() # Establish communication queues @@ -339,7 +350,7 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu block_bytes = block_hash.encode('utf-8')[2:] old_block_number = block_number # Set to current block - update_curr_block(block_number, block_bytes, difficulty, check_block) + update_curr_block(curr_diff, curr_block, curr_block_num, block_number, block_bytes, difficulty, check_block) # Set new block events for each solver to start for w in solvers: @@ -373,7 +384,7 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu block_bytes = block_hash.encode('utf-8')[2:] difficulty = subtensor.difficulty - update_curr_block(block_number, block_bytes, difficulty, check_block) + update_curr_block(curr_diff, curr_block, curr_block_num, block_number, block_bytes, difficulty, check_block) # Set new block events for each solver for w in solvers: w.newBlockEvent.set() diff --git a/tests/integration_tests/test_subtensor.py b/tests/integration_tests/test_subtensor.py index 2d411f1fc0..203c092770 100644 --- a/tests/integration_tests/test_subtensor.py +++ b/tests/integration_tests/test_subtensor.py @@ -15,17 +15,17 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -from typing import DefaultDict -from unittest import mock + +import multiprocessing from unittest.mock import patch import bittensor import pytest -import psutil import unittest import time import random from unittest.mock import MagicMock from bittensor.utils.balance import Balance +from bittensor.utils import Solver, update_curr_block from substrateinterface import Keypair from bittensor._subtensor.subtensor_mock import mock_subtensor class TestSubtensor(unittest.TestCase): @@ -411,11 +411,10 @@ def process_events(self): self.subtensor.substrate.submit_extrinsic = MagicMock(return_value = success()) # should return True - assert self.subtensor.register(wallet=wallet,) + assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5) == True # calls until True and once again before exiting subtensor class # This assertion is currently broken when difficulty is too low - #assert wallet.is_registered.call_count == workblocks_before_is_registered + 2 - + #assert wallet.is_registered.call_count == workblocks_before_is_registered + 2 def test_registration_partly_failed( self ): class failed(): @@ -448,7 +447,7 @@ def process_events(self): self.subtensor.substrate.submit_extrinsic = MagicMock(side_effect = submit_extrinsic) # should return True - assert self.subtensor.register(wallet=wallet,) == True + assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5) == True def test_registration_failed( self ): class failed(): diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 03065ccdca..ba7249c9dd 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -8,6 +8,7 @@ import os import random import torch +import multiprocessing from sys import platform from substrateinterface.base import Keypair @@ -233,5 +234,18 @@ def test_is_valid_ed25519_pubkey(): assert bittensor.utils.is_valid_ed25519_pubkey(good_pubkey) assert not bittensor.utils.is_valid_ed25519_pubkey(bad_pubkey) +def test_registration_diff_pack_unpack(): + fake_diff = pow(2, 31)# this is under 32 bits + + mock_diff = multiprocessing.Array('Q', [0, 0], lock=True) # [high, low] + + bittensor.utils.registration_diff_pack(fake_diff, mock_diff) + assert bittensor.utils.registration_diff_unpack(mock_diff) == fake_diff + + fake_diff = pow(2, 32) * pow(2, 4) # this should be too large if the bit shift is wrong (32 + 4 bits) + + bittensor.utils.registration_diff_pack(fake_diff, mock_diff) + assert bittensor.utils.registration_diff_unpack(mock_diff) == fake_diff + if __name__ == "__main__": test_solve_for_difficulty_fast_registered_already() \ No newline at end of file From c546d3ffe221b9e3edf8e241da58abea990d2520 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 15 Aug 2022 12:28:04 -0400 Subject: [PATCH 04/28] [Feature] [cubit] CUDA registration solver (#868) * added cuda solver * boost versions to fix pip error * allow choosing device id * fix solution check to use keccak * adds params for cuda and dev_id to register * list devices by name during selection * add block number logging * fix calculation of hashrate * fix update interval default * add --TPB arg to register * add update_interval flag * switch back to old looping/work structure * change typing * device count is a function * stop early if wallet registered * add update interval and num proc flag * add better number output * optimize multiproc cpu reg keeping proc until solution * fix test * change import to cubit * fix import and default * up default should have default in CLI call * add comments about params * fix config var access * add cubit as extra * handle stale pow differently check registration after failure * restrict number of processes for integration test * fix stale check * use wallet.is_registered instead * attempt to fix test issue * fix my test * oops typo * typo again ugh * remove print out * fix partly reg test * fix if solution None * fix test? * fix patch * add args for cuda to subtensor * add cuda args to reregister call * add to wallet register the cuda args * fix refs and tests * add for val test also * fix tests with rereg * fix patch for tests * add mock_register to subtensor passed instead * move register under the check for isregistered * use patch obj instead * fit patch object --- bittensor/_cli/__init__.py | 34 ++++- bittensor/_cli/cli_impl.py | 10 +- bittensor/_dataset/dataset_impl.py | 3 +- bittensor/_subtensor/__init__.py | 12 +- bittensor/_subtensor/subtensor_impl.py | 105 +++++++++----- bittensor/_wallet/wallet_impl.py | 77 +++++++--- bittensor/utils/__init__.py | 136 ++++++++++++++++-- bittensor/utils/register_cuda.py | 87 +++++++++++ requirements.txt | 2 +- setup.py | 3 + tests/integration_tests/test_subtensor.py | 40 +++--- .../unit_tests/bittensor_tests/test_neuron.py | 68 ++++++--- 12 files changed, 461 insertions(+), 116 deletions(-) create mode 100644 bittensor/utils/register_cuda.py diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index 765df518cb..599b4bd134 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -18,15 +18,18 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import argparse import os import sys -import argparse +from typing import List import bittensor -from rich.prompt import Prompt -from rich.prompt import Confirm +import torch +from rich.prompt import Confirm, Prompt from substrateinterface.utils.ss58 import ss58_decode, ss58_encode + from . import cli_impl + console = bittensor.__console__ class cli: @@ -250,6 +253,7 @@ def config() -> 'bittensor.config': 'register', help='''Register a wallet to a network.''' ) + unstake_parser = cmd_parsers.add_parser( 'unstake', help='''Unstake from hotkey accounts.''' @@ -802,6 +806,28 @@ def check_register_config( config: 'bittensor.Config' ): hotkey = Prompt.ask("Enter hotkey name", default = bittensor.defaults.wallet.hotkey) config.wallet.hotkey = str(hotkey) + if not config.no_prompt and config.subtensor.register.cuda.use_cuda == bittensor.defaults.subtensor.register.cuda.use_cuda: + # Ask about cuda registration only if a CUDA device is available. + if torch.cuda.is_available(): + cuda = Confirm.ask("Would you like to try CUDA registration?\n") + config.subtensor.register.cuda.use_cuda = cuda + # Only ask about which CUDA device if the user has more than one CUDA device. + if cuda and config.subtensor.register.cuda.dev_id == bittensor.defaults.subtensor.register.cuda.dev_id and torch.cuda.device_count() > 0: + devices: List[str] = [str(x) for x in range(torch.cuda.device_count())] + device_names: List[str] = [torch.cuda.get_device_name(x) for x in range(torch.cuda.device_count())] + console.print("Available CUDA devices:") + choices_str: str = "" + for i, device in enumerate(devices): + choices_str += (" {}: {}\n".format(device, device_names[i])) + console.print(choices_str) + dev_id = Prompt.ask("Which GPU would you like to use?", choices=devices, default=str(bittensor.defaults.subtensor.register.cuda.dev_id)) + try: + dev_id = int(dev_id) + except ValueError: + console.error(":cross_mark:[red]Invalid GPU device[/red] [bold white]{}[/bold white]\nAvailable CUDA devices:{}".format(dev_id, choices_str)) + sys.exit(1) + config.subtensor.register.cuda.dev_id = dev_id + def check_new_coldkey_config( config: 'bittensor.Config' ): if config.wallet.get('name') == bittensor.defaults.wallet.name and not config.no_prompt: wallet_name = Prompt.ask("Enter wallet name", default = bittensor.defaults.wallet.name) @@ -885,4 +911,4 @@ def check_help_config( config: 'bittensor.Config'): def check_update_config( config: 'bittensor.Config'): if not config.no_prompt: answer = Prompt.ask('This will update the local bittensor package', choices = ['Y','N'], default = 'Y') - config.answer = answer \ No newline at end of file + config.answer = answer diff --git a/bittensor/_cli/cli_impl.py b/bittensor/_cli/cli_impl.py index 54682b1b8f..26c9eadc08 100644 --- a/bittensor/_cli/cli_impl.py +++ b/bittensor/_cli/cli_impl.py @@ -239,7 +239,15 @@ def register( self ): """ wallet = bittensor.wallet( config = self.config ) subtensor = bittensor.subtensor( config = self.config ) - subtensor.register( wallet = wallet, prompt = not self.config.no_prompt, num_processes = self.config.subtensor.register.num_processes, update_interval = self.config.subtensor.register.update_interval ) + subtensor.register( + wallet = wallet, + prompt = not self.config.no_prompt, + TPB = self.config.subtensor.register.cuda.get('TPB', None), + update_interval = self.config.subtensor.register.get('update_interval', None), + num_processes = self.config.subtensor.register.get('num_processes', None), + cuda = self.config.subtensor.register.cuda.get('use_cuda', None), + dev_id = self.config.subtensor.register.cuda.get('dev_id', None) + ) def transfer( self ): r""" Transfer token of amount to destination. diff --git a/bittensor/_dataset/dataset_impl.py b/bittensor/_dataset/dataset_impl.py index f86f0c7aff..f104b632bf 100644 --- a/bittensor/_dataset/dataset_impl.py +++ b/bittensor/_dataset/dataset_impl.py @@ -160,8 +160,7 @@ def __init__( self.build_hash_table() - if not os.path.isdir(os.path.expanduser(data_dir)): - os.makedirs(os.path.expanduser(data_dir)) + os.makedirs(os.path.expanduser(data_dir), exist_ok=True) self.data_queue = ThreadQueue( producer_target = self.reserve_multiple_data, diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index 88a88c7cdf..0d53fda8b2 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -186,7 +186,12 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument('--' + prefix_str + 'subtensor._mock', action='store_true', help='To turn on subtensor mocking for testing purposes.', default=bittensor.defaults.subtensor._mock) parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest='subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) - parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '-u', dest='subtensor.register.update_interval', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) + parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', dest='subtensor.register.update_interval', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) + # registration args. Used for register and re-register and anything that calls register. + parser.add_argument( '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', '--' + prefix_str + 'subtensor.register.cuda.use_cuda', dest=f'{prefix_str}subtensor.register.cuda.use_cuda', default=bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set true to use CUDA.''', action='store_true', required=False ) + parser.add_argument( '--' + prefix_str + 'cuda.dev_id', '--' + prefix_str + 'subtensor.register.cuda.dev_id', dest=f'{prefix_str}subtensor.register.cuda.dev_id', type=int, default=bittensor.defaults.subtensor.register.cuda.dev_id, help='''Set the CUDA device id. Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) + parser.add_argument( '--' + prefix_str + 'cuda.TPB', '--' + prefix_str + 'subtensor.register.cuda.TPB', dest=f'{prefix_str}subtensor.register.cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) + except argparse.ArgumentError: # re-parsing arguments. pass @@ -204,6 +209,11 @@ def add_defaults(cls, defaults ): defaults.subtensor.register.num_processes = os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') if os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') != None else None # uses processor count by default within the function defaults.subtensor.register.update_interval = os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') if os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') != None else 50_000 + defaults.subtensor.register.cuda = bittensor.Config() + defaults.subtensor.register.cuda.dev_id = 0 + defaults.subtensor.register.cuda.use_cuda = False + defaults.subtensor.register.cuda.TPB = 256 + @staticmethod def check_config( config: 'bittensor.Config' ): assert config.subtensor diff --git a/bittensor/_subtensor/subtensor_impl.py b/bittensor/_subtensor/subtensor_impl.py index e71fcca7fa..c58126facd 100644 --- a/bittensor/_subtensor/subtensor_impl.py +++ b/bittensor/_subtensor/subtensor_impl.py @@ -15,8 +15,9 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. import torch -from rich.prompt import Confirm +from rich.prompt import Confirm, Prompt from typing import List, Dict, Union, Optional +from multiprocessing import Process import bittensor from tqdm import tqdm @@ -440,6 +441,9 @@ def register ( wait_for_finalization: bool = True, prompt: bool = False, max_allowed_attempts: int = 3, + cuda: bool = False, + dev_id: int = 0, + TPB: int = 256, num_processes: Optional[int] = None, update_interval: Optional[int] = None, ) -> bool: @@ -455,6 +459,18 @@ def register ( or returns false if the extrinsic fails to be finalized within the timeout. prompt (bool): If true, the call waits for confirmation from the user before proceeding. + max_allowed_attempts (int): + Maximum number of attempts to register the wallet. + cuda (bool): + If true, the wallet should be registered on the cuda device. + dev_id (int): + The cuda device id. + TPB (int): + The number of threads per block (cuda). + num_processes (int): + The number of processes to use to register. + update_interval (int): + The number of nonces to solve between updates. Returns: success (bool): flag is true if extrinsic was finalized or uncluded in the block. @@ -475,7 +491,14 @@ def register ( attempts = 1 while True: # Solve latest POW. - pow_result = bittensor.utils.create_pow( self, wallet, num_processes=num_processes, update_interval=update_interval ) + if cuda: + if not torch.cuda.is_available(): + if prompt: + bittensor.__console__.error('CUDA is not available.') + return False + pow_result = bittensor.utils.create_pow( self, wallet, cuda, dev_id, TPB, num_processes=num_processes, update_interval=update_interval ) + else: + pow_result = bittensor.utils.create_pow( self, wallet, num_processes=num_processes, update_interval=update_interval) with bittensor.__console__.status(":satellite: Registering...({}/{})".format(attempts,max_allowed_attempts)) as status: # pow failed @@ -488,52 +511,62 @@ def register ( # pow successful, proceed to submit pow to chain for registration else: #check if pow result is still valid - while pow_result['block_number'] >= self.get_current_block() - 3: - with self.substrate as substrate: - # create extrinsic call - call = substrate.compose_call( - call_module='SubtensorModule', - call_function='register', - call_params={ - 'block_number': pow_result['block_number'], - 'nonce': pow_result['nonce'], - 'work': bittensor.utils.hex_bytes_to_u8_list( pow_result['work'] ), - 'hotkey': wallet.hotkey.ss58_address, - 'coldkey': wallet.coldkeypub.ss58_address - } - ) - extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.hotkey ) - response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization ) - - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") + if pow_result['block_number'] < self.get_current_block() - 3: + bittensor.__console__.print( "[red]POW is stale.[/red]" ) + continue + + with self.substrate as substrate: + # create extrinsic call + call = substrate.compose_call( + call_module='SubtensorModule', + call_function='register', + call_params={ + 'block_number': pow_result['block_number'], + 'nonce': pow_result['nonce'], + 'work': bittensor.utils.hex_bytes_to_u8_list( pow_result['work'] ), + 'hotkey': wallet.hotkey.ss58_address, + 'coldkey': wallet.coldkeypub.ss58_address + } + ) + extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.hotkey ) + response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization ) + + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") + return True + + # process if registration successful, try again if pow is still valid + response.process_events() + if not response.is_success: + if 'key is already registered' in response.error_message: + # Error meant that the key is already registered. + bittensor.__console__.print(":white_heavy_check_mark: [green]Already Registered[/green]") return True - - # process if registration successful, try again if pow is still valid - response.process_events() - if not response.is_success: - bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) - time.sleep(1) - continue - - # Successful registration, final check for neuron and pubkey - else: - bittensor.__console__.print(":satellite: Checking Balance...") - neuron = self.neuron_for_pubkey( wallet.hotkey.ss58_address ) + + bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) + time.sleep(1) + + # Successful registration, final check for neuron and pubkey + else: + bittensor.__console__.print(":satellite: Checking Balance...") + if wallet.is_registered( self ): bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") return True + else: + # neuron not found, try again + bittensor.__console__.print(":cross_mark: [red]Unknown error. Neuron not found.[/red]") + return False #Failed registration, retry pow attempts += 1 if attempts > max_allowed_attempts: bittensor.__console__.print( "[red]No more attempts.[/red]" ) - return False + return False else: status.update( ":satellite: Failed registration, retrying pow ...({}/{})".format(attempts, max_allowed_attempts)) continue - def serve ( self, wallet: 'bittensor.wallet', diff --git a/bittensor/_wallet/wallet_impl.py b/bittensor/_wallet/wallet_impl.py index 158f8d211f..993b09930a 100644 --- a/bittensor/_wallet/wallet_impl.py +++ b/bittensor/_wallet/wallet_impl.py @@ -245,33 +245,76 @@ def reregister( # Check if the wallet should reregister if not self.config.wallet.get('reregister'): sys.exit(0) - return self.register(subtensor=subtensor, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, prompt=prompt) + + subtensor.register( + wallet = self, + prompt = prompt, + TPB = self.config.subtensor.register.cuda.get('TPB', None), + update_interval = self.config.subtensor.register.cuda.get('update_interval', None), + num_processes = self.config.subtensor.register.get('num_processes', None), + cuda = self.config.subtensor.register.cuda.get('use_cuda', None), + dev_id = self.config.subtensor.register.cuda.get('dev_id', None), + wait_for_inclusion = wait_for_inclusion, + wait_for_finalization = wait_for_finalization, + ) + + return self def register ( self, subtensor: 'bittensor.Subtensor' = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = True, - prompt: bool = False + prompt: bool = False, + max_allowed_attempts: int = 3, + cuda: bool = False, + dev_id: int = 0, + TPB: int = 256, + num_processes: Optional[int] = None, + update_interval: Optional[int] = None, ) -> 'bittensor.Wallet': - """ Registers this wallet on the chain. - Args: - wait_for_inclusion (bool): - if set, waits for the extrinsic to enter a block before returning true, - or returns false if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): - if set, waits for the extrinsic to be finalized on the chain before returning true, - or returns false if the extrinsic fails to be finalized within the timeout. - subtensor( 'bittensor.Subtensor' ): - Bittensor subtensor connection. Overrides with defaults if None. - prompt (bool): - If true, the call waits for confirmation from the user before proceeding. - Return: - This wallet. + """ Registers the wallet to chain. + Args: + subtensor( 'bittensor.Subtensor' ): + Bittensor subtensor connection. Overrides with defaults if None. + wait_for_inclusion (bool): + If set, waits for the extrinsic to enter a block before returning true, + or returns false if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): + If set, waits for the extrinsic to be finalized on the chain before returning true, + or returns false if the extrinsic fails to be finalized within the timeout. + prompt (bool): + If true, the call waits for confirmation from the user before proceeding. + max_allowed_attempts (int): + Maximum number of attempts to register the wallet. + cuda (bool): + If true, the wallet should be registered on the cuda device. + dev_id (int): + The cuda device id. + TPB (int): + The number of threads per block (cuda). + num_processes (int): + The number of processes to use to register. + update_interval (int): + The number of nonces to solve between updates. + Returns: + success (bool): + flag is true if extrinsic was finalized or uncluded in the block. + If we did not wait for finalization / inclusion, the response is true. """ # Get chain connection. if subtensor == None: subtensor = bittensor.subtensor() - subtensor.register( wallet = self, wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization, prompt=prompt ) + subtensor.register( + wallet = self, + wait_for_inclusion = wait_for_inclusion, + wait_for_finalization = wait_for_finalization, + prompt=prompt, max_allowed_attempts=max_allowed_attempts, + cuda=cuda, + dev_id=dev_id, + TPB=TPB, + num_processes=num_processes, + update_interval=update_interval + ) return self diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 29fe81f878..195f9872f6 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -1,4 +1,5 @@ import binascii +import datetime import hashlib import math import multiprocessing @@ -15,9 +16,11 @@ import requests import torch from Crypto.Hash import keccak -from substrateinterface import Keypair +from substrateinterface import Keypair, KeypairType from substrateinterface.utils import ss58 +from .register_cuda import reset_cuda, solve_cuda + def indexed_values_to_dataframe ( prefix: Union[str, int], @@ -257,7 +260,6 @@ def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, bloc # Check if seal meets difficulty product = seal_number * difficulty if product < limit: - print(f"{solver.proc_num} found a solution: {nonce}, {block_number}, {str(block_bytes)}, {str(seal)}, {difficulty}") # Found a solution, save it. return POWSolution(nonce, block_number, difficulty, seal), time.time() - start @@ -416,7 +418,7 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = except Empty: break - + message = f"""Solving time spent: {time.time() - start_time} Difficulty: [bold white]{millify(difficulty)}[/bold white] @@ -424,23 +426,131 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = Block: [bold white]{block_number}[/bold white] Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white] Best: [bold white]{binascii.hexlify(bytes(best_seal) if best_seal else bytes(0))}[/bold white]""" - status.update(message.replace(" ", "")) - + status.update(message.replace(" ", "")) + # exited while, solution contains the nonce or wallet is registered stopEvent.set() # stop all other processes status.stop() return solution + +def get_human_readable(num, suffix="H"): + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(num) < 1000.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1000.0 + return f"{num:.1f}Y{suffix}" + +def millify(n: int): + millnames = ['',' K',' M',' B',' T'] + n = float(n) + millidx = max(0,min(len(millnames)-1, + int(math.floor(0 if n == 0 else math.log10(abs(n))/3)))) + + return '{:.0f}{}'.format(n / 10**(3 * millidx), millnames[millidx]) + +def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: int = 0 ) -> Optional[POWSolution]: + """ + Solves the registration fast using CUDA + Args: + subtensor: bittensor.Subtensor + The subtensor node to grab blocks + wallet: bittensor.Wallet + The wallet to register + update_interval: int + The number of nonces to try before checking for more blocks + TPB: int + The number of threads per block. CUDA param that should match the GPU capability + dev_id: int + The CUDA device ID to execute the registration on + """ + if not torch.cuda.is_available(): + raise Exception("CUDA not available") + + if update_interval is None: + update_interval = 50_000 + + block_number = subtensor.get_current_block() + difficulty = subtensor.difficulty + block_hash = subtensor.substrate.get_block_hash( block_number ) + while block_hash == None: + block_hash = subtensor.substrate.get_block_hash( block_number ) + block_bytes = block_hash.encode('utf-8')[2:] + + nonce = 0 + limit = int(math.pow(2,256)) - 1 + start_time = time.time() + + console = bittensor.__console__ + status = console.status("Solving") + + solution = -1 + start_time = time.time() + interval_time = start_time + status.start() + while solution == -1 and not wallet.is_registered(subtensor): + solution, seal = solve_cuda(nonce, + update_interval, + TPB, + block_bytes, + block_number, + difficulty, + limit, + dev_id) + + if (solution != -1): + # Attempt to reset CUDA device + reset_cuda() + status.stop() + new_bn = subtensor.get_current_block() + print(f"Found solution for bn: {block_number}; Newest: {new_bn}") + return POWSolution(solution, block_number, difficulty, seal) + + nonce += (TPB * update_interval) + if (nonce >= int(math.pow(2,63))): + nonce = 0 + itrs_per_sec = (TPB * update_interval) / (time.time() - interval_time) + interval_time = time.time() + difficulty = subtensor.difficulty + block_number = subtensor.get_current_block() + block_hash = subtensor.substrate.get_block_hash( block_number) + while block_hash == None: + block_hash = subtensor.substrate.get_block_hash( block_number) + block_bytes = block_hash.encode('utf-8')[2:] + + message = f"""Solving + time spent: {datetime.timedelta(seconds=time.time() - start_time)} + Nonce: [bold white]{nonce}[/bold white] + Difficulty: [bold white]{millify(difficulty)}[/bold white] + Iters: [bold white]{get_human_readable(int(itrs_per_sec), "H")}/s[/bold white] + Block: [bold white]{block_number}[/bold white] + Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white]""" + status.update(message.replace(" ", "")) + + # exited while, found_solution contains the nonce or wallet is registered + if solution == -1: # didn't find solution + reset_cuda() + status.stop() + return None + + else: + reset_cuda() + # Shouldn't get here + status.stop() + return None + +def create_pow( subtensor, wallet, cuda: bool = False, dev_id: int = 0, tpb: int = 256, num_processes: int = None, update_interval: int = None ) -> Optional[Dict[str, Any]]: + if cuda: + solution: POWSolution = solve_for_difficulty_fast_cuda( subtensor, wallet, dev_id=dev_id, TPB=tpb, update_interval=update_interval ) + else: + solution: POWSolution = solve_for_difficulty_fast( subtensor, wallet, num_processes=num_processes, update_interval=update_interval ) -def create_pow( subtensor, wallet, num_processes: Optional[int] = None, update_interval: Optional[int] = None ) -> Optional[Dict[str, Any]]: - solution: POWSolution = solve_for_difficulty_fast( subtensor, wallet, num_processes=num_processes, update_interval=update_interval ) - nonce, block_number, difficulty, seal = solution.nonce, solution.block_number, solution.difficulty, solution.seal - return None if nonce is None else { - 'nonce': nonce, - 'difficulty': difficulty, - 'block_number': block_number, - 'work': binascii.hexlify(seal) + return None if solution is None else { + 'nonce': solution.nonce, + 'difficulty': solution.difficulty, + 'block_number': solution.block_number, + 'work': binascii.hexlify(solution.seal) } def version_checking(): diff --git a/bittensor/utils/register_cuda.py b/bittensor/utils/register_cuda.py new file mode 100644 index 0000000000..f64f4777b4 --- /dev/null +++ b/bittensor/utils/register_cuda.py @@ -0,0 +1,87 @@ +import binascii +import hashlib +import math +from typing import Tuple + +import numpy as np +from Crypto.Hash import keccak + + +def solve_cuda(nonce_start: np.int64, update_interval: np.int64, TPB: int, block_bytes: bytes, bn: int, difficulty: int, limit: int, dev_id: int = 0) -> Tuple[np.int64, bytes]: + """ + Solves the PoW problem using CUDA. + Args: + nonce_start: int64 + Starting nonce. + update_interval: int64 + Number of nonces to solve before updating block information. + TPB: int + Threads per block. + block_bytes: bytes + Bytes of the block hash. 64 bytes. + difficulty: int256 + Difficulty of the PoW problem. + limit: int256 + Upper limit of the nonce. + dev_id: int (default=0) + The CUDA device ID + Returns: + Tuple[int64, bytes] + Tuple of the nonce and the seal corresponding to the solution. + Returns -1 for nonce if no solution is found. + """ + + try: + import cubit + except ImportError: + raise ImportError("Please install cubit") + + + upper = int(limit // difficulty) + + upper_bytes = upper.to_bytes(32, byteorder='little', signed=False) + + def seal_meets_difficulty( seal:bytes, difficulty:int ): + seal_number = int.from_bytes(seal, "big") + product = seal_number * difficulty + limit = int(math.pow(2,256))- 1 + + return product < limit + + def hex_bytes_to_u8_list( hex_bytes: bytes ): + hex_chunks = [int(hex_bytes[i:i+2], 16) for i in range(0, len(hex_bytes), 2)] + return hex_chunks + + def create_seal_hash( block_bytes:bytes, nonce:int ) -> bytes: + nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) + pre_seal = nonce_bytes + block_bytes + seal_sh256 = hashlib.sha256( bytearray(hex_bytes_to_u8_list(pre_seal)) ).digest() + kec = keccak.new(digest_bits=256) + seal = kec.update( seal_sh256 ).digest() + return seal + + # Call cython function + # int blockSize, uint64 nonce_start, uint64 update_interval, const unsigned char[:] limit, + # const unsigned char[:] block_bytes, int dev_id + solution = cubit.solve_cuda(TPB, nonce_start, update_interval, upper_bytes, block_bytes, dev_id) # 0 is first GPU + seal = None + if solution != -1: + print(f"Checking solution: {solution} for bn: {bn}") + seal = create_seal_hash(block_bytes, solution) + if seal_meets_difficulty(seal, difficulty): + return solution, seal + else: + return -1, b'\x00' * 32 + + return solution, seal + +def reset_cuda(): + """ + Resets the CUDA environment. + """ + try: + import cubit + except ImportError: + raise ImportError("Please install cubit") + + cubit.reset_cuda() diff --git a/requirements.txt b/requirements.txt index 1e2367e28e..392b7ec0c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,8 @@ cryptography>=3.1.1 idna>=2.10 jinja2>=3.0 fuzzywuzzy==0.18.0 +google-api-python-client>=2.6.0 python-levenshtein==0.12.1 -google-api-python-client grpcio==1.42.0 grpcio-tools==1.42.0 hypothesis>=6.47.4 diff --git a/setup.py b/setup.py index 4168052160..d4a9723bcd 100644 --- a/setup.py +++ b/setup.py @@ -69,4 +69,7 @@ 'Topic :: Software Development :: Libraries :: Python Modules', ], python_requires='>=3.7', + extras_requires={ + 'cubit': ['cubit>=1.0.5 @ git+https://github.com/opentensor/cubit.git'] + } ) diff --git a/tests/integration_tests/test_subtensor.py b/tests/integration_tests/test_subtensor.py index 203c092770..60639d7a28 100644 --- a/tests/integration_tests/test_subtensor.py +++ b/tests/integration_tests/test_subtensor.py @@ -402,19 +402,21 @@ def process_events(self): mock_neuron.is_null = True with patch('bittensor.Subtensor.difficulty'): + with patch('multiprocessing.queues.Queue.get') as mock_queue_get: + mock_queue_get.return_value = None - wallet = bittensor.wallet(_mock=True) - wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) + wallet = bittensor.wallet(_mock=True) + wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) - self.subtensor.difficulty= 1 - self.subtensor.neuron_for_pubkey = MagicMock( return_value=mock_neuron ) - self.subtensor.substrate.submit_extrinsic = MagicMock(return_value = success()) + self.subtensor.difficulty= 1 + self.subtensor.neuron_for_pubkey = MagicMock( return_value=mock_neuron ) + self.subtensor.substrate.submit_extrinsic = MagicMock(return_value = success()) - # should return True - assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5) == True - # calls until True and once again before exiting subtensor class - # This assertion is currently broken when difficulty is too low - #assert wallet.is_registered.call_count == workblocks_before_is_registered + 2 + # should return True + assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5 ) == True + # calls until True and once again before exiting subtensor class + # This assertion is currently broken when difficulty is too low + assert wallet.is_registered.call_count == workblocks_before_is_registered + 2 def test_registration_partly_failed( self ): class failed(): @@ -430,21 +432,21 @@ def __init__(self): def process_events(self): return True - is_registered_return_values = [False for _ in range(100)] - submit_extrinsic = [failed(), failed(), success()] + submit_extrinsic_mock = MagicMock( side_effect = [failed(), failed(), success()]) + + def is_registered_side_effect(*args, **kwargs): + nonlocal submit_extrinsic_mock + return submit_extrinsic_mock.call_count < 3 + current_block = [i for i in range(0,100)] - mock_neuron = MagicMock() - mock_neuron.is_null = True with patch('bittensor.Subtensor.difficulty'): wallet = bittensor.wallet(_mock=True) - wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) - + wallet.is_registered = MagicMock( side_effect=is_registered_side_effect ) - self.subtensor.difficulty= 1 + self.subtensor.difficulty = 1 self.subtensor.get_current_block = MagicMock(side_effect=current_block) - self.subtensor.neuron_for_pubkey = MagicMock( return_value=mock_neuron ) - self.subtensor.substrate.submit_extrinsic = MagicMock(side_effect = submit_extrinsic) + self.subtensor.substrate.submit_extrinsic = submit_extrinsic_mock # should return True assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5) == True diff --git a/tests/unit_tests/bittensor_tests/test_neuron.py b/tests/unit_tests/bittensor_tests/test_neuron.py index 02ee588d22..9fa79e7768 100644 --- a/tests/unit_tests/bittensor_tests/test_neuron.py +++ b/tests/unit_tests/bittensor_tests/test_neuron.py @@ -1,3 +1,4 @@ +from atexit import register from types import SimpleNamespace from unittest.mock import MagicMock, patch from more_itertools import side_effect @@ -67,6 +68,12 @@ def test_coreserver_reregister_flag_false_exit(): config.wallet = bittensor.Config() config.wallet.reregister = False # don't reregister the wallet + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + config.subtensor.register.cuda.use_cuda = False # don't use cuda on test + # No need to specify the other config options as they are default to None + mock_wallet = bittensor.wallet.mock() mock_wallet.config = config @@ -85,15 +92,15 @@ def exit_early(*args, **kwargs): metagraph=MagicMock(), spec=bittensor.neurons.core_server.neuron, subtensor=MagicMock( - network="mock" + network="mock", + register=mock_register ), config=config, ) - with patch.multiple( - 'bittensor.Wallet', - register=mock_register, - is_registered=MagicMock(return_value=False), # mock the wallet as not registered + with patch.object( + mock_wallet, + 'is_registered', MagicMock(return_value=False), # mock the wallet as not registered ): # Should exit without calling register @@ -113,6 +120,12 @@ def test_coreserver_reregister_flag_true(): config.wallet = bittensor.Config() config.wallet.reregister = True # try to reregister the wallet + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + config.subtensor.register.cuda.use_cuda = False # don't use cuda on test + # No need to specify the other config options as they are default to None + mock_wallet = bittensor.wallet.mock() mock_wallet.config = config @@ -131,17 +144,16 @@ def exit_early(*args, **kwargs): metagraph=MagicMock(), spec=bittensor.neurons.core_server.neuron, subtensor=MagicMock( - network="mock" + network="mock", + register=mock_register, ), config=config, ) - with patch.multiple( - 'bittensor.Wallet', - register=mock_register, - is_registered=MagicMock(return_value=False), # mock the wallet as not registered + with patch.object( + mock_wallet, + 'is_registered', MagicMock(return_value=False), # mock the wallet as not registered ): - # Should not exit with pytest.raises(MockException): # Should raise MockException @@ -157,6 +169,12 @@ def test_corevalidator_reregister_flag_false_exit(): config.wallet = bittensor.Config() config.wallet.reregister = False # don't reregister the wallet + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + config.subtensor.register.cuda.use_cuda = False # don't use cuda on test + # No need to specify the other config options as they are default to None + mock_wallet = bittensor.wallet.mock() mock_wallet.config = config @@ -172,17 +190,17 @@ def exit_early(*args, **kwargs): wallet=mock_wallet, spec=bittensor.neurons.core_validator.neuron, subtensor=MagicMock( - network="mock" + network="mock", + register=mock_register, ), config=config, ) - with patch.multiple( - 'bittensor.Wallet', - register=mock_register, - is_registered=MagicMock(return_value=False), # mock the wallet as not registered + with patch.object( + mock_wallet, + 'is_registered', MagicMock(return_value=False), # mock the wallet as not registered ): - + # Should exit without calling register with pytest.raises(SystemExit) as pytest_wrapped_e: # Should not raise MockException @@ -200,6 +218,12 @@ def test_corevalidator_reregister_flag_true(): config.wallet = bittensor.Config() config.wallet.reregister = True # try to reregister the wallet + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + config.subtensor.register.cuda.use_cuda = False # don't use cuda on test + # No need to specify the other config options as they are default to None + mock_wallet = bittensor.wallet.mock() mock_wallet.config = config @@ -215,15 +239,15 @@ def exit_early(*args, **kwargs): wallet=mock_wallet, spec=bittensor.neurons.core_validator.neuron, subtensor=MagicMock( - network="mock" + network="mock", + register=mock_register, ), config=config, ) - with patch.multiple( - 'bittensor.Wallet', - register=mock_register, - is_registered=MagicMock(return_value=False), # mock the wallet as not registered + with patch.object( + mock_wallet, + 'is_registered', MagicMock(return_value=False), # mock the wallet as not registered ): # Should not exit From fb6be7c306b0c099e549adf95621f4cec3d90483 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 15 Aug 2022 14:46:34 -0400 Subject: [PATCH 05/28] Fix/move overview args to cli (#867) * move cli args to CLI and fix overview * use dot get * fix tests * add hotkeys/all_hotkeys to (un)stake * fix default * fix default in unstake * add unstake multiple * add add stake multiple * move all/hotkeys back to wallet args * convert to balance first add catch for unstake multi * fix ref to wallet * fix test patch for multi hotkeys * try to fix tests * fix tests patch * fix mock wallet length * don't use new? * fix call args get * typo * fix typo --- bittensor/_cli/__init__.py | 25 +- bittensor/_cli/cli_impl.py | 55 ++-- bittensor/_subtensor/subtensor_impl.py | 351 +++++++++++++++++++++++++ bittensor/_wallet/__init__.py | 9 +- tests/integration_tests/test_cli.py | 77 +++--- 5 files changed, 438 insertions(+), 79 deletions(-) diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index 599b4bd134..97ccd03ead 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -91,6 +91,27 @@ def config() -> 'bittensor.config': help='''Set the output width of the overview. Defaults to automatic width from terminal.''', default=None, ) + overview_parser.add_argument( + '--sort_by', + '--wallet.sort_by', + dest='sort_by', + required=False, + action='store', + default="", + type=str, + help='''Sort the hotkeys by the specified column title (e.g. name, uid, axon).''' + ) + overview_parser.add_argument( + '--sort_order', + '--wallet.sort_order', + dest="sort_order", + required=False, + action='store', + default="ascending", + type=str, + help='''Sort the hotkeys in the specified ordering. (ascending/asc or descending/desc/reverse)''' + ) + bittensor.wallet.add_args( overview_parser ) bittensor.subtensor.add_args( overview_parser ) @@ -501,6 +522,7 @@ def config() -> 'bittensor.config': help='''Set true to avoid prompting the user.''', default=False, ) + bittensor.wallet.add_args( unstake_parser ) bittensor.subtensor.add_args( unstake_parser ) @@ -539,6 +561,7 @@ def config() -> 'bittensor.config': help='''Set true to avoid prompting the user.''', default=False, ) + bittensor.wallet.add_args( stake_parser ) bittensor.subtensor.add_args( stake_parser ) @@ -696,7 +719,7 @@ def check_unstake_config( config: 'bittensor.Config' ): if config.wallet.get('all_hotkeys'): hotkeys = "all hotkeys" elif config.wallet.get('hotkeys'): - hotkeys = str(config.hotkeys).replace('[', '').replace(']', '') + hotkeys = str(config.wallet.hotkeys).replace('[', '').replace(']', '') else: hotkeys = str(config.wallet.hotkey) if not Confirm.ask("Unstake all Tao from: [bold]'{}'[/bold]?".format(hotkeys)): diff --git a/bittensor/_cli/cli_impl.py b/bittensor/_cli/cli_impl.py index 26c9eadc08..bdce4358dc 100644 --- a/bittensor/_cli/cli_impl.py +++ b/bittensor/_cli/cli_impl.py @@ -17,7 +17,7 @@ import os import sys -from typing import List, Union +from typing import List, Union, Optional from cachetools import Cache @@ -259,7 +259,6 @@ def transfer( self ): def unstake( self ): r""" Unstake token of amount from hotkey(s). """ - # TODO: Implement this without re-unlocking the coldkey. config = self.config.copy() config.hotkey = None wallet = bittensor.wallet( config = self.config ) @@ -271,7 +270,7 @@ def unstake( self ): all_hotkeys: List[bittensor.wallet] = self._get_hotkey_wallets_for_wallet( wallet = wallet ) # Exclude hotkeys that are specified. wallets_to_unstake_from = [ - wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys') + wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys', []) ] elif self.config.wallet.get('hotkeys'): @@ -284,9 +283,7 @@ def unstake( self ): subtensor.unstake( wallet, amount = None if self.config.get('unstake_all') else self.config.get('amount'), wait_for_inclusion = True, prompt = not self.config.no_prompt ) return None - wallet_0: 'bittensor.wallet' = wallets_to_unstake_from[0] - # Decrypt coldkey for all wallet(s) to use - wallet_0.coldkey + final_wallets: List['bittensor.wallet'] = [] final_amounts: List[Union[float, Balance]] = [] @@ -295,9 +292,6 @@ def unstake( self ): if not wallet.is_registered(): # Skip unregistered hotkeys. continue - # Assign decrypted coldkey from wallet_0 - # so we don't have to decrypt again - wallet._coldkey = wallet_0._coldkey unstake_amount_tao: float = self.config.get('amount') if self.config.get('max_stake'): @@ -315,19 +309,17 @@ def unstake( self ): if not self.config.no_prompt: if not Confirm.ask("Do you want to unstake from the following keys:\n" + \ "".join([ - f" [bold white]- {wallet.hotkey_str}: {amount}𝜏[/bold white]\n" for wallet, amount in zip(final_wallets, final_amounts) + f" [bold white]- {wallet.hotkey_str}: {amount.tao}𝜏[/bold white]\n" for wallet, amount in zip(final_wallets, final_amounts) ]) ): return None - - for wallet, amount in zip(final_wallets, final_amounts): - subtensor.unstake( wallet, amount = None if self.config.get('unstake_all') else amount, wait_for_inclusion = True, prompt = False ) + + subtensor.unstake_multiple( wallets = final_wallets, amounts = None if self.config.get('unstake_all') else final_amounts, wait_for_inclusion = True, prompt = False ) def stake( self ): r""" Stake token of amount to hotkey(s). """ - # TODO: Implement this without re-unlocking the coldkey. config = self.config.copy() config.hotkey = None wallet = bittensor.wallet( config = config ) @@ -339,7 +331,7 @@ def stake( self ): all_hotkeys: List[bittensor.wallet] = self._get_hotkey_wallets_for_wallet( wallet = wallet ) # Exclude hotkeys that are specified. wallets_to_stake_to = [ - wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys') + wallet for wallet in all_hotkeys if wallet.hotkey_str not in self.config.wallet.get('hotkeys', []) ] elif self.config.wallet.get('hotkeys'): @@ -394,8 +386,7 @@ def stake( self ): ): return None - for wallet, amount in zip(final_wallets, final_amounts): - subtensor.add_stake( wallet, amount = None if self.config.get('stake_all') else amount, wait_for_inclusion = True, prompt = False ) + subtensor.add_stake_multiple( wallets = final_wallets, amounts = None if self.config.get('stake_all') else final_amounts, wait_for_inclusion = True, prompt = False ) def set_weights( self ): @@ -604,24 +595,14 @@ def overview(self): all_hotkeys = [] total_balance = bittensor.Balance(0) - - # We are printing for every wallet. + + # We are printing for every coldkey. if self.config.all: cold_wallets = CLI._get_coldkey_wallets_for_path(self.config.wallet.path) for cold_wallet in tqdm(cold_wallets, desc="Pulling balances"): if cold_wallet.coldkeypub_file.exists_on_device() and not cold_wallet.coldkeypub_file.is_encrypted(): total_balance = total_balance + subtensor.get_balance( cold_wallet.coldkeypub.ss58_address ) all_hotkeys = CLI._get_all_wallets_for_path( self.config.wallet.path ) - - # We are printing for a select number of hotkeys. - elif self.config.wallet.hotkeys: - # Only show hotkeys for wallets in the list - all_hotkeys = [hotkey for hotkey in all_hotkeys if hotkey.hotkey_str in self.config.wallet.hotkeys] - coldkey_wallet = bittensor.wallet( config = self.config ) - if coldkey_wallet.coldkeypub_file.exists_on_device() and not coldkey_wallet.coldkeypub_file.is_encrypted(): - total_balance = subtensor.get_balance( coldkey_wallet.coldkeypub.ss58_address ) - - # We are printing for all keys under the wallet. else: # We are only printing keys for a single coldkey coldkey_wallet = bittensor.wallet( config = self.config ) @@ -632,6 +613,16 @@ def overview(self): return all_hotkeys = CLI._get_hotkey_wallets_for_wallet( coldkey_wallet ) + # We are printing for a select number of hotkeys from all_hotkeys. + + if self.config.wallet.get('hotkeys', []): + if not self.config.get('all_hotkeys', False): + # We are only showing hotkeys that are specified. + all_hotkeys = [hotkey for hotkey in all_hotkeys if hotkey.hotkey_str in self.config.wallet.hotkeys] + else: + # We are excluding the specified hotkeys from all_hotkeys. + all_hotkeys = [hotkey for hotkey in all_hotkeys if hotkey.hotkey_str not in self.config.wallet.hotkeys] + # Check we have keys to display. if len(all_hotkeys) == 0: console.print("[red]No wallets found.[/red]") @@ -740,10 +731,10 @@ def overview(self): console.clear() - sort_by: str = self.config.wallet.sort_by - sort_order: str = self.config.wallet.sort_order + sort_by: Optional[str] = self.config.get('sort_by', None) + sort_order: Optional[str] = self.config.get('sort_order', None) - if sort_by != "": + if sort_by is not None and sort_by != "": column_to_sort_by: int = 0 highest_matching_ratio: int = 0 sort_descending: bool = False # Default sort_order to ascending diff --git a/bittensor/_subtensor/subtensor_impl.py b/bittensor/_subtensor/subtensor_impl.py index c58126facd..baac5d3b70 100644 --- a/bittensor/_subtensor/subtensor_impl.py +++ b/bittensor/_subtensor/subtensor_impl.py @@ -762,6 +762,197 @@ def add_stake( return False + def add_stake_multiple ( + self, + wallets: List['bittensor.wallet'], + amounts: List[Union[Balance, float]] = None, + wait_for_inclusion: bool = True, + wait_for_finalization: bool = False, + prompt: bool = False, + ) -> bool: + r""" Adds stake to each wallet hotkey in the list, using each amount, from the common coldkey. + Args: + wallets (List[bittensor.wallet]): + List of wallets to stake. + amounts (List[Union[Balance, float]]): + List of amounts to stake. If None, stake all to the first hotkey. + wait_for_inclusion (bool): + if set, waits for the extrinsic to enter a block before returning true, + or returns false if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): + if set, waits for the extrinsic to be finalized on the chain before returning true, + or returns false if the extrinsic fails to be finalized within the timeout. + prompt (bool): + If true, the call waits for confirmation from the user before proceeding. + Returns: + success (bool): + flag is true if extrinsic was finalized or included in the block. + flag is true if any wallet was staked. + If we did not wait for finalization / inclusion, the response is true. + """ + if not isinstance(wallets, list): + raise TypeError("wallets must be a list of bittensor.wallet") + + if len(wallets) == 0: + return True + + if amounts is not None and len(amounts) != len(wallets): + raise ValueError("amounts must be a list of the same length as wallets") + + if amounts is not None and not all(isinstance(amount, (Balance, float)) for amount in amounts): + raise TypeError("amounts must be a [list of bittensor.Balance or float] or None") + + if amounts is None: + amounts = [None] * len(wallets) + else: + # Convert to Balance + amounts = [bittensor.Balance.from_tao(amount) if isinstance(amount, float) else amount for amount in amounts ] + + if sum(amount.tao for amount in amounts) == 0: + # Staking 0 tao + return True + + wallet_0: 'bittensor.wallet' = wallets[0] + # Decrypt coldkey for all wallet(s) to use + wallet_0.coldkey + + neurons = [] + with bittensor.__console__.status(":satellite: Syncing with chain: [white]{}[/white] ...".format(self.network)): + old_balance = self.get_balance( wallet_0.coldkey.ss58_address ) + + for wallet in wallets: + neuron = self.neuron_for_pubkey( ss58_hotkey = wallet.hotkey.ss58_address ) + + if neuron.is_null: + neurons.append( None ) + continue + + neurons.append( neuron ) + + # Remove existential balance to keep key alive. + ## Keys must maintain a balance of at least 1000 rao to stay alive. + total_staking_rao = sum([amount.rao if amount is not None else 0 for amount in amounts]) + if total_staking_rao == 0: + # Staking all to the first wallet. + if old_balance.rao > 1000: + old_balance -= bittensor.Balance.from_rao(1000) + + elif total_staking_rao < 1000: + # Staking less than 1000 rao to the wallets. + pass + else: + # Staking more than 1000 rao to the wallets. + ## Reduce the amount to stake to each wallet to keep the balance above 1000 rao. + percent_reduction = 1 - (1000 / total_staking_rao) + amounts = [amount * percent_reduction for amount in amounts] + + successful_stakes = 0 + for wallet, amount, neuron in zip(wallets, amounts, neurons): + if neuron is None: + bittensor.__console__.print(":cross_mark: [red]Hotkey: {} is not registered. Skipping ...[/red]".format( wallet.hotkey_str )) + continue + + if wallet.coldkeypub.ss58_address != wallet_0.coldkeypub.ss58_address: + bittensor.__console__.print(":cross_mark: [red]Hotkey: {} is not under the same coldkey. Skipping ...[/red]".format( wallet.hotkey_str )) + continue + + # Assign decrypted coldkey from wallet_0 + # so we don't have to decrypt again + wallet._coldkey = wallet_0._coldkey + staking_all = False + # Convert to bittensor.Balance + if amount == None: + # Stake it all. + staking_balance = bittensor.Balance.from_tao( old_balance.tao ) + staking_all = True + + elif not isinstance(amount, bittensor.Balance ): + staking_balance = bittensor.Balance.from_tao( amount ) + else: + staking_balance = amount + + # Estimate staking fee. + stake_fee = None # To be filled. + with bittensor.__console__.status(":satellite: Estimating Staking Fees..."): + with self.substrate as substrate: + call = substrate.compose_call( + call_module='SubtensorModule', + call_function='add_stake', + call_params={ + 'hotkey': wallet.hotkey.ss58_address, + 'ammount_staked': staking_balance.rao + } + ) + payment_info = substrate.get_payment_info(call = call, keypair = wallet.coldkey) + if payment_info: + stake_fee = bittensor.Balance.from_rao(payment_info['partialFee']) + bittensor.__console__.print("[green]Estimated Fee: {}[/green]".format( stake_fee )) + else: + stake_fee = bittensor.Balance.from_tao( 0.2 ) + bittensor.__console__.print(":cross_mark: [red]Failed[/red]: could not estimate staking fee, assuming base fee of 0.2") + + # Check enough to stake + if staking_all: + staking_balance -= stake_fee + max(staking_balance, bittensor.Balance.from_tao(0)) + + if staking_balance > old_balance - stake_fee: + bittensor.__console__.print(":cross_mark: [red]Not enough balance[/red]: [green]{}[/green] to stake: [blue]{}[/blue] from coldkey: [white]{}[/white]".format(old_balance, staking_balance, wallet.name)) + continue + + # Ask before moving on. + if prompt: + if not Confirm.ask("Do you want to stake:\n[bold white] amount: {}\n hotkey: {}\n fee: {}[/bold white ]?".format( staking_balance, wallet.hotkey_str, stake_fee) ): + continue + + with bittensor.__console__.status(":satellite: Staking to chain: [white]{}[/white] ...".format(self.network)): + with self.substrate as substrate: + call = substrate.compose_call( + call_module='SubtensorModule', + call_function='add_stake', + call_params={ + 'hotkey': wallet.hotkey.ss58_address, + 'ammount_staked': staking_balance.rao + } + ) + extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.coldkey ) + response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") + old_balance -= staking_balance + stake_fee + successful_stakes += 1 + if staking_all: + # If staked all, no need to continue + break + + continue + + response.process_events() + if response.is_success: + bittensor.__console__.print(":white_heavy_check_mark: [green]Finalized[/green]") + else: + bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) + + if response.is_success: + block = self.get_current_block() + new_stake = bittensor.Balance.from_tao( self.neuron_for_uid( uid = neuron.uid, block = block ).stake) + new_balance = self.get_balance( wallet.coldkey.ss58_address ) + bittensor.__console__.print("Stake ({}): [blue]{}[/blue] :arrow_right: [green]{}[/green]".format( neuron.uid, neuron.stake, new_stake )) + old_balance = new_balance + successful_stakes += 1 + if staking_all: + # If staked all, no need to continue + break + + if successful_stakes != 0: + with bittensor.__console__.status(":satellite: Checking Balance on: ([white]{}[/white] ...".format(self.network)): + new_balance = self.get_balance( wallet.coldkey.ss58_address ) + bittensor.__console__.print("Balance: [blue]{}[/blue] :arrow_right: [green]{}[/green]".format( old_balance, new_balance )) + return True + + return False + def transfer( self, wallet: 'bittensor.wallet', @@ -990,6 +1181,166 @@ def unstake ( return True return False + + def unstake_multiple ( + self, + wallets: List['bittensor.wallet'], + amounts: List[Union[Balance, float]] = None, + wait_for_inclusion: bool = True, + wait_for_finalization: bool = False, + prompt: bool = False, + ) -> bool: + r""" Removes stake from each wallet hotkey in the list, using each amount, to their common coldkey. + Args: + wallets (List[bittensor.wallet]): + List of wallets to unstake. + amounts (List[Union[Balance, float]]): + List of amounts to unstake. If None, unstake all. + wait_for_inclusion (bool): + if set, waits for the extrinsic to enter a block before returning true, + or returns false if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): + if set, waits for the extrinsic to be finalized on the chain before returning true, + or returns false if the extrinsic fails to be finalized within the timeout. + prompt (bool): + If true, the call waits for confirmation from the user before proceeding. + Returns: + success (bool): + flag is true if extrinsic was finalized or included in the block. + flag is true if any wallet was unstaked. + If we did not wait for finalization / inclusion, the response is true. + """ + if not isinstance(wallets, list): + raise TypeError("wallets must be a list of bittensor.wallet") + + if len(wallets) == 0: + return True + + if amounts is not None and len(amounts) != len(wallets): + raise ValueError("amounts must be a list of the same length as wallets") + + if amounts is not None and not all(isinstance(amount, (Balance, float)) for amount in amounts): + raise TypeError("amounts must be a [list of bittensor.Balance or float] or None") + + if amounts is None: + amounts = [None] * len(wallets) + else: + # Convert to Balance + amounts = [bittensor.Balance.from_tao(amount) if isinstance(amount, float) else amount for amount in amounts ] + + if sum(amount.tao for amount in amounts) == 0: + # Staking 0 tao + return True + + + wallet_0: 'bittensor.wallet' = wallets[0] + # Decrypt coldkey for all wallet(s) to use + wallet_0.coldkey + + neurons = [] + with bittensor.__console__.status(":satellite: Syncing with chain: [white]{}[/white] ...".format(self.network)): + old_balance = self.get_balance( wallet_0.coldkey.ss58_address ) + + for wallet in wallets: + neuron = self.neuron_for_pubkey( ss58_hotkey = wallet.hotkey.ss58_address ) + + if neuron.is_null: + neurons.append( None ) + continue + + neurons.append( neuron ) + + successful_unstakes = 0 + for wallet, amount, neuron in zip(wallets, amounts, neurons): + if neuron is None: + bittensor.__console__.print(":cross_mark: [red]Hotkey: {} is not registered. Skipping ...[/red]".format( wallet.hotkey_str )) + continue + + if wallet.coldkeypub.ss58_address != wallet_0.coldkeypub.ss58_address: + bittensor.__console__.print(":cross_mark: [red]Hotkey: {} is not under the same coldkey. Skipping ...[/red]".format( wallet.hotkey_str )) + continue + + # Assign decrypted coldkey from wallet_0 + # so we don't have to decrypt again + wallet._coldkey = wallet_0._coldkey + + # Covert to bittensor.Balance + if amount == None: + # Unstake it all. + unstaking_balance = bittensor.Balance.from_tao( neuron.stake ) + elif not isinstance(amount, bittensor.Balance ): + unstaking_balance = bittensor.Balance.from_tao( amount ) + else: + unstaking_balance = amount + + # Check enough to unstake. + stake_on_uid = bittensor.Balance.from_tao( neuron.stake ) + if unstaking_balance > stake_on_uid: + bittensor.__console__.print(":cross_mark: [red]Not enough stake[/red]: [green]{}[/green] to unstake: [blue]{}[/blue] from hotkey: [white]{}[/white]".format(stake_on_uid, unstaking_balance, wallet.hotkey_str)) + continue + + # Estimate unstaking fee. + unstake_fee = None # To be filled. + with bittensor.__console__.status(":satellite: Estimating Staking Fees..."): + with self.substrate as substrate: + call = substrate.compose_call( + call_module='SubtensorModule', + call_function='remove_stake', + call_params={ + 'hotkey': wallet.hotkey.ss58_address, + 'ammount_unstaked': unstaking_balance.rao + } + ) + payment_info = substrate.get_payment_info(call = call, keypair = wallet.coldkey) + if payment_info: + unstake_fee = bittensor.Balance.from_rao(payment_info['partialFee']) + bittensor.__console__.print("[green]Estimated Fee: {}[/green]".format( unstake_fee )) + else: + unstake_fee = bittensor.Balance.from_tao( 0.2 ) + bittensor.__console__.print(":cross_mark: [red]Failed[/red]: could not estimate staking fee, assuming base fee of 0.2") + + # Ask before moving on. + if prompt: + if not Confirm.ask("Do you want to unstake:\n[bold white] amount: {}\n hotkey: {}\n fee: {}[/bold white ]?".format( unstaking_balance, wallet.hotkey_str, unstake_fee) ): + continue + + with bittensor.__console__.status(":satellite: Unstaking from chain: [white]{}[/white] ...".format(self.network)): + with self.substrate as substrate: + call = substrate.compose_call( + call_module='SubtensorModule', + call_function='remove_stake', + call_params={ + 'hotkey': wallet.hotkey.ss58_address, + 'ammount_unstaked': unstaking_balance.rao + } + ) + extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.coldkey ) + response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") + successful_unstakes += 1 + continue + + response.process_events() + if response.is_success: + bittensor.__console__.print(":white_heavy_check_mark: [green]Finalized[/green]") + else: + bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) + + if response.is_success: + block = self.get_current_block() + new_stake = bittensor.Balance.from_tao( self.neuron_for_uid( uid = neuron.uid, block = block ).stake) + bittensor.__console__.print("Stake ({}): [blue]{}[/blue] :arrow_right: [green]{}[/green]".format( neuron.uid, stake_on_uid, new_stake )) + successful_unstakes += 1 + + if successful_unstakes != 0: + with bittensor.__console__.status(":satellite: Checking Balance on: ([white]{}[/white] ...".format(self.network)): + new_balance = self.get_balance( wallet.coldkey.ss58_address ) + bittensor.__console__.print("Balance: [blue]{}[/blue] :arrow_right: [green]{}[/green]".format( old_balance, new_balance )) + return True + + return False def set_weights( self, diff --git a/bittensor/_wallet/__init__.py b/bittensor/_wallet/__init__.py index df8fb4fdf7..9849d50664 100644 --- a/bittensor/_wallet/__init__.py +++ b/bittensor/_wallet/__init__.py @@ -111,11 +111,11 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument('--' + prefix_str + 'wallet.hotkey', required=False, default=bittensor.defaults.wallet.hotkey, help='''The name of wallet's hotkey.''') parser.add_argument('--' + prefix_str + 'wallet.path', required=False, default=bittensor.defaults.wallet.path, help='''The path to your bittensor wallets''') parser.add_argument('--' + prefix_str + 'wallet._mock', action='store_true', default=bittensor.defaults.wallet._mock, help='To turn on wallet mocking for testing purposes.') + parser.add_argument('--' + prefix_str + 'wallet.hotkeys', '--' + prefix_str + 'wallet.exclude_hotkeys', required=False, action='store', default=bittensor.defaults.wallet.hotkeys, type=str, nargs='*', help='''Specify the hotkeys by name. (e.g. hk1 hk2 hk3)''') parser.add_argument('--' + prefix_str + 'wallet.all_hotkeys', required=False, action='store_true', default=bittensor.defaults.wallet.all_hotkeys, help='''To specify all hotkeys. Specifying hotkeys will exclude them from this all.''') - parser.add_argument('--' + prefix_str + 'wallet.sort_by', required=False, action='store', default=bittensor.defaults.wallet.sort_by, type=str, help='''Sort the hotkeys by the specified column title (e.g. name, uid, axon).''') - parser.add_argument('--' + prefix_str + 'wallet.sort_order', required=False, action='store', default=bittensor.defaults.wallet.sort_order, type=str, help='''Sort the hotkeys in the specified ordering. (ascending/asc or descending/desc/reverse)''') parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, action='store', default=bittensor.defaults.wallet.reregister, type=bool, help='''Whether to reregister the wallet if it is not already registered.''') + except argparse.ArgumentError as e: import pdb #pdb.set_trace() @@ -134,8 +134,6 @@ def add_defaults(cls, defaults): # CLI defaults for Overview defaults.wallet.hotkeys = [] defaults.wallet.all_hotkeys = False - defaults.wallet.sort_by = "" - defaults.wallet.sort_order = "ascending" # Defaults for registration defaults.wallet.reregister = True @@ -148,6 +146,5 @@ def check_config(cls, config: 'bittensor.Config' ): assert isinstance(config.wallet.get('hotkey', bittensor.defaults.wallet.hotkey), str ) or config.wallet.get('hotkey', bittensor.defaults.wallet.hotkey) == None assert isinstance(config.wallet.path, str) assert isinstance(config.wallet.hotkeys, list) - assert isinstance(config.wallet.sort_by, str) - assert isinstance(config.wallet.sort_order, str) assert isinstance(config.wallet.reregister, bool) + assert isinstance(config.wallet.all_hotkeys, bool) diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index ef76720630..d8276422e0 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -442,7 +442,7 @@ def test_unstake_with_specific_hotkeys( self ): with patch('bittensor.wallet') as mock_create_wallet: mock_create_wallet.side_effect = mock_wallets - with patch('bittensor.Subtensor.unstake', return_value=True) as mock_unstake: + with patch('bittensor.Subtensor.unstake_multiple', return_value=True) as mock_unstake: cli.run() mock_create_wallet.assert_has_calls( [ @@ -451,7 +451,7 @@ def test_unstake_with_specific_hotkeys( self ): any_order=True ) mock_unstake.assert_has_calls( - [call(mock_wallets[i+1], amount=5.0, wait_for_inclusion=True, prompt=False) for i in range(len(mock_wallets[1:]))], + [call(wallets=mock_wallets[1:], amounts=[5.0]*len(mock_wallets[1:]), wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -500,11 +500,11 @@ def test_unstake_with_all_hotkeys( self ): with patch.object(cli, '_get_hotkey_wallets_for_wallet') as mock_get_all_wallets: mock_get_all_wallets.return_value = mock_wallets - with patch('bittensor.Subtensor.unstake', return_value=True) as mock_unstake: + with patch('bittensor.Subtensor.unstake_multiple', return_value=True) as mock_unstake: cli.run() mock_get_all_wallets.assert_called_once() mock_unstake.assert_has_calls( - [call(mock_wallets[i+1], amount=5.0, wait_for_inclusion=True, prompt=False) for i in range(len(mock_wallets[1:]))], + [call(wallets=mock_wallets, amounts=[5.0]*len(mock_wallets), wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -553,13 +553,11 @@ def test_unstake_with_exclude_hotkeys_from_all( self ): with patch.object(cli, '_get_hotkey_wallets_for_wallet') as mock_get_all_wallets: mock_get_all_wallets.return_value = mock_wallets - with patch('bittensor.Subtensor.unstake', return_value=True) as mock_unstake: + with patch('bittensor.Subtensor.unstake_multiple', return_value=True) as mock_unstake: cli.run() mock_get_all_wallets.assert_called_once() mock_unstake.assert_has_calls( - [call(mock_wallets[i], amount=5.0, wait_for_inclusion=True, prompt=False) - for i in (0, 2) # Don't call for hk1 - ], + [call(wallets=[mock_wallets[0], mock_wallets[2]], amounts=[5.0, 5.0], wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -628,7 +626,7 @@ def test_unstake_with_multiple_hotkeys_max_stake( self ): with patch('bittensor.wallet') as mock_create_wallet: mock_create_wallet.side_effect = mock_wallets - with patch('bittensor.Subtensor.unstake', return_value=True) as mock_unstake: + with patch('bittensor.Subtensor.unstake_multiple', return_value=True) as mock_unstake: cli.run() mock_create_wallet.assert_has_calls( [ @@ -637,7 +635,7 @@ def test_unstake_with_multiple_hotkeys_max_stake( self ): any_order=True ) mock_unstake.assert_has_calls( - [call(mock_wallet, amount=CLOSE_IN_VALUE((mock_stakes[mock_wallet.hotkey_str] - config.max_stake).tao, 0.001), wait_for_inclusion=True, prompt=False) for mock_wallet in mock_wallets[1:]], + [call(wallets=mock_wallets[1:], amounts=[CLOSE_IN_VALUE((mock_stakes[mock_wallet.hotkey_str] - config.max_stake).tao, 0.001) for mock_wallet in mock_wallets[1:]], wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -706,7 +704,7 @@ def test_unstake_with_multiple_hotkeys_max_stake_not_enough_stake( self ): with patch('bittensor.wallet') as mock_create_wallet: mock_create_wallet.side_effect = mock_wallets - with patch('bittensor.Subtensor.unstake', return_value=True) as mock_unstake: + with patch('bittensor.Subtensor.unstake_multiple', return_value=True) as mock_unstake: cli.run() mock_create_wallet.assert_has_calls( [ @@ -715,15 +713,16 @@ def test_unstake_with_multiple_hotkeys_max_stake_not_enough_stake( self ): any_order=True ) mock_unstake.assert_called() - for mock_call in mock_unstake.mock_calls: - # Python 3.7 - ## https://docs.python.org/3.7/library/unittest.mock.html#call - ## Uses the 1st index as args list - ## call.args only works in Python 3.8+ - mock_wallet = mock_call[1][0] - # We shouldn't unstake from hk1 as it has less than max_stake staked - assert mock_wallet.hotkey_str != 'hk1' + # Python 3.7 + ## https://docs.python.org/3.7/library/unittest.mock.html#call + ## Uses the 1st index as args list + ## call.args only works in Python 3.8+ + mock_wallets_ = mock_unstake.mock_calls[0][2]['wallets'] + + + # We shouldn't unstake from hk1 as it has less than max_stake staked + assert all(mock_wallet.hotkey_str != 'hk1' for mock_wallet in mock_wallets_) def test_stake_with_specific_hotkeys( self ): bittensor.subtensor.register = MagicMock(return_value = True) @@ -782,7 +781,7 @@ def test_stake_with_specific_hotkeys( self ): with patch('bittensor.wallet') as mock_create_wallet: mock_create_wallet.side_effect = mock_wallets - with patch('bittensor.Subtensor.add_stake', return_value=True) as mock_add_stake: + with patch('bittensor.Subtensor.add_stake_multiple', return_value=True) as mock_add_stake: cli.run() mock_create_wallet.assert_has_calls( [ @@ -791,7 +790,7 @@ def test_stake_with_specific_hotkeys( self ): any_order=True ) mock_add_stake.assert_has_calls( - [call(mock_wallets[i+1], amount=5.0, wait_for_inclusion=True, prompt=False) for i in range(len(mock_wallets[1:]))], + [call(wallets=mock_wallets[1:], amounts=[5.0] * len(mock_wallets[1:]), wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -826,7 +825,6 @@ def test_stake_with_all_hotkeys( self ): ) for hk in mock_hotkeys ] - # The 0th wallet is created twice during unstake mock_wallets[0]._coldkey = mock_coldkey mock_wallets[0].coldkey = MagicMock( return_value=mock_coldkey @@ -839,11 +837,11 @@ def test_stake_with_all_hotkeys( self ): with patch.object(cli, '_get_hotkey_wallets_for_wallet') as mock_get_all_wallets: mock_get_all_wallets.return_value = mock_wallets - with patch('bittensor.Subtensor.add_stake', return_value=True) as mock_add_stake: + with patch('bittensor.Subtensor.add_stake_multiple', return_value=True) as mock_add_stake: cli.run() mock_get_all_wallets.assert_called_once() mock_add_stake.assert_has_calls( - [call(mock_wallets[i+1], amount=5.0, wait_for_inclusion=True, prompt=False) for i in range(len(mock_wallets[1:]))], + [call(wallets=mock_wallets, amounts=[5.0] * len(mock_wallets), wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -891,13 +889,11 @@ def test_stake_with_exclude_hotkeys_from_all( self ): with patch.object(cli, '_get_hotkey_wallets_for_wallet') as mock_get_all_wallets: mock_get_all_wallets.return_value = mock_wallets - with patch('bittensor.Subtensor.add_stake', return_value=True) as mock_add_stake: + with patch('bittensor.Subtensor.add_stake_multiple', return_value=True) as mock_add_stake: cli.run() mock_get_all_wallets.assert_called_once() mock_add_stake.assert_has_calls( - [call(mock_wallets[i], amount=5.0, wait_for_inclusion=True, prompt=False) - for i in (0, 2) # Don't call stake for hk1 - ], + [call(wallets=[mock_wallets[0], mock_wallets[2]], amounts=[5.0, 5.0], wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -971,7 +967,7 @@ def test_stake_with_multiple_hotkeys_max_stake( self ): with patch('bittensor.wallet') as mock_create_wallet: mock_create_wallet.side_effect = mock_wallets - with patch('bittensor.Subtensor.add_stake', return_value=True) as mock_add_stake: + with patch('bittensor.Subtensor.add_stake_multiple', return_value=True) as mock_add_stake: cli.run() mock_create_wallet.assert_has_calls( [ @@ -980,7 +976,7 @@ def test_stake_with_multiple_hotkeys_max_stake( self ): any_order=True ) mock_add_stake.assert_has_calls( - [call(mock_wallet, amount=CLOSE_IN_VALUE((config.max_stake - mock_stakes[mock_wallet.hotkey_str].tao), 0.001), wait_for_inclusion=True, prompt=False) for mock_wallet in mock_wallets[1:]], + [call(wallets=mock_wallets[1:], amounts=[CLOSE_IN_VALUE((config.max_stake - mock_stakes[mock_wallet.hotkey_str].tao), 0.001) for mock_wallet in mock_wallets[1:]], wait_for_inclusion=True, prompt=False)], any_order = True ) @@ -1054,7 +1050,7 @@ def test_stake_with_multiple_hotkeys_max_stake_not_enough_balance( self ): with patch('bittensor.wallet') as mock_create_wallet: mock_create_wallet.side_effect = mock_wallets - with patch('bittensor.Subtensor.add_stake', return_value=True) as mock_add_stake: + with patch('bittensor.Subtensor.add_stake_multiple', return_value=True) as mock_add_stake: cli.run() mock_create_wallet.assert_has_calls( [ @@ -1066,14 +1062,15 @@ def test_stake_with_multiple_hotkeys_max_stake_not_enough_balance( self ): mock_add_stake.assert_called_once() total_staked = 0.0 - for mock_call in mock_add_stake.mock_calls: - # Python 3.7 - ## https://docs.python.org/3.7/library/unittest.mock.html#call - ## Uses the 2nd index as kwargs dict - ## call.kwargs only works in Python 3.8+ - staked_this_call = mock_call[2]['amount'] - - total_staked += staked_this_call + + # Python 3.7 + ## https://docs.python.org/3.7/library/unittest.mock.html#call + ## Uses the 2nd index as kwargs dict + ## call.kwargs only works in Python 3.8+ + amounts_passed = mock_add_stake.mock_calls[0][2]['amounts'] + + total_staked = sum(amounts_passed) + # We should not try to stake more than the mock_balance assert CLOSE_IN_VALUE(total_staked, 0.001) == mock_balance.tao @@ -1265,7 +1262,7 @@ def test_inspect( self ): def test_list( self ): # Mock IO for wallet - with patch('bittensor.wallet.__new__', side_effect=[MagicMock( + with patch('bittensor.wallet', side_effect=[MagicMock( coldkeypub_file=MagicMock( exists_on_device=MagicMock( return_value=True # Wallet exists From b9ca29807b7de0d02a347b0ef866f0c85b4076f2 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 16 Aug 2022 09:50:22 -0400 Subject: [PATCH 06/28] Add/address CUDA reg changes (#879) * added cuda solver * boost versions to fix pip error * allow choosing device id * fix solution check to use keccak * adds params for cuda and dev_id to register * list devices by name during selection * add block number logging * fix calculation of hashrate * fix update interval default * add --TPB arg to register * add update_interval flag * switch back to old looping/work structure * change typing * device count is a function * stop early if wallet registered * add update interval and num proc flag * add better number output * optimize multiproc cpu reg keeping proc until solution * fix test * change import to cubit * fix import and default * up default should have default in CLI call * add comments about params * fix config var access * add cubit as extra * handle stale pow differently check registration after failure * restrict number of processes for integration test * fix stale check * use wallet.is_registered instead * attempt to fix test issue * fix my test * oops typo * typo again ugh * remove print out * fix partly reg test * fix if solution None * fix test? * fix patch * add args for cuda to subtensor * add cuda args to reregister call * add to wallet register the cuda args * fix refs and tests * add for val test also * fix tests with rereg * fix patch for tests * add mock_register to subtensor passed instead * move register under the check for isregistered * use patch obj instead * fit patch object * fix prompt * remove unneeded if * modify POW submit to use rolling submit again * add backoff to block get from network * add test for backoff get block * suppress the dev id flag if not set * remove dest so it uses first arg * fix pow submit loop * move registration status with * fix max attempts check * remove status in subtensor.register * add submit status * change to neuron get instead * fix count * try to patch live display * fix patch * . * separate test cases * add POWNotStale and tests * add more test cases for block get with retry * fix return to None * fix arg order --- bittensor/_cli/__init__.py | 4 +- bittensor/_subtensor/__init__.py | 8 +- bittensor/_subtensor/subtensor_impl.py | 115 +++++++++-------- bittensor/utils/__init__.py | 36 ++++-- requirements.txt | 1 + tests/integration_tests/test_subtensor.py | 41 ++++-- .../bittensor_tests/utils/test_utils.py | 122 ++++++++++++++++-- 7 files changed, 227 insertions(+), 100 deletions(-) diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index 97ccd03ead..c9e5ecedfe 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -832,10 +832,10 @@ def check_register_config( config: 'bittensor.Config' ): if not config.no_prompt and config.subtensor.register.cuda.use_cuda == bittensor.defaults.subtensor.register.cuda.use_cuda: # Ask about cuda registration only if a CUDA device is available. if torch.cuda.is_available(): - cuda = Confirm.ask("Would you like to try CUDA registration?\n") + cuda = Confirm.ask("Detected CUDA device, use CUDA for registration?\n") config.subtensor.register.cuda.use_cuda = cuda # Only ask about which CUDA device if the user has more than one CUDA device. - if cuda and config.subtensor.register.cuda.dev_id == bittensor.defaults.subtensor.register.cuda.dev_id and torch.cuda.device_count() > 0: + if cuda and config.subtensor.register.cuda.get('dev_id') is None and torch.cuda.device_count() > 0: devices: List[str] = [str(x) for x in range(torch.cuda.device_count())] device_names: List[str] = [torch.cuda.get_device_name(x) for x in range(torch.cuda.device_count())] console.print("Available CUDA devices:") diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index 0d53fda8b2..a994c1e79a 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -186,11 +186,11 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument('--' + prefix_str + 'subtensor._mock', action='store_true', help='To turn on subtensor mocking for testing purposes.', default=bittensor.defaults.subtensor._mock) parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest='subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) - parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', dest='subtensor.register.update_interval', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) + parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) # registration args. Used for register and re-register and anything that calls register. - parser.add_argument( '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', '--' + prefix_str + 'subtensor.register.cuda.use_cuda', dest=f'{prefix_str}subtensor.register.cuda.use_cuda', default=bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set true to use CUDA.''', action='store_true', required=False ) - parser.add_argument( '--' + prefix_str + 'cuda.dev_id', '--' + prefix_str + 'subtensor.register.cuda.dev_id', dest=f'{prefix_str}subtensor.register.cuda.dev_id', type=int, default=bittensor.defaults.subtensor.register.cuda.dev_id, help='''Set the CUDA device id. Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) - parser.add_argument( '--' + prefix_str + 'cuda.TPB', '--' + prefix_str + 'subtensor.register.cuda.TPB', dest=f'{prefix_str}subtensor.register.cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set true to use CUDA.''', action='store_true', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, default=argparse.SUPPRESS, help='''Set the CUDA device id. Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) except argparse.ArgumentError: # re-parsing arguments. diff --git a/bittensor/_subtensor/subtensor_impl.py b/bittensor/_subtensor/subtensor_impl.py index baac5d3b70..48d67d07e7 100644 --- a/bittensor/_subtensor/subtensor_impl.py +++ b/bittensor/_subtensor/subtensor_impl.py @@ -490,6 +490,7 @@ def register ( # Attempt rolling registration. attempts = 1 while True: + bittensor.__console__.print(":satellite: Registering...({}/{})".format(attempts, max_allowed_attempts)) # Solve latest POW. if cuda: if not torch.cuda.is_available(): @@ -499,7 +500,6 @@ def register ( pow_result = bittensor.utils.create_pow( self, wallet, cuda, dev_id, TPB, num_processes=num_processes, update_interval=update_interval ) else: pow_result = bittensor.utils.create_pow( self, wallet, num_processes=num_processes, update_interval=update_interval) - with bittensor.__console__.status(":satellite: Registering...({}/{})".format(attempts,max_allowed_attempts)) as status: # pow failed if not pow_result: @@ -510,62 +510,64 @@ def register ( # pow successful, proceed to submit pow to chain for registration else: - #check if pow result is still valid - if pow_result['block_number'] < self.get_current_block() - 3: - bittensor.__console__.print( "[red]POW is stale.[/red]" ) - continue - - with self.substrate as substrate: - # create extrinsic call - call = substrate.compose_call( - call_module='SubtensorModule', - call_function='register', - call_params={ - 'block_number': pow_result['block_number'], - 'nonce': pow_result['nonce'], - 'work': bittensor.utils.hex_bytes_to_u8_list( pow_result['work'] ), - 'hotkey': wallet.hotkey.ss58_address, - 'coldkey': wallet.coldkeypub.ss58_address - } - ) - extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.hotkey ) - response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization ) - - # We only wait here if we expect finalization. - if not wait_for_finalization and not wait_for_inclusion: - bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") - return True - - # process if registration successful, try again if pow is still valid - response.process_events() - if not response.is_success: - if 'key is already registered' in response.error_message: - # Error meant that the key is already registered. - bittensor.__console__.print(":white_heavy_check_mark: [green]Already Registered[/green]") - return True - - bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) - time.sleep(1) - - # Successful registration, final check for neuron and pubkey + with bittensor.__console__.status(":satellite: Submitting POW..."): + # check if pow result is still valid + while bittensor.utils.POWNotStale(self, pow_result): + with self.substrate as substrate: + # create extrinsic call + call = substrate.compose_call( + call_module='SubtensorModule', + call_function='register', + call_params={ + 'block_number': pow_result['block_number'], + 'nonce': pow_result['nonce'], + 'work': bittensor.utils.hex_bytes_to_u8_list( pow_result['work'] ), + 'hotkey': wallet.hotkey.ss58_address, + 'coldkey': wallet.coldkeypub.ss58_address + } + ) + extrinsic = substrate.create_signed_extrinsic( call = call, keypair = wallet.hotkey ) + response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization ) + + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + bittensor.__console__.print(":white_heavy_check_mark: [green]Sent[/green]") + return True + + # process if registration successful, try again if pow is still valid + response.process_events() + if not response.is_success: + if 'key is already registered' in response.error_message: + # Error meant that the key is already registered. + bittensor.__console__.print(":white_heavy_check_mark: [green]Already Registered[/green]") + return True + + bittensor.__console__.print(":cross_mark: [red]Failed[/red]: error:{}".format(response.error_message)) + time.sleep(0.5) + + # Successful registration, final check for neuron and pubkey + else: + bittensor.__console__.print(":satellite: Checking Balance...") + neuron = self.neuron_for_pubkey( wallet.hotkey.ss58_address ) + if not neuron.is_null: + bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") + return True + else: + # neuron not found, try again + bittensor.__console__.print(":cross_mark: [red]Unknown error. Neuron not found.[/red]") + continue else: - bittensor.__console__.print(":satellite: Checking Balance...") - if wallet.is_registered( self ): - bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") - return True - else: - # neuron not found, try again - bittensor.__console__.print(":cross_mark: [red]Unknown error. Neuron not found.[/red]") - return False - - #Failed registration, retry pow - attempts += 1 - if attempts > max_allowed_attempts: + # Exited loop because pow is no longer valid. + bittensor.__console__.print( "[red]POW is stale.[/red]" ) + return False + if attempts < max_allowed_attempts: + #Failed registration, retry pow + attempts += 1 + bittensor.__console__.print( ":satellite: Failed registration, retrying pow ...({}/{})".format(attempts, max_allowed_attempts)) + else: + # Failed to register after max attempts. bittensor.__console__.print( "[red]No more attempts.[/red]" ) return False - else: - status.update( ":satellite: Failed registration, retrying pow ...({}/{})".format(attempts, max_allowed_attempts)) - continue def serve ( self, @@ -865,10 +867,9 @@ def add_stake_multiple ( # Stake it all. staking_balance = bittensor.Balance.from_tao( old_balance.tao ) staking_all = True - - elif not isinstance(amount, bittensor.Balance ): - staking_balance = bittensor.Balance.from_tao( amount ) else: + # Amounts are cast to balance earlier in the function + assert isinstance(amount, bittensor.Balance) staking_balance = amount # Estimate staking fee. diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 195f9872f6..3a5b353b8d 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -11,12 +11,13 @@ from queue import Empty from typing import Any, Dict, Optional, Tuple, Union +import backoff import bittensor import pandas import requests import torch from Crypto.Hash import keccak -from substrateinterface import Keypair, KeypairType +from substrateinterface import Keypair from substrateinterface.utils import ss58 from .register_cuda import reset_cuda, solve_cuda @@ -125,6 +126,12 @@ def millify(n: int): return '{:.0f}{}'.format(n / 10**(3 * millidx), millnames[millidx]) +def POWNotStale(subtensor: 'bittensor.Subtensor', pow_result: Dict) -> bool: + """Returns True if the POW is not stale. + This means the block the POW is solved for is within 3 blocks of the current block. + """ + return pow_result['block_number'] >= subtensor.get_current_block() - 3 + @dataclass class POWSolution: """A solution to the registration PoW problem.""" @@ -449,6 +456,18 @@ def millify(n: int): return '{:.0f}{}'.format(n / 10**(3 * millidx), millnames[millidx]) +@backoff.on_exception(backoff.constant, + Exception, + interval=1, + max_tries=3) +def get_block_with_retry(subtensor: 'bittensor.Subtensor') -> Tuple[int, int, bytes]: + block_number = subtensor.get_current_block() + difficulty = subtensor.difficulty + block_hash = subtensor.substrate.get_block_hash( block_number ) + if block_hash is None: + raise Exception("Network error. Could not connect to substrate to get block hash") + return block_number, difficulty, block_hash + def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: int = 0 ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA @@ -469,12 +488,8 @@ def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'b if update_interval is None: update_interval = 50_000 - - block_number = subtensor.get_current_block() - difficulty = subtensor.difficulty - block_hash = subtensor.substrate.get_block_hash( block_number ) - while block_hash == None: - block_hash = subtensor.substrate.get_block_hash( block_number ) + + block_number, difficulty, block_hash = get_block_with_retry(subtensor) block_bytes = block_hash.encode('utf-8')[2:] nonce = 0 @@ -512,11 +527,8 @@ def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'b nonce = 0 itrs_per_sec = (TPB * update_interval) / (time.time() - interval_time) interval_time = time.time() - difficulty = subtensor.difficulty - block_number = subtensor.get_current_block() - block_hash = subtensor.substrate.get_block_hash( block_number) - while block_hash == None: - block_hash = subtensor.substrate.get_block_hash( block_number) + + block_number, difficulty, block_hash = get_block_with_retry(subtensor) block_bytes = block_hash.encode('utf-8')[2:] message = f"""Solving diff --git a/requirements.txt b/requirements.txt index 392b7ec0c9..1c7f9b8fd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ argparse base58>=2.0.1 +backoff>=2.1.0 certifi>=2020.11.8 cryptography>=3.1.1 idna>=2.10 diff --git a/tests/integration_tests/test_subtensor.py b/tests/integration_tests/test_subtensor.py index 60639d7a28..8c1ae1967e 100644 --- a/tests/integration_tests/test_subtensor.py +++ b/tests/integration_tests/test_subtensor.py @@ -402,21 +402,29 @@ def process_events(self): mock_neuron.is_null = True with patch('bittensor.Subtensor.difficulty'): - with patch('multiprocessing.queues.Queue.get') as mock_queue_get: - mock_queue_get.return_value = None + # patch solution queue to return None + with patch('multiprocessing.queues.Queue.get', return_value=None) as mock_queue_get: + # patch time queue size check + with patch('multiprocessing.queues.Queue.qsize', return_value=0): - wallet = bittensor.wallet(_mock=True) - wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) + wallet = bittensor.wallet(_mock=True) + wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) - self.subtensor.difficulty= 1 - self.subtensor.neuron_for_pubkey = MagicMock( return_value=mock_neuron ) - self.subtensor.substrate.submit_extrinsic = MagicMock(return_value = success()) + self.subtensor.difficulty= 1 + self.subtensor.neuron_for_pubkey = MagicMock( return_value=mock_neuron ) + self.subtensor.substrate.submit_extrinsic = MagicMock(return_value = success()) - # should return True - assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5 ) == True - # calls until True and once again before exiting subtensor class - # This assertion is currently broken when difficulty is too low - assert wallet.is_registered.call_count == workblocks_before_is_registered + 2 + with patch('bittensor.__console__.status') as mock_set_status: + # Need to patch the console status to avoid opening a parallel live display + mock_set_status.__enter__ = MagicMock(return_value=True) + mock_set_status.__exit__ = MagicMock(return_value=True) + + # should return True + assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5 ) == True + + # calls until True and once again before exiting subtensor class + # This assertion is currently broken when difficulty is too low + assert wallet.is_registered.call_count == workblocks_before_is_registered + 2 def test_registration_partly_failed( self ): class failed(): @@ -448,8 +456,13 @@ def is_registered_side_effect(*args, **kwargs): self.subtensor.get_current_block = MagicMock(side_effect=current_block) self.subtensor.substrate.submit_extrinsic = submit_extrinsic_mock - # should return True - assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5) == True + with patch('bittensor.__console__.status') as mock_set_status: + # Need to patch the console status to avoid opening a parallel live display + mock_set_status.__enter__ = MagicMock(return_value=True) + mock_set_status.__exit__ = MagicMock(return_value=True) + + # should return True + assert self.subtensor.register(wallet=wallet, num_processes=3, update_interval=5) == True def test_registration_failed( self ): class failed(): diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index ba7249c9dd..5d0643bc08 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -1,5 +1,6 @@ import binascii import hashlib +import unittest import bittensor import sys import subprocess @@ -15,7 +16,7 @@ from _pytest.fixtures import fixture from loguru import logger -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch @@ -234,18 +235,117 @@ def test_is_valid_ed25519_pubkey(): assert bittensor.utils.is_valid_ed25519_pubkey(good_pubkey) assert not bittensor.utils.is_valid_ed25519_pubkey(bad_pubkey) -def test_registration_diff_pack_unpack(): - fake_diff = pow(2, 31)# this is under 32 bits - - mock_diff = multiprocessing.Array('Q', [0, 0], lock=True) # [high, low] - - bittensor.utils.registration_diff_pack(fake_diff, mock_diff) - assert bittensor.utils.registration_diff_unpack(mock_diff) == fake_diff +def test_registration_diff_pack_unpack_under_32_bits(): + fake_diff = pow(2, 31)# this is under 32 bits + + mock_diff = multiprocessing.Array('Q', [0, 0], lock=True) # [high, low] + + bittensor.utils.registration_diff_pack(fake_diff, mock_diff) + assert bittensor.utils.registration_diff_unpack(mock_diff) == fake_diff + +def test_registration_diff_pack_unpack_over_32_bits(): + mock_diff = multiprocessing.Array('Q', [0, 0], lock=True) # [high, low] + fake_diff = pow(2, 32) * pow(2, 4) # this should be too large if the bit shift is wrong (32 + 4 bits) + + bittensor.utils.registration_diff_pack(fake_diff, mock_diff) + assert bittensor.utils.registration_diff_unpack(mock_diff) == fake_diff + +class TestGetBlockWithRetry(unittest.TestCase): + def test_get_block_with_retry_network_error_exit(self): + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=1), + difficulty=1, + substrate=MagicMock( + get_block_hash=MagicMock(side_effect=Exception('network error')) + ) + ) + with pytest.raises(Exception): + # this should raise an exception because the network error is retried only 3 times + bittensor.utils.get_block_with_retry(mock_subtensor) + + def test_get_block_with_retry_network_error_no_error(self): + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=1), + difficulty=1, + substrate=MagicMock( + get_block_hash=MagicMock(return_value=b'ba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279') + ) + ) + + # this should not raise an exception because there is no error + bittensor.utils.get_block_with_retry(mock_subtensor) + + def test_get_block_with_retry_network_error_none_twice(self): + # Should retry twice then succeed on the third try + tries = 0 + def block_none_twice(block_hash: bytes): + nonlocal tries + if tries == 1: + return block_hash + else: + tries += 1 + return None - fake_diff = pow(2, 32) * pow(2, 4) # this should be too large if the bit shift is wrong (32 + 4 bits) - bittensor.utils.registration_diff_pack(fake_diff, mock_diff) - assert bittensor.utils.registration_diff_unpack(mock_diff) == fake_diff + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=1), + difficulty=1, + substrate=MagicMock( + get_block_hash=MagicMock(side_effect=block_none_twice(b'ba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279')) + ) + ) + + # this should not raise an exception because there is no error on the third try + bittensor.utils.get_block_with_retry(mock_subtensor) +class TestPOWNotStale(unittest.TestCase): + def test_pow_not_stale_same_block_number(self): + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=1), + ) + mock_solution = { + "block_number": 1, + } + + assert bittensor.utils.POWNotStale(mock_subtensor, mock_solution) + + def test_pow_not_stale_diff_block_number(self): + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=2), + ) + mock_solution = { + "block_number": 1, # 1 less than current block number + } + + assert bittensor.utils.POWNotStale(mock_subtensor, mock_solution) + + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=3), + ) + mock_solution = { + "block_number": 1, # 2 less than current block number + } + + assert bittensor.utils.POWNotStale(mock_subtensor, mock_solution) + + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=4), + ) + mock_solution = { + "block_number": 1, # 3 less than current block number + } + + assert bittensor.utils.POWNotStale(mock_subtensor, mock_solution) + + def test_pow_not_stale_diff_block_number_too_old(self): + mock_subtensor = MagicMock( + get_current_block=MagicMock(return_value=5), + ) + mock_solution = { + "block_number": 1, # 4 less than current block number, stale + } + + assert not bittensor.utils.POWNotStale(mock_subtensor, mock_solution) + if __name__ == "__main__": test_solve_for_difficulty_fast_registered_already() \ No newline at end of file From 09697c9c37eb39b333b5c9680f687bb4144912cc Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 23 Aug 2022 13:04:08 -0400 Subject: [PATCH 07/28] [Fix] --help command (#884) * fix btcli help * add test to check --help works * update help message * parse after modification * fix ordering * fix test * add pytest rasies for exit * fix expectation with py310 --- bin/btcli | 9 ++++-- bittensor/_axon/__init__.py | 4 +-- bittensor/_cli/__init__.py | 24 +++++++++------ bittensor/_config/__init__.py | 47 ++++++++++++++++++++++++----- tests/integration_tests/test_cli.py | 32 ++++++++++++++++++-- 5 files changed, 92 insertions(+), 24 deletions(-) diff --git a/bin/btcli b/bin/btcli index b302727073..68bb4fecd7 100755 --- a/bin/btcli +++ b/bin/btcli @@ -1,11 +1,16 @@ #!/usr/bin/env python3 -import bittensor +import sys + +import bittensor + if __name__ == '__main__': - bittensor.cli().run() + args = sys.argv[1:] + bittensor.cli(args=args).run() # The MIT License (MIT) # Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation diff --git a/bittensor/_axon/__init__.py b/bittensor/_axon/__init__.py index 0018ed26c7..60f78d9dd1 100644 --- a/bittensor/_axon/__init__.py +++ b/bittensor/_axon/__init__.py @@ -227,13 +227,13 @@ def add_args( cls, parser: argparse.ArgumentParser, prefix: str = None ): def add_defaults(cls, defaults): """ Adds parser defaults to object from enviroment variables. """ - defaults.axon = bittensor.config() + defaults.axon = bittensor.Config() defaults.axon.port = os.getenv('BT_AXON_PORT') if os.getenv('BT_AXON_PORT') != None else 8091 defaults.axon.ip = os.getenv('BT_AXON_IP') if os.getenv('BT_AXON_IP') != None else '[::]' defaults.axon.max_workers = os.getenv('BT_AXON_MAX_WORERS') if os.getenv('BT_AXON_MAX_WORERS') != None else 10 defaults.axon.maximum_concurrent_rpcs = os.getenv('BT_AXON_MAXIMUM_CONCURRENT_RPCS') if os.getenv('BT_AXON_MAXIMUM_CONCURRENT_RPCS') != None else 400 - defaults.axon.priority = bittensor.config() + defaults.axon.priority = bittensor.Config() defaults.axon.priority.max_workers = os.getenv('BT_AXON_PRIORITY_MAX_WORKERS') if os.getenv('BT_AXON_PRIORITY_MAX_WORKERS') != None else 10 defaults.axon.priority.maxsize = os.getenv('BT_AXON_PRIORITY_MAXSIZE') if os.getenv('BT_AXON_PRIORITY_MAXSIZE') != None else -1 diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index c9e5ecedfe..3b21794db4 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -3,6 +3,7 @@ """ # The MIT License (MIT) # Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -21,12 +22,11 @@ import argparse import os import sys -from typing import List +from typing import List, Optional import bittensor import torch from rich.prompt import Confirm, Prompt -from substrateinterface.utils.ss58 import ss58_decode, ss58_encode from . import cli_impl @@ -34,28 +34,34 @@ class cli: """ - Create and init the CLI class, which handles the coldkey, hotkey and tau transfer + Create and init the CLI class, which handles the coldkey, hotkey and tao transfer """ def __new__( - cls, - config: 'bittensor.Config' = None, + cls, + config: Optional['bittensor.Config'] = None, + args: Optional[List[str]] = None, ) -> 'bittensor.CLI': r""" Creates a new bittensor.cli from passed arguments. Args: config (:obj:`bittensor.Config`, `optional`): bittensor.cli.config() + args (`List[str]`, `optional`): + The arguments to parse from the command line. """ if config == None: - config = cli.config() + config = cli.config(args) cli.check_config( config ) return cli_impl.CLI( config = config) @staticmethod - def config() -> 'bittensor.config': + def config(args: List[str]) -> 'bittensor.config': """ From the argument parser, add config to bittensor.executor and local config Return: bittensor.config object """ - parser = argparse.ArgumentParser(description="Bittensor cli", usage="btcli ", add_help=True) + parser = argparse.ArgumentParser( + description=f"bittensor cli v{bittensor.__version__}", + usage="btcli ", + add_help=True) cmd_parsers = parser.add_subparsers(dest='command') overview_parser = cmd_parsers.add_parser( @@ -611,7 +617,7 @@ def config() -> 'bittensor.config': required=False ) - return bittensor.config( parser ) + return bittensor.config( parser, args=args ) @staticmethod def check_config (config: 'bittensor.Config'): diff --git a/bittensor/_config/__init__.py b/bittensor/_config/__init__.py index a1e984dab6..0a18c7c8f5 100644 --- a/bittensor/_config/__init__.py +++ b/bittensor/_config/__init__.py @@ -3,6 +3,7 @@ """ # The MIT License (MIT) # Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -19,12 +20,14 @@ # DEALINGS IN THE SOFTWARE. import os -from argparse import ArgumentParser +import sys +from argparse import ArgumentParser, Namespace +from typing import List, Optional +import bittensor import yaml from loguru import logger -import bittensor from . import config_impl logger = logger.opt(colors=True) @@ -37,13 +40,15 @@ class InvalidConfigFile(Exception): """ In place of YAMLError """ - def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): + def __new__( cls, parser: ArgumentParser = None, strict: bool = False, args: Optional[List[str]] = None ): r""" Translates the passed parser into a nested Bittensor config. Args: parser (argparse.Parser): Command line parser object. strict (bool): If true, the command line arguments are strictly parsed. + args (list of str): + Command line arguments. Returns: config (bittensor.Config): Nested config object created from parser arguments. @@ -69,8 +74,15 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): except Exception as e: config_file_path = None + # Get args from argv if not passed in. + if args == None: + args = sys.argv[1:] + + # Parse args not strict + params = cls.__parse_args__(args=args, parser=parser, strict=False) + # 2. Optionally check for --strict, if stict we will parse the args strictly. - strict = parser.parse_known_args()[0].strict + strict = params.strict if config_file_path != None: config_file_path = os.path.expanduser(config_file_path) @@ -83,10 +95,8 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): print('Error in loading: {} using default parser settings'.format(e)) # 2. Continue with loading in params. - if not strict: - params = parser.parse_known_args()[0] - else: - params = parser.parse_args() + params = cls.__parse_args__(args=args, parser=parser, strict=strict) + _config = config_impl.Config() # Splits params on dot syntax i.e neuron.axon_port @@ -107,6 +117,27 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False ): return _config + @staticmethod + def __parse_args__( args: List[str], parser: ArgumentParser = None, strict: bool = False) -> Namespace: + """Parses the passed args use the passed parser. + Args: + args (List[str]): + List of arguments to parse. + parser (argparse.ArgumentParser): + Command line parser object. + strict (bool): + If true, the command line arguments are strictly parsed. + Returns: + Namespace: + Namespace object created from parser arguments. + """ + if not strict: + params = parser.parse_known_args(args=args)[0] + else: + params = parser.parse_args(args=args) + + return params + @staticmethod def full(): """ From the parser, add arguments to multiple bittensor sub-modules diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index d8276422e0..65bdd8f67a 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -1,5 +1,6 @@ # The MIT License (MIT) # Copyright © 2022 Yuma Rao +# Copyright © 2022 Opentensor Foundation # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -16,18 +17,16 @@ # DEALINGS IN THE SOFTWARE. -import sys import unittest from types import SimpleNamespace from typing import Dict from unittest.mock import ANY, MagicMock, call, patch +import pytest import bittensor from bittensor._subtensor.subtensor_mock import mock_subtensor from bittensor.utils.balance import Balance from substrateinterface.base import Keypair -from substrateinterface.exceptions import SubstrateRequestException - from tests.helpers import CLOSE_IN_VALUE @@ -1327,6 +1326,33 @@ def test_list_no_wallet( self ): # This shouldn't raise an error anymore cli.run() +def test_btcli_help(): + """ + Verify the correct help text is output when the --help flag is passed + """ + with pytest.raises(SystemExit) as pytest_wrapped_e: + with patch('argparse.ArgumentParser._print_message', return_value=None) as mock_print_message: + args = [ + '--help' + ] + bittensor.cli(args=args).run() + + # Should try to print help + mock_print_message.assert_called_once() + + call_args = mock_print_message.call_args + args, _ = call_args + help_out = args[0] + + # Expected help output even if parser isn't working well + ## py3.6-3.9 or py3.10+ + assert 'optional arguments' in help_out or 'options' in help_out + # Expected help output if all commands are listed + assert 'positional arguments' in help_out + # Verify that cli is printing the help message for + assert 'overview' in help_out + assert 'run' in help_out + if __name__ == "__main__": cli = TestCli() From f8845de01acadb294f876f8f42a40834569e11f8 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 15 Aug 2022 20:44:19 +0200 Subject: [PATCH 08/28] Update rich>=12.5.1 to fix /[ delimitation --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1c7f9b8fd6..c96f9a27e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ pytest-rerunfailures coveralls pytest-cov pyyaml -rich +rich>=12.5.1 retry requests>=2.25.0 scalecodec>=1.0.35 From c95aeef1822b56ad9b710e555c1faecc107c38a9 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 15 Aug 2022 20:23:33 +0200 Subject: [PATCH 09/28] Count responsive uid when has synapse_keys not nan --- bittensor/_neuron/text/core_validator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 4329f9265b..d6c066110a 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -543,6 +543,7 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): zkey = key + '!' # zeroing key stats.setdefault(zkey, 0.) # initialize zkey val to zero to gradually increase with observations if key in _stats and not math.isnan(_stats[key]): + responsive_uids += [_uid] stats[zkey] = (1 - self.alpha) * stats[zkey] + self.alpha * _stats[key] else: stats[zkey] = (1 - self.alpha) * stats[zkey] # + self.alpha * 0 @@ -555,7 +556,6 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): updates = 'updates_' + key if updates in stats: stats[updates] += 1 # increment number of normal EMA updates made - responsive_uids += [_uid] else: stats.setdefault(updates, 1) # add updates fields for new uid entries From 28b63a9b06430ec87001e687a13f2ef28191b099 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 13:31:47 +0200 Subject: [PATCH 10/28] Ensure each UID is queried once in validator nucleus Persist object variable self.permute_uids across forward calls, select num_endpoints uids each forward call, reset to new permutation of all UIDs once empty. Removes factors of variability between validators by ensuring each UID is queried the same number of times. --- .../_neuron/text/core_validator/__init__.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index d6c066110a..3c0e9d113d 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -660,6 +660,7 @@ def __init__( self, config, device, subtensor ): self.config = config self.device = device self.max_n = subtensor.max_n + self.permute_uids = [] # iterable of next UIDs to query, reset to permuted UIDs when empty tokenizer = bittensor.tokenizer() self.pad_token = tokenizer(tokenizer.pad_token)['input_ids'][0] @@ -794,18 +795,26 @@ def forward( # Ensure number of queried neurons does not exceed metagraph.n num_endpoints = min([self.config.nucleus.topk, metagraph.n]) - logger.info(f'Forward \t| Routing forward [{time.time() - start_time:.3g}s]') - logger.info(f'Dendrite \t| Request {num_endpoints} x {list(inputs_seq.shape)}') - request_start_time = time.time() + # === Ensure each UID is queried once === + # Persist object variable self.permute_uids across forward calls. + # Reset to new permutation of all UIDs once empty. + if len(self.permute_uids) == 0: # no more UIDs to query + self.permute_uids = torch.randperm(metagraph.n) # reset to new permutation of all UIDs # === Randomly select num_endpoints UIDs === - random_uids = torch.randperm(metagraph.n)[:num_endpoints] + random_uids = self.permute_uids[:num_endpoints] # newest selection of UIDs to query + self.permute_uids = self.permute_uids[num_endpoints:] # slice out remaining selection # === Get endpoint information for the selected UIDs === # We index into the metagraph's endpoints and return a list of the filtered set of endpoints we wish to query. # random_endpoints: List[bittensor.endpoints]: endpoint information for filtered uids. # len(neurons) == self.config.nucleus.topk random_endpoints = [metagraph.endpoints[uid] for uid in random_uids] + num_endpoints = len(random_endpoints) # in case len(self.permute_uids) < num_endpoints during random_uids select + + logger.info(f'Forward \t| Routing forward [{time.time() - start_time:.3g}s]') + logger.info(f'Dendrite \t| Request {num_endpoints} x {list(inputs_seq.shape)}') + request_start_time = time.time() # === Define which synapse we want to use === # The synapse defines the task we are sending to the neurons From ab8e18588f174bf49d2723eb0e726514e3cf49c2 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 14:11:28 +0200 Subject: [PATCH 11/28] Ensure each UID is queried at least once in validator neuron Assumes nucleus samples without replacement by permuting range(metagraph.n). Removes another factor of variability between validators, namely how many UIDs are sampled during each validator epoch, which is influenced by the validator speed. --- bittensor/_neuron/text/core_validator/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 3c0e9d113d..9bf37d3a32 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -372,7 +372,8 @@ def run_epoch( self ): epoch_queried_uids = set() start_block = self.subtensor.block - while self.subtensor.block < start_block + blocks_per_epoch: + while (self.subtensor.block < start_block + blocks_per_epoch or + len(epoch_queried_uids) < self.metagraph.n): # ensure each UID is queried at least once - assumes nucleus samples without replacement start_time = time.time() # === Forward === From 960426dcd59bb10b7360c4ba2573cd6d3c659568 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 14:13:28 +0200 Subject: [PATCH 12/28] Skip repeat metagraph_sync in validator run start --- bittensor/_neuron/text/core_validator/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 9bf37d3a32..a33e9f346f 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -365,7 +365,8 @@ def run_epoch( self ): # Each block length lasts blocks_per_epoch blocks. # This gives us a consistent network wide timer. # Here we run until blocks_per_epochs have progressed. - self.metagraph_sync() # Reset metagraph. + if self.epoch > 0: # skip first epoch: already synced at start of run + self.metagraph_sync() # Reset metagraph. epoch_steps = 0 epoch_responsive_uids = set() From ab0424e0a718340fc0d32d000348b8f56417ad62 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 14:43:32 +0200 Subject: [PATCH 13/28] Add set weights console message in validator --- bittensor/_neuron/text/core_validator/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index a33e9f346f..7dd2789e56 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -372,6 +372,7 @@ def run_epoch( self ): epoch_responsive_uids = set() epoch_queried_uids = set() + epoch_start_time = time.time() start_block = self.subtensor.block while (self.subtensor.block < start_block + blocks_per_epoch or len(epoch_queried_uids) < self.metagraph.n): # ensure each UID is queried at least once - assumes nucleus samples without replacement @@ -436,8 +437,9 @@ def run_epoch( self ): f'[dim] Epoch {self.epoch}[/dim] | ' f'[bright_green not bold]{len(responsive_uids)}[/bright_green not bold]/' f'[white]{len(queried_uids)}[/white] ' - f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' - f'[[yellow]{step_time:.3g}[/yellow]s]') + f'[[yellow]{step_time:.3g}[/yellow]s] ' + f'[dim white not bold][green]{len(epoch_responsive_uids)}[/green]/' + f'{len(epoch_queried_uids)}[/dim white not bold]') if self.config.logging.debug or self.config.logging.trace: # === Print stats update (table) === @@ -487,6 +489,15 @@ def run_epoch( self ): if self.config.logging.debug or self.config.logging.trace: self.weights_table(sample_uids, sample_weights) # print weights table + # set weights console message (every epoch) + print(f"[white not bold]{datetime.datetime.now():%Y-%m-%d %H:%M:%S}[/white not bold]{' ' * 4} | " + f"{f'[bright_white]Set weights[/bright_white]'.center(16 + len('[bright_white][/bright_white]'))} | " + f'[bright_green not bold]{len(sample_weights)}[/bright_green not bold] weights | ' + f'[bright_green not bold]{len(epoch_responsive_uids)}[/bright_green not bold]/' + f'[white]{len(epoch_queried_uids)}[/white] ' + f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' + f'[[yellow]{epoch_start_time - time.time():.3g}[/yellow]s]') + self.subtensor.set_weights( uids=sample_uids.detach().to('cpu'), weights=sample_weights.detach().to('cpu'), From 32c7df9f83ec7f08f1b67c917a3fdedf8494bdaf Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 15:10:51 +0200 Subject: [PATCH 14/28] Average synergy over responsives in validation query set --- bittensor/_neuron/text/core_validator/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 7dd2789e56..a2dac75153 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -1234,6 +1234,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens # Synergy = measured performance above expected performance # Measured in effective number of model parameters, just like base Shapley values. syn_loss_diff = {} # expected_loss - measured_loss (where > 0) + responsives = [uid for uid, stat in stats.items() if 'loss' + ext in stat] for _first, first in stats.items(): if 'loss' + ext not in first: continue @@ -1251,6 +1252,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens measured_loss = synergy(first, second, target, ext) loss_diff_share = torch.clamp(expected_loss - measured_loss, 0) / 2 # record direct loss diff + loss_diff_share /= len(responsives) # average over responsives first['synergy_loss_diff' + ext] += loss_diff_share second['synergy_loss_diff' + ext] += loss_diff_share @@ -1266,6 +1268,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens pow_expected_params = torch.pow(expected_params, scaling_law_power) synergy_share = torch.clamp(pow_measured_params - pow_expected_params, 0) / 2 + synergy_share /= len(responsives) # average over responsives first['synergy' + ext] += synergy_share # share synergy amongst coalition members second['synergy' + ext] += synergy_share From 55ae9dd98e44bd7a3ffbf1b3681b3f0aedcaecc3 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 15:21:52 +0200 Subject: [PATCH 15/28] Add synergy_scaling_law_power to validator nucleus Set synergy_scaling_law_power independent of scaling_law_power, since synergy likely needs a higher power after synergy averaging. --- .../_neuron/text/core_validator/__init__.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index a2dac75153..e0f73f9828 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -716,6 +716,7 @@ def add_args( cls, parser ): parser.add_argument('--nucleus.noise_multiplier', type=float, help='Standard deviation multipler on weights', default=2 ) parser.add_argument('--nucleus.dendrite_backward', action='store_true', help='Pass backward request to the server side or not', default=False ) parser.add_argument('--nucleus.scaling_law_power', type=float, help='Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) + parser.add_argument('--nucleus.synergy_scaling_law_power', type=float, help='Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) @classmethod def config ( cls ): @@ -869,8 +870,9 @@ def forward( # === Prepare validation parameter set === console_width = self.config.get('width', None) # console width for rich table displays of synapse measures validation_params = (random_uids, query_responses, return_ops, times, routing_score, - inputs, val_len, self.loss_fct, self.config.nucleus.scaling_law_power, console_width, - self.config.logging.debug or self.config.logging.trace) + inputs, val_len, self.loss_fct, + self.config.nucleus.scaling_law_power, self.config.nucleus.synergy_scaling_law_power, + console_width, self.config.logging.debug or self.config.logging.trace) loss = torch.tensor(0.).to(self.device) # to accumulate neuron_loss and routing_loss over synapses neuron_stats = {} # to gather neuron synapse validation measures and statistics @@ -898,7 +900,8 @@ def scaling_law_loss_to_params(loss): def textcausallm(uids: torch.Tensor, query_responses: List[List[torch.FloatTensor]], return_ops: List[torch.LongTensor], times: List[torch.FloatTensor], routing_score: torch.FloatTensor, - inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, scaling_law_power: float, + inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, + scaling_law_power: float, synergy_scaling_law_power: float, console_width: int, logging, synapse: 'bittensor.TextCausalLM' = None, index_s: int = 0 ) -> Tuple[torch.FloatTensor, Dict]: r""" @@ -923,6 +926,8 @@ def textcausallm(uids: torch.Tensor, query_responses: List[List[torch.FloatTenso CrossEntropy loss function to use. scaling_law_power (:obj:`float`, `required`): Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. + synergy_scaling_law_power (:obj:`float`, `required`): + Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. console_width (:obj:`int`, `required`): Config console width for table print. logging (:obj:`bool`, `required`): @@ -979,9 +984,9 @@ def _synergy(first, second, target, _ext): synergy_start_time = time.time() syn_loss_diff = shapley_synergy(stats, _synergy, ext='', target=inputs_seq[:, 1:], - scaling_law_power=scaling_law_power) + scaling_law_power=synergy_scaling_law_power) syn_loss_diff_val = shapley_synergy(stats, _synergy, ext='_val', target=inputs_val, - scaling_law_power=scaling_law_power) + scaling_law_power=synergy_scaling_law_power) # === Shapley value combination === # Combine base values with synergy approximation to get final Shapley values. @@ -1020,7 +1025,8 @@ def _synergy(first, second, target, _ext): def textcausallmnext(uids: torch.Tensor, query_responses: List[List[torch.FloatTensor]], return_ops: List[torch.LongTensor], times: List[torch.FloatTensor], routing_score: torch.FloatTensor, - inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, scaling_law_power: float, + inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, + scaling_law_power: float, synergy_scaling_law_power: float, console_width: int, logging, synapse: 'bittensor.TextCausalLMNext' = None, index_s: int = 0 ) -> Tuple[torch.FloatTensor, Dict]: r""" @@ -1045,6 +1051,8 @@ def textcausallmnext(uids: torch.Tensor, query_responses: List[List[torch.FloatT CrossEntropy loss function to use. scaling_law_power (:obj:`float`, `required`): Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. + synergy_scaling_law_power (:obj:`float`, `required`): + Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. console_width (:obj:`int`, `required`): Config console width for table print. logging (:obj:`bool`, `required`): @@ -1097,7 +1105,7 @@ def _synergy(first, second, target, ext): synergy_start_time = time.time() - syn_loss_diff = shapley_synergy(stats, _synergy, '_nxt', scaling_law_power=scaling_law_power) + syn_loss_diff = shapley_synergy(stats, _synergy, '_nxt', scaling_law_power=synergy_scaling_law_power) # === Shapley value combination === # Combine base values with synergy approximation to get final Shapley values. From 746605c12cb1692a92d19b81adea717440a2eb4d Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 15:23:13 +0200 Subject: [PATCH 16/28] Increase synergy_scaling_law_power from 0.5 to 0.6 Synergy averaging now significantly reduces synergy Shapley contribution compared to the base Shapley value, so the power needs to be increased to compensate. --- bittensor/_neuron/text/core_validator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index e0f73f9828..90932f5a0f 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -716,7 +716,7 @@ def add_args( cls, parser ): parser.add_argument('--nucleus.noise_multiplier', type=float, help='Standard deviation multipler on weights', default=2 ) parser.add_argument('--nucleus.dendrite_backward', action='store_true', help='Pass backward request to the server side or not', default=False ) parser.add_argument('--nucleus.scaling_law_power', type=float, help='Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) - parser.add_argument('--nucleus.synergy_scaling_law_power', type=float, help='Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) + parser.add_argument('--nucleus.synergy_scaling_law_power', type=float, help='Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.6) @classmethod def config ( cls ): From 0967462b8749dec07fd370d4bc8a7c8ff170fb0d Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 17:53:35 +0200 Subject: [PATCH 17/28] Update set weights console message --- bittensor/_neuron/text/core_validator/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 90932f5a0f..597eb93542 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -492,11 +492,16 @@ def run_epoch( self ): # set weights console message (every epoch) print(f"[white not bold]{datetime.datetime.now():%Y-%m-%d %H:%M:%S}[/white not bold]{' ' * 4} | " f"{f'[bright_white]Set weights[/bright_white]'.center(16 + len('[bright_white][/bright_white]'))} | " - f'[bright_green not bold]{len(sample_weights)}[/bright_green not bold] weights | ' + f'[bright_green not bold]{len(sample_weights)}[/bright_green not bold] [dim]weights set[/dim] | ' f'[bright_green not bold]{len(epoch_responsive_uids)}[/bright_green not bold]/' f'[white]{len(epoch_queried_uids)}[/white] ' f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' - f'[[yellow]{epoch_start_time - time.time():.3g}[/yellow]s]') + f'[[yellow]{epoch_start_time - time.time():.0f}[/yellow]s] | ' + f'weights sum:{sample_weights.sum().item():.2g} ' + f'[white] max:[bold]{sample_weights.max().item():.4g}[/bold] / ' + f'min:[bold]{sample_weights.min().item():.4g}[/bold] [/white] ' + f'\[{sample_weights.max().item() / sample_weights.min().item():.1f}:1] ' + f'({max_allowed_ratio} allowed)') self.subtensor.set_weights( uids=sample_uids.detach().to('cpu'), From 7f9d0a7e10d229407c0b027155fb78dbd30b5949 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 18:54:49 +0200 Subject: [PATCH 18/28] Update set weights console message --- bittensor/_neuron/text/core_validator/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 597eb93542..4451848a86 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -496,8 +496,8 @@ def run_epoch( self ): f'[bright_green not bold]{len(epoch_responsive_uids)}[/bright_green not bold]/' f'[white]{len(epoch_queried_uids)}[/white] ' f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' - f'[[yellow]{epoch_start_time - time.time():.0f}[/yellow]s] | ' - f'weights sum:{sample_weights.sum().item():.2g} ' + f'[[yellow]{time.time() - epoch_start_time:.0f}[/yellow]s] | ' + f'[dim]weights[/dim] sum:{sample_weights.sum().item():.2g} ' f'[white] max:[bold]{sample_weights.max().item():.4g}[/bold] / ' f'min:[bold]{sample_weights.min().item():.4g}[/bold] [/white] ' f'\[{sample_weights.max().item() / sample_weights.min().item():.1f}:1] ' From ddd0c51a647a939b74b5de81d80b0481a8ae2bb0 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 19:34:30 +0200 Subject: [PATCH 19/28] Clear validator nucleus UID permutation before epoch --- bittensor/_neuron/text/core_validator/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 4451848a86..ab9b86acf3 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -368,11 +368,13 @@ def run_epoch( self ): if self.epoch > 0: # skip first epoch: already synced at start of run self.metagraph_sync() # Reset metagraph. + self.nucleus.permute_uids = [] # clear nucleus permutation before epoch + epoch_steps = 0 epoch_responsive_uids = set() epoch_queried_uids = set() - epoch_start_time = time.time() + start_block = self.subtensor.block while (self.subtensor.block < start_block + blocks_per_epoch or len(epoch_queried_uids) < self.metagraph.n): # ensure each UID is queried at least once - assumes nucleus samples without replacement From 9b6285f6148ae245c27fc0655e6e026cb0a74b65 Mon Sep 17 00:00:00 2001 From: opentaco Date: Thu, 1 Sep 2022 20:36:05 +0200 Subject: [PATCH 20/28] Exclude lowest quantile from weight setting in validator Intended to pair with normalize_max_multiple to reduce sensitivity to bottom outliers by excluding the lowest quantile, effectively setting their weight to zero. Quantile of 10% chosen in added neuron parameter: exclude_quantile --- bittensor/_neuron/text/core_validator/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index ab9b86acf3..37e44f3dbe 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -211,6 +211,7 @@ def add_args( cls, parser ): parser.add_argument('--neuron.wait_for_finalization', action='store_true', help='''when setting weights the miner waits for trnasaction finalization.''', default=False) parser.add_argument('--neuron.forward_num', type=int, help='''How much forward request before a backward call.''', default=3) parser.add_argument('--neuron.validation_synapse', type=str, help='''Synapse used for validation.''', default='TextCausalLMNext', choices = ['TextCausalLMNext', 'TextCausalLM']) + parser.add_argument('--neuron.exclude_quantile', type=float, help='Exclude the lowest quantile from weight setting.', default=0.1) @classmethod def config ( cls ): @@ -628,6 +629,13 @@ def calculate_weights(self, responsive_uids: Set, queried_uids: Set): sample_uids = preferred_uids[:weights_to_set] # slice to weights_to_set sample_weights = neuron_weights[:weights_to_set] # slice to weights_to_set + # === Exclude lowest quantile from weight setting === + max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weights + exclude_quantile = min([self.config.neuron.exclude_quantile, max_exclude]) # reduce quantile to meet min_allowed_weights + lowest_quantile = sample_weights.quantile(exclude_quantile) # find lowest quantile threshold + sample_uids = sample_uids[lowest_quantile <= sample_weights] # exclude uids with weights below quantile + sample_weights = sample_weights[lowest_quantile <= sample_weights] # exclude weights below quantile + # === Normalize and apply max_allowed_ratio === sample_weights = bittensor.utils.weight_utils.normalize_max_multiple(x=sample_weights, multiple=max_allowed_ratio) From b116d419fd9d8bab1ecda9daafb78d72aff80b84 Mon Sep 17 00:00:00 2001 From: opentaco Date: Thu, 1 Sep 2022 22:37:44 +0200 Subject: [PATCH 21/28] Skip quantile exclusion if not min_allowed_weights --- bittensor/_neuron/text/core_validator/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 37e44f3dbe..77a40fad0a 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -630,11 +630,12 @@ def calculate_weights(self, responsive_uids: Set, queried_uids: Set): sample_weights = neuron_weights[:weights_to_set] # slice to weights_to_set # === Exclude lowest quantile from weight setting === - max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weights - exclude_quantile = min([self.config.neuron.exclude_quantile, max_exclude]) # reduce quantile to meet min_allowed_weights - lowest_quantile = sample_weights.quantile(exclude_quantile) # find lowest quantile threshold - sample_uids = sample_uids[lowest_quantile <= sample_weights] # exclude uids with weights below quantile - sample_weights = sample_weights[lowest_quantile <= sample_weights] # exclude weights below quantile + if min_allowed_weights <= len(sample_weights): + max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weights + exclude_quantile = min([self.config.neuron.exclude_quantile, max_exclude]) # reduce quantile to meet min_allowed_weights + lowest_quantile = sample_weights.quantile(exclude_quantile) # find lowest quantile threshold + sample_uids = sample_uids[lowest_quantile <= sample_weights] # exclude uids with weights below quantile + sample_weights = sample_weights[lowest_quantile <= sample_weights] # exclude weights below quantile # === Normalize and apply max_allowed_ratio === sample_weights = bittensor.utils.weight_utils.normalize_max_multiple(x=sample_weights, From 82bcd143da995a10aa0bbdd9cd4e615cfd862181 Mon Sep 17 00:00:00 2001 From: opentaco Date: Thu, 1 Sep 2022 22:40:50 +0200 Subject: [PATCH 22/28] Skip quantile exclusion if not min_allowed_weights --- bittensor/_neuron/text/core_validator/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 77a40fad0a..4fb962bcda 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -630,8 +630,8 @@ def calculate_weights(self, responsive_uids: Set, queried_uids: Set): sample_weights = neuron_weights[:weights_to_set] # slice to weights_to_set # === Exclude lowest quantile from weight setting === - if min_allowed_weights <= len(sample_weights): - max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weights + max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weights + if 0 < max_exclude: exclude_quantile = min([self.config.neuron.exclude_quantile, max_exclude]) # reduce quantile to meet min_allowed_weights lowest_quantile = sample_weights.quantile(exclude_quantile) # find lowest quantile threshold sample_uids = sample_uids[lowest_quantile <= sample_weights] # exclude uids with weights below quantile From cda4ebbcf7524220271240aeb1831ebd2bdbdf46 Mon Sep 17 00:00:00 2001 From: opentaco Date: Thu, 1 Sep 2022 23:11:26 +0200 Subject: [PATCH 23/28] Add logging info on weight sampling --- bittensor/_neuron/text/core_validator/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 4fb962bcda..226b335a00 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -629,17 +629,25 @@ def calculate_weights(self, responsive_uids: Set, queried_uids: Set): sample_uids = preferred_uids[:weights_to_set] # slice to weights_to_set sample_weights = neuron_weights[:weights_to_set] # slice to weights_to_set + logger.info(f'{len(sample_weights)} Shapley values | min:{sample_weights.min()} max:{sample_weights.max()}') + # === Exclude lowest quantile from weight setting === - max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weights + max_exclude = (len(sample_weights) - min_allowed_weights) / len(sample_weights) # max excludable weight quantile if 0 < max_exclude: exclude_quantile = min([self.config.neuron.exclude_quantile, max_exclude]) # reduce quantile to meet min_allowed_weights lowest_quantile = sample_weights.quantile(exclude_quantile) # find lowest quantile threshold sample_uids = sample_uids[lowest_quantile <= sample_weights] # exclude uids with weights below quantile sample_weights = sample_weights[lowest_quantile <= sample_weights] # exclude weights below quantile + logger.info(f'Exclude {exclude_quantile} quantile ({lowest_quantile}) | ' + f'{len(sample_weights)} Shapley values | min:{sample_weights.min()} max:{sample_weights.max()}') + # === Normalize and apply max_allowed_ratio === sample_weights = bittensor.utils.weight_utils.normalize_max_multiple(x=sample_weights, multiple=max_allowed_ratio) + logger.info(f'{len(sample_weights)} normalize_max_multiple | ' + f'min:{sample_weights.min()} max:{sample_weights.max()}') + return sample_uids, sample_weights def weights_table(self, sample_uids, sample_weights, include_uids=None, num_rows: int = None): From e5198d0b116c1ae4815e6c51c87a9871daae004f Mon Sep 17 00:00:00 2001 From: isabella618033 <49876827+isabella618033@users.noreply.github.com> Date: Thu, 1 Sep 2022 18:00:03 -0400 Subject: [PATCH 24/28] Bit 553 bug fixes (#886) * init added timeout for priority thread pool * axon text complete * fix responsive uid and core_server blocks_per_set_weight * server error mesage on validator fix * clean up * moved synaspe timeout to axon config * fix * fix * . * test fix * fix * fix * . * . * . * . * . * . * . * . * . Co-authored-by: Eugene-hu <85906264+Eugene-hu@users.noreply.github.com> Co-authored-by: Unconst <32490803+unconst@users.noreply.github.com> --- bittensor/_axon/__init__.py | 32 +++++++-- bittensor/_axon/axon_impl.py | 17 +++-- bittensor/_dendrite/dendrite_impl.py | 1 - .../_neuron/text/core_server/__init__.py | 1 - .../_neuron/text/core_server/nucleus_impl.py | 4 +- bittensor/_neuron/text/core_server/run.py | 5 +- .../_neuron/text/core_validator/__init__.py | 1 + bittensor/_threadpool/__init__.py | 1 - .../_threadpool/priority_thread_pool_impl.py | 2 +- tests/integration_tests/test_dendrite.py | 63 +++++++++++++++-- tests/integration_tests/test_wallet.py | 5 -- tests/unit_tests/bittensor_tests/test_axon.py | 70 ++++++++++++++++++- .../unit_tests/bittensor_tests/test_config.py | 37 ++++------ .../bittensor_tests/test_forward_backward.py | 2 +- .../unit_tests/bittensor_tests/test_neuron.py | 2 - .../bittensor_tests/test_receptor.py | 4 +- .../bittensor_tests/test_receptor_pool.py | 4 +- 17 files changed, 190 insertions(+), 61 deletions(-) diff --git a/bittensor/_axon/__init__.py b/bittensor/_axon/__init__.py index 60f78d9dd1..afa1d0979f 100644 --- a/bittensor/_axon/__init__.py +++ b/bittensor/_axon/__init__.py @@ -55,8 +55,13 @@ def __new__( synapse_causal_lm: 'Callable' = None, synapse_causal_lm_next: 'Callable' = None, synapse_seq_2_seq: 'Callable' = None, + synapse_lasthidden_timeout: int = None, + synapse_causallm_timeout: int = None, + synapse_causallmnext_timeout: int = None, + synapse_seq2seq_timeout: int = None, synapse_checks: 'Callable' = None, thread_pool: 'futures.ThreadPoolExecutor' = None, + priority_threadpool: 'bittensor.prioritythreadpool' = None, server: 'grpc._Server' = None, port: int = None, ip: str = None, @@ -120,6 +125,10 @@ def __new__( config.axon.forward_timeout = forward_timeout if forward_timeout != None else config.axon.forward_timeout config.axon.backward_timeout = backward_timeout if backward_timeout != None else config.axon.backward_timeout config.axon.compression = compression if compression != None else config.axon.compression + config.axon.lasthidden_timeout = synapse_lasthidden_timeout if synapse_lasthidden_timeout != None else config.axon.lasthidden_timeout + config.axon.causallm_timeout = synapse_causallm_timeout if synapse_causallm_timeout != None else config.axon.causallm_timeout + config.axon.causallmnext_timeout = synapse_causallmnext_timeout if synapse_causallmnext_timeout is not None else config.axon.causallmnext_timeout + config.axon.seq2seq_timeout = synapse_seq2seq_timeout if synapse_seq2seq_timeout != None else config.axon.seq2seq_timeout axon.check_config( config ) # Determine the grpc compression algorithm @@ -147,15 +156,20 @@ def __new__( synapses[bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM] = synapse_causal_lm synapses[bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT] = synapse_causal_lm_next synapses[bittensor.proto.Synapse.SynapseType.TEXT_SEQ_2_SEQ] = synapse_seq_2_seq + + synapse_timeouts = { + bittensor.proto.Synapse.SynapseType.TEXT_LAST_HIDDEN_STATE: config.axon.lasthidden_timeout, + bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM: config.axon.causallm_timeout, + bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT: config.axon.causallmnext_timeout, + bittensor.proto.Synapse.SynapseType.TEXT_SEQ_2_SEQ: config.axon.seq2seq_timeout + } synapse_check_function = synapse_checks if synapse_checks != None else axon.default_synapse_check - if priority != None: + if priority != None and priority_threadpool == None: priority_threadpool = bittensor.prioritythreadpool(config=config) - else: - priority_threadpool = None - axon_instance = axon_impl.Axon( + axon_instance = axon_impl.Axon( wallet = wallet, server = server, ip = config.axon.ip, @@ -164,6 +178,7 @@ def __new__( backward = backward_text, synapses = synapses, synapse_checks = synapse_check_function, + synapse_timeouts = synapse_timeouts, priority = priority, priority_threadpool = priority_threadpool, forward_timeout = config.axon.forward_timeout, @@ -217,6 +232,14 @@ def add_args( cls, parser: argparse.ArgumentParser, prefix: str = None ): help='''maximum size of tasks in priority queue''', default = bittensor.defaults.axon.priority.maxsize) parser.add_argument('--' + prefix_str + 'axon.compression', type=str, help='''Which compression algorithm to use for compression (gzip, deflate, NoCompression) ''', default = bittensor.defaults.axon.compression) + parser.add_argument('--' + prefix_str + 'axon.lasthidden_timeout', type = int, + help='Timeout for last hidden synapse', default= bittensor.__blocktime__) + parser.add_argument('--' + prefix_str + 'axon.causallm_timeout', type = int, + help='Timeout for causallm synapse', default= bittensor.__blocktime__) + parser.add_argument('--' + prefix_str + 'axon.causallmnext_timeout', type = int, + help='Timeout for causallmnext synapse', default= bittensor.__blocktime__) + parser.add_argument('--' + prefix_str + 'axon.seq2seq_timeout', type = int, + help='Timeout for seq2seq synapse', default= 3*bittensor.__blocktime__) except argparse.ArgumentError: # re-parsing arguments. pass @@ -325,7 +348,6 @@ def intercept_service(self, continuation, handler_call_details): self.message = str(e) return self._deny - def vertification(self,meta): r"""vertification of signature in metadata. Uses the pubkey and nounce """ diff --git a/bittensor/_axon/axon_impl.py b/bittensor/_axon/axon_impl.py index 2aafa4f9b7..8edc91cf98 100644 --- a/bittensor/_axon/axon_impl.py +++ b/bittensor/_axon/axon_impl.py @@ -32,6 +32,7 @@ import bittensor import bittensor.utils.stats as stat_utils +from datetime import datetime logger = logger.opt(colors=True) @@ -48,6 +49,7 @@ def __init__( backward: 'Callable', synapses: dict, synapse_checks: 'Callable', + synapse_timeouts: dict, priority: 'Callable' = None, priority_threadpool: 'bittensor.prioritythreadpool' = None, forward_timeout: int = None, @@ -81,6 +83,7 @@ def __init__( self.backward_timeout = backward_timeout self.synapse_callbacks = synapses self.synapse_checks = synapse_checks + self.synapse_timeouts = synapse_timeouts self.stats = self._init_stats() self.started = None self.optimizer_step = None @@ -184,6 +187,7 @@ def _forward(self, request): synapse_responses = [ synapse.empty() for synapse in synapses ] # We fill nones for non success. synapse_is_response = [ False for _ in synapses ] synapse_call_times = [ 0 for _ in synapses ] + synapse_timeout = min( [self.synapse_timeouts[s.synapse_type] for s in synapses] + [bittensor.__blocktime__] ) start_time = clock.time() # ================================================================== @@ -199,7 +203,7 @@ def check_if_should_return() -> bool: # ============================================================== # ==== Function which prints all log statements per synapse ==== # ============================================================== - def finalize_codes_stats_and_logs(): + def finalize_codes_stats_and_logs( message = None): for index, synapse in enumerate( synapses ): request.synapses [ index ].return_code = synapse_codes[ index ] # Set synapse wire proto codes. request.synapses [ index ].message = synapse_messages[ index ] # Set synapse wire proto message @@ -212,7 +216,7 @@ def finalize_codes_stats_and_logs(): pubkey = request.hotkey, inputs = synapse_inputs [index] , outputs = None if synapse_responses[index] == None else list( synapse_responses[index].shape ), - message = synapse_messages[ index ], + message = synapse_messages[ index ] if message == None else message, synapse = synapse.synapse_type ) @@ -280,9 +284,9 @@ def finalize_codes_stats_and_logs(): inputs_x = deserialized_forward_tensors, synapses = synapses, priority = priority, - hotkey= request.hotkey + hotkey = request.hotkey ) - forward_response_tensors, forward_codes, forward_messages = future.result( timeout= self.forward_timeout ) + forward_response_tensors, forward_codes, forward_messages = future.result( timeout = synapse_timeout - (clock.time() - start_time) ) else: forward_response_tensors, forward_codes, forward_messages = self.forward_callback( @@ -318,11 +322,10 @@ def finalize_codes_stats_and_logs(): except Exception as e: code = bittensor.proto.ReturnCode.UnknownException call_time = clock.time() - start_time - message = str ( e ) synapse_codes = [code for _ in synapses ] synapse_call_times = [call_time for _ in synapses ] - synapse_messages = [ message for _ in synapses ] - finalize_codes_stats_and_logs() + synapse_messages = [ 'Exception on Server' for _ in synapses ] + finalize_codes_stats_and_logs(message = str(e)) return [], bittensor.proto.ReturnCode.UnknownException, request.synapses # ================================================= diff --git a/bittensor/_dendrite/dendrite_impl.py b/bittensor/_dendrite/dendrite_impl.py index de117f0fe9..2b973ae925 100644 --- a/bittensor/_dendrite/dendrite_impl.py +++ b/bittensor/_dendrite/dendrite_impl.py @@ -860,4 +860,3 @@ def to_wandb( self ): except Exception as e: bittensor.logging.error( prefix='failed dendrite.to_wandb()', sufix = str(e)) return {} - diff --git a/bittensor/_neuron/text/core_server/__init__.py b/bittensor/_neuron/text/core_server/__init__.py index 2fefb88748..e1ee4c17fb 100644 --- a/bittensor/_neuron/text/core_server/__init__.py +++ b/bittensor/_neuron/text/core_server/__init__.py @@ -73,7 +73,6 @@ def __init__( causallmnext = None, seq2seq = None, synapse_list = None, - ): if config == None: config = server.config() config = config; diff --git a/bittensor/_neuron/text/core_server/nucleus_impl.py b/bittensor/_neuron/text/core_server/nucleus_impl.py index 661e0cf38e..f2d7d2c19b 100644 --- a/bittensor/_neuron/text/core_server/nucleus_impl.py +++ b/bittensor/_neuron/text/core_server/nucleus_impl.py @@ -123,7 +123,6 @@ def __init__(self, # -- keeps track of gradients applied self.backward_gradients_count = 0 - def set_fine_tuning_params(self) -> Tuple[bool, str]: r''' Set to tune only the parameter of the last layer Returns: @@ -538,7 +537,7 @@ def config (): parser.add_argument('--neuron.blacklist.stake', type=float, help='Amount of stake (tao) in order not to get blacklisted', default=10) parser.add_argument('--neuron.blocks_per_epoch', type=int, help='Blocks per epoch', default=10) parser.add_argument('--neuron.blacklist.time', type=int, help='how often a peer can query you (seconds) ', default=1) - parser.add_argument('--neuron.blocks_per_set_weights', type=float, help='how often to set weights', default=100) + parser.add_argument('--neuron.blocks_per_set_weights', type=float, help='how often to set weights', default=-1) parser.add_argument('--neuron.metagraph_sync', type=float, help='how often to sync the metagraph', default=100000) parser.add_argument('--neuron.blacklist_allow_non_registered', action='store_true', help='''If true, allow non-registered peers''', default=False) parser.add_argument('--neuron.disable_blacklist', action='store_true', help='Turns off blacklisting', default=False) @@ -564,4 +563,3 @@ def config (): bittensor.dataset.add_args( parser ) bittensor.metagraph.add_args( parser ) return bittensor.config( parser ) - diff --git a/bittensor/_neuron/text/core_server/run.py b/bittensor/_neuron/text/core_server/run.py index acc095bef3..9ef5a65630 100644 --- a/bittensor/_neuron/text/core_server/run.py +++ b/bittensor/_neuron/text/core_server/run.py @@ -331,7 +331,8 @@ def backward_callback(inputs_x:torch.FloatTensor, grads_dy:torch.FloatTensor, sy ) last_set_block = subtensor.get_current_block() - + blocks_per_epoch = subtensor.blocks_per_epoch if config.neuron.blocks_per_epoch == -1 else config.neuron.blocks_per_epoch + blocks_per_set_weights = subtensor.blocks_per_epoch if config.neuron.blocks_per_set_weights == -1 else config.neuron.blocks_per_set_weights # --- Run Forever. while True: @@ -404,7 +405,7 @@ def backward_callback(inputs_x:torch.FloatTensor, grads_dy:torch.FloatTensor, sy wandb.log( { **wandb_data, **wandb_info_axon, **local_data }, step = current_block ) wandb.log( { 'stats': wandb.Table( dataframe = df ) }, step = current_block ) - if current_block - last_set_block > config.neuron.blocks_per_set_weights: + if current_block - last_set_block > blocks_per_set_weights: try: bittensor.__console__.print('[green]Current Status:[/green]', {**wandb_data, **local_data}) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index ab9b86acf3..f858ecdf77 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -551,6 +551,7 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): responsive_uids = [] for _uid, _stats in neuron_stats.items(): stats = self.neuron_stats.setdefault(_uid, {}) + responsive_uids += [_uid] # === EMA zeroing update === # Push zero into EMA for synapse_keys to exponentially decay weighting keys if neuron non-responsive diff --git a/bittensor/_threadpool/__init__.py b/bittensor/_threadpool/__init__.py index 63cc8d4a24..d04e0f4c14 100644 --- a/bittensor/_threadpool/__init__.py +++ b/bittensor/_threadpool/__init__.py @@ -48,7 +48,6 @@ def __new__( config.axon.priority.maxsize = maxsize if maxsize != None else config.axon.priority.maxsize prioritythreadpool.check_config( config ) - return priority_thread_pool_impl.PriorityThreadPoolExecutor(maxsize = config.axon.priority.maxsize, max_workers = config.axon.priority.max_workers) @classmethod diff --git a/bittensor/_threadpool/priority_thread_pool_impl.py b/bittensor/_threadpool/priority_thread_pool_impl.py index a162d4f680..adcabbe8f2 100644 --- a/bittensor/_threadpool/priority_thread_pool_impl.py +++ b/bittensor/_threadpool/priority_thread_pool_impl.py @@ -166,10 +166,10 @@ def submit(self, fn, *args, **kwargs): start_time = time.time() if 'priority' in kwargs: del kwargs['priority'] + f = _base.Future() w = _WorkItem(f, fn, start_time, args, kwargs) - self._work_queue.put((-float(priority + eplison), w), block=False) self._adjust_thread_count() return f diff --git a/tests/integration_tests/test_dendrite.py b/tests/integration_tests/test_dendrite.py index df0920441b..b2a243eb14 100644 --- a/tests/integration_tests/test_dendrite.py +++ b/tests/integration_tests/test_dendrite.py @@ -15,6 +15,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import time import torch import pytest import bittensor @@ -260,9 +261,6 @@ def test_dendrite_multiple(): def test_dendrite_to_df(): dendrite.to_dataframe(bittensor.metagraph(_mock=True).sync()) -def test_dend_del(): - dendrite.__del__() - def test_successful_synapse(): wallet = bittensor.wallet() def forward_generate( inputs_x, synapse, model_output = None): @@ -310,7 +308,7 @@ def forward_casual_lm_next(inputs_x, synapse, model_output=None): print(codes) assert list(codes[0]) == [bittensor.proto.ReturnCode.Success] * len(synapses) - + def test_failing_synapse(): wallet = bittensor.wallet() def faulty( inputs_x, synapse, model_output = None): @@ -370,6 +368,8 @@ def forward_casual_lm_next(inputs_x, synapse, model_output=None): axon.attach_synapse_callback(faulty, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT) return_tensors, codes, times = dendrite.text(endpoints=endpoint, inputs=inputs, synapses=synapses) assert list(codes[0]) == [bittensor.proto.ReturnCode.UnknownException] * len(synapses) + + axon.stop() def test_missing_synapse(): wallet = bittensor.wallet() @@ -424,9 +424,62 @@ def forward_casual_lm_next(inputs_x, synapse, model_output=None): assert list(codes[0]) == [bittensor.proto.ReturnCode.Success, bittensor.proto.ReturnCode.Success, bittensor.proto.ReturnCode.Success, bittensor.proto.ReturnCode.NotImplemented] + axon.stop() + +def test_dendrite_timeout(): + wallet = bittensor.wallet() + def forward_hidden_state( inputs_x, synapse, model_output = None): + time.sleep(3) + return None, None, torch.rand(inputs_x.shape[0], inputs_x.shape[1], bittensor.__network_dim__) + + def forward_casual_lm(inputs_x, synapse, model_output = None): + time.sleep(3) + return None, None, torch.rand(inputs_x.shape[0], inputs_x.shape[1], bittensor.__vocab_size__) + + def forward_casual_lm_next(inputs_x, synapse, model_output=None): + time.sleep(3) + return None, None, synapse.nill_forward_response_tensor(inputs_x) + + axon = bittensor.axon ( + port = 8098, + ip = '0.0.0.0', + wallet = wallet, + ) + + axon.start() + + endpoint = bittensor.endpoint( + version = bittensor.__version_as_int__, + uid = 0, + hotkey = wallet.hotkey.ss58_address, + ip = '0.0.0.0', + ip_type = 4, + port = 8098, + modality = 0, + coldkey = wallet.coldkeypub.ss58_address + ) + + dendrite = bittensor.dendrite() + inputs = next(dataset) + synapses = [bittensor.synapse.TextLastHiddenState(), bittensor.synapse.TextCausalLM(), + bittensor.synapse.TextCausalLMNext()] + + axon.attach_synapse_callback( forward_hidden_state, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_LAST_HIDDEN_STATE ) + axon.attach_synapse_callback( forward_casual_lm, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM ) + axon.attach_synapse_callback(forward_casual_lm_next, synapse_type=bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT) + + return_tensors, codes, times = dendrite.text( endpoints=endpoint, inputs=inputs, synapses=synapses, timeout = 2) + assert list(codes[0]) == [bittensor.proto.ReturnCode.Timeout, bittensor.proto.ReturnCode.Timeout, + bittensor.proto.ReturnCode.Timeout] + + axon.stop() + +def test_dend_del(): + dendrite.__del__() + def test_clear(): dataset.close() if __name__ == "__main__": bittensor.logging(debug = True) - test_dendrite_forward_tensor() \ No newline at end of file + test_dendrite_timeout() \ No newline at end of file diff --git a/tests/integration_tests/test_wallet.py b/tests/integration_tests/test_wallet.py index 505b1cb631..ad86abff3b 100644 --- a/tests/integration_tests/test_wallet.py +++ b/tests/integration_tests/test_wallet.py @@ -125,7 +125,6 @@ def test_wallet_coldkeypub_create(): check_keys_exists(the_wallet, coldkey_exists=False, hotkey_exists=False) # Don't check the coldkey or hotkey assert the_wallet.coldkeypub.ss58_address == "5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" - def test_wallet_add_stake(): subtensor = bittensor.subtensor(network = 'nobunaga') the_wallet = init_wallet().create(coldkey_use_password = False, hotkey_use_password = False) @@ -167,7 +166,6 @@ def test_wallet_transfer(): the_wallet.is_registered = MagicMock(return_value = False) the_wallet.remove_stake(subtensor = subtensor) - def test_wallet_mock(): wallet = bittensor.wallet(_mock=True) assert wallet.hotkey_file.exists_on_device() @@ -177,7 +175,6 @@ def test_wallet_mock(): assert wallet.coldkey assert wallet.coldkeypub - def test_wallet_mock_from_config(): config = bittensor.wallet.config() config.wallet.name = 'mock' @@ -189,7 +186,6 @@ def test_wallet_mock_from_config(): assert wallet.coldkey assert wallet.coldkeypub - def test_wallet_mock_from_name(): wallet = bittensor.wallet(name = 'mock') assert wallet.hotkey_file.exists_on_device() @@ -199,7 +195,6 @@ def test_wallet_mock_from_name(): assert wallet.coldkey assert wallet.coldkeypub - def test_wallet_mock_from_func(): wallet = bittensor.wallet.mock() assert wallet.hotkey_file.exists_on_device() diff --git a/tests/unit_tests/bittensor_tests/test_axon.py b/tests/unit_tests/bittensor_tests/test_axon.py index 20040e0b3f..8f2bfd22fa 100644 --- a/tests/unit_tests/bittensor_tests/test_axon.py +++ b/tests/unit_tests/bittensor_tests/test_axon.py @@ -27,6 +27,8 @@ from bittensor.utils.test_utils import get_random_unused_port import concurrent +from concurrent.futures import ThreadPoolExecutor + wallet = bittensor.wallet.mock() axon = bittensor.axon(wallet = wallet) bittensor.logging(debug = True) @@ -812,6 +814,70 @@ def forward( inputs_x: torch.FloatTensor, synapses , model_output = None): response, code, synapses = axon._forward( request ) assert code == bittensor.proto.ReturnCode.Success + +def test_forward_priority_timeout(): + def priority(pubkey:str, request_type:str, inputs_x): + return 100 + + def forward( inputs_x: torch.FloatTensor, synapses, hotkey): + time.sleep(15) + + axon = bittensor.axon(wallet = wallet, priority= priority, forward_timeout = 5) + axon.attach_forward_callback(forward) + + inputs_raw = torch.rand(1,1) + synapses = [bittensor.synapse.TextLastHiddenState()] + serializer = bittensor.serializer( serializer_type = bittensor.proto.Serializer.MSGPACK ) + inputs_serialized = serializer.serialize(inputs_raw, from_type = bittensor.proto.TensorType.TORCH) + request = bittensor.proto.TensorMessage( + version = bittensor.__version_as_int__, + tensors=[inputs_serialized], + hotkey = axon.wallet.hotkey.ss58_address, + synapses= [ syn.serialize_to_wire_proto() for syn in synapses ] + ) + + response, code, synapses = axon._forward( request ) + assert code == bittensor.proto.ReturnCode.Timeout + + axon.stop() + +def test_forward_priority_2nd_request_timeout(): + def priority(pubkey:str, request_type:str, inputs_x): + return 100 + + axon = bittensor.axon(wallet = wallet, priority= priority, priority_threadpool = bittensor.prioritythreadpool(max_workers = 1)) + + def forward( inputs_x: torch.FloatTensor, synapses , model_output = None): + time.sleep(1) + return None, dict(), torch.zeros( [inputs_x.shape[0], inputs_x.shape[1], bittensor.__network_dim__]) + + axon.attach_synapse_callback( forward, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_LAST_HIDDEN_STATE) + inputs_raw = torch.rand(3, 3) + synapses = [bittensor.synapse.TextLastHiddenState()] + serializer = bittensor.serializer( serializer_type = bittensor.proto.Serializer.MSGPACK ) + inputs_serialized = synapses[0].serialize_forward_request_tensor(inputs_raw) + request = bittensor.proto.TensorMessage( + version = bittensor.__version_as_int__, + tensors=[inputs_serialized], + synapses= [ syn.serialize_to_wire_proto() for syn in synapses ], + hotkey = axon.wallet.hotkey.ss58_address, + ) + start_time = time.time() + executor = ThreadPoolExecutor(2) + future = executor.submit(axon._forward, (request)) + future2 = executor.submit(axon._forward, (request)) + response, code, synapses = future.result() + assert code == bittensor.proto.ReturnCode.Success + + try: + response2, code2, synapses2 = future2.result(timeout = 1 - (time.time() - start_time)) + except concurrent.futures.TimeoutError: + pass + else: + raise AssertionError('Expected to Timeout') + + axon.stop() + def test_backward_response_success_text_priority(): def priority(pubkey:str, request_type:str, inputs_x): @@ -1044,5 +1110,7 @@ def test_axon_is_destroyed(): if __name__ == "__main__": # test_forward_joint_success() - test_forward_joint_missing_synapse() + # test_forward_joint_missing_synapse() + # test_forward_priority_timeout() + test_forward_priority_2nd_request_timeout() # test_forward_joint_faulty_synapse() \ No newline at end of file diff --git a/tests/unit_tests/bittensor_tests/test_config.py b/tests/unit_tests/bittensor_tests/test_config.py index 6713a5f5db..981bc3beab 100644 --- a/tests/unit_tests/bittensor_tests/test_config.py +++ b/tests/unit_tests/bittensor_tests/test_config.py @@ -21,6 +21,8 @@ import argparse import pytest +bittensor.logging(debug = True) + def test_loaded_config(): with pytest.raises(NotImplementedError): bittensor.Config(loaded_config=True) @@ -77,31 +79,21 @@ def test_prefix(): config_non_strict = bittensor.config( parser, strict=False) config_strict = bittensor.config( parser, strict=True) - bittensor.dendrite( config_strict ) - bittensor.dendrite( config_non_strict ) - bittensor.dendrite( config_strict.second ) - bittensor.dendrite( config_non_strict.second ) - - bittensor.axon( config_strict ) - bittensor.axon( config_non_strict ) - bittensor.axon( config_strict.second ) - bittensor.axon( config_non_strict.second ) - - bittensor.dataset( config_strict ) - bittensor.dataset( config_non_strict ) - bittensor.dataset( config_strict.second ) - bittensor.dataset( config_non_strict.second ) + bittensor.dendrite( config_strict ).__del__() + bittensor.dendrite( config_non_strict ).__del__() + bittensor.dendrite( config_strict.second ).__del__() + bittensor.dendrite( config_non_strict.second ).__del__() - bittensor.axon( config_strict ) - bittensor.axon( config_non_strict ) - bittensor.axon( config_strict.second ) - bittensor.axon( config_non_strict.second ) + bittensor.axon( config_strict ).stop() + bittensor.axon( config_non_strict ).stop() + bittensor.axon( config_strict.second ).stop() + bittensor.axon( config_non_strict.second ).stop() bittensor.metagraph( config_strict ) bittensor.metagraph( config_non_strict ) bittensor.metagraph( config_strict.second ) bittensor.metagraph( config_non_strict.second ) - + bittensor.wallet( config_strict ) bittensor.wallet( config_non_strict ) bittensor.wallet( config_strict.second ) @@ -138,6 +130,7 @@ def test_to_defaults(): config.to_defaults() if __name__ == "__main__": - test_loaded_config() - test_strict() - test_to_defaults() \ No newline at end of file + # test_loaded_config() + # test_strict() + # test_to_defaults() + test_prefix() \ No newline at end of file diff --git a/tests/unit_tests/bittensor_tests/test_forward_backward.py b/tests/unit_tests/bittensor_tests/test_forward_backward.py index 48296f2cc0..ee5fcb47ea 100644 --- a/tests/unit_tests/bittensor_tests/test_forward_backward.py +++ b/tests/unit_tests/bittensor_tests/test_forward_backward.py @@ -292,5 +292,5 @@ def test_dendrite_del(): del dendrite_mock if __name__ == "__main__": - test_dendrite_backward_multiple() + pass diff --git a/tests/unit_tests/bittensor_tests/test_neuron.py b/tests/unit_tests/bittensor_tests/test_neuron.py index 9fa79e7768..4d5195cb1f 100644 --- a/tests/unit_tests/bittensor_tests/test_neuron.py +++ b/tests/unit_tests/bittensor_tests/test_neuron.py @@ -260,7 +260,5 @@ def exit_early(*args, **kwargs): # Should try to register the neuron mock_register.assert_called_once() - - if __name__ == '__main__': pass diff --git a/tests/unit_tests/bittensor_tests/test_receptor.py b/tests/unit_tests/bittensor_tests/test_receptor.py index d1d3458be2..c1b2dc82c9 100644 --- a/tests/unit_tests/bittensor_tests/test_receptor.py +++ b/tests/unit_tests/bittensor_tests/test_receptor.py @@ -613,7 +613,7 @@ def forward_casual_lm_next(input, synapse): def test_axon_receptor_connection_forward_unimplemented(): axon = bittensor.axon ( - port = 8081, + port = 8091, ip = '127.0.0.1', wallet = wallet, ) @@ -624,7 +624,7 @@ def test_axon_receptor_connection_forward_unimplemented(): uid = 0, ip = '127.0.0.1', ip_type = 4, - port = 8081, + port = 8091, hotkey = wallet.hotkey.ss58_address, coldkey = wallet.coldkey.ss58_address, modality = 2 diff --git a/tests/unit_tests/bittensor_tests/test_receptor_pool.py b/tests/unit_tests/bittensor_tests/test_receptor_pool.py index 337be0923d..165e787198 100644 --- a/tests/unit_tests/bittensor_tests/test_receptor_pool.py +++ b/tests/unit_tests/bittensor_tests/test_receptor_pool.py @@ -72,14 +72,13 @@ def test_receptor_pool_backward(): torch.tensor([])]] receptor_pool.backward( endpoints, synapses, x, grads, timeout=1) - def test_receptor_pool_max_workers_forward(): neuron_obj2 = bittensor.endpoint( version = bittensor.__version_as_int__, uid = 0, ip = '0.0.0.1', ip_type = 4, - port = 12345, + port = 12346, hotkey = wallet2.hotkey.ss58_address, coldkey = wallet2.coldkey.ss58_address, modality = 0 @@ -116,6 +115,7 @@ def test_receptor_pool_forward_success(): tensors = [y_hidden_serialized, y_causallm_serialized, y_causallmnext_serialized, y_seq_2_seq_serialized] ) + receptor_pool = bittensor.receptor_pool(wallet=wallet,max_active_receptors=1) receptor_pool._get_or_create_receptor_for_endpoint(neuron_obj) receptor_pool.receptors[neuron_obj.hotkey].stub.Forward = MagicMock( return_value = mock_return_val ) resp1, codes, _ = receptor_pool.forward( endpoints, synapses, x, timeout=1) From 3862c9dd86e9f25f553ac62892efee09d752043d Mon Sep 17 00:00:00 2001 From: Eugene Date: Fri, 2 Sep 2022 12:10:17 -0400 Subject: [PATCH 25/28] version update to match github releases --- bittensor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/__init__.py b/bittensor/__init__.py index c92a69c810..091e5896d4 100644 --- a/bittensor/__init__.py +++ b/bittensor/__init__.py @@ -18,7 +18,7 @@ from rich.console import Console # Bittensor code and protocol version. -__version__ = '3.0.0' +__version__ = '3.3.3' version_split = __version__.split(".") __version_as_int__ = (100 * int(version_split[0])) + (10 * int(version_split[1])) + (1 * int(version_split[2])) From 9aefc9ee60488ad2189e7801cb9071ad12ee3162 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Fri, 2 Sep 2022 14:47:26 -0400 Subject: [PATCH 26/28] add check to add ws:// if needed (#896) Co-authored-by: Eugene-hu <85906264+Eugene-hu@users.noreply.github.com> --- bittensor/_subtensor/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index a994c1e79a..8c0be7c88f 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -137,11 +137,16 @@ def __new__( config.subtensor.chain_endpoint = subtensor.determine_chain_endpoint( bittensor.defaults.subtensor.network ) config.subtensor.network = bittensor.defaults.subtensor.network + # make sure it's wss:// or ws:// + endpoint_url: str = config.subtensor.chain_endpoint + if endpoint_url[0:6] != "wss://" and endpoint_url[0:5] != "ws://": + endpoint_url = "ws://{}".format(endpoint_url) + substrate = SubstrateInterface( ss58_format = bittensor.__ss58_format__, type_registry_preset='substrate-node-template', type_registry = __type_registery__, - url = "ws://{}".format(config.subtensor.chain_endpoint), + url = endpoint_url, use_remote_preset=True ) From 8126e87848638716a85db754afc906ceff905045 Mon Sep 17 00:00:00 2001 From: opentaco Date: Sun, 4 Sep 2022 12:14:21 +0200 Subject: [PATCH 27/28] Remove extra responsive_uids insertion in validator --- bittensor/_neuron/text/core_validator/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index f52f37109f..226b335a00 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -552,7 +552,6 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): responsive_uids = [] for _uid, _stats in neuron_stats.items(): stats = self.neuron_stats.setdefault(_uid, {}) - responsive_uids += [_uid] # === EMA zeroing update === # Push zero into EMA for synapse_keys to exponentially decay weighting keys if neuron non-responsive From 21a261b33e335fa79b8d2290171d350a065b8bde Mon Sep 17 00:00:00 2001 From: opentaco Date: Sun, 4 Sep 2022 12:23:16 +0200 Subject: [PATCH 28/28] Constrain validator epoch length to a maximum period Normal epoch duration is blocks_per_epoch if all UIDs have been queried. Try to query each UID at least once - assumes nucleus samples without replacement, but keep maximum epoch duration at 2 * blocks_per_epoch. Typically a low resource validator can complete full network querying in ~350 blocks, so 2 * blocks_per_epoch should normally suffice. --- bittensor/_neuron/text/core_validator/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 226b335a00..57e1f840be 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -377,8 +377,12 @@ def run_epoch( self ): epoch_start_time = time.time() start_block = self.subtensor.block + # normal epoch duration is blocks_per_epoch if all UIDs have been queried + # try to query each UID at least once - assumes nucleus samples without replacement + # but keep maximum epoch duration at 2 * blocks_per_epoch while (self.subtensor.block < start_block + blocks_per_epoch or - len(epoch_queried_uids) < self.metagraph.n): # ensure each UID is queried at least once - assumes nucleus samples without replacement + (self.subtensor.block < start_block + 2 * blocks_per_epoch and + len(epoch_queried_uids) < self.metagraph.n)): start_time = time.time() # === Forward ===