diff --git a/openvalidators/forward.py b/openvalidators/forward.py index 19aa0cb..e24198a 100644 --- a/openvalidators/forward.py +++ b/openvalidators/forward.py @@ -34,6 +34,7 @@ ) from openvalidators.utils import check_uid_availability + def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor: """Returns k available random uids from the metagraph. Args: @@ -48,7 +49,7 @@ def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor for uid in range(self.metagraph.n.item()): uid_is_available = check_uid_availability(self.metagraph, uid, self.config.neuron.vpermit_tao_limit) - uid_is_not_excluded = (exclude is None or uid not in exclude) + uid_is_not_excluded = exclude is None or uid not in exclude if uid_is_available and uid_is_not_excluded: candidate_uids.append(uid) @@ -76,8 +77,9 @@ def is_successful_completion(self, response: bt.DendriteCall, min_len: int = 10, return len_check and filter_check -async def scoring_completions(self, prompt: str, scoring_template: str, responses: List[bt.DendriteCall], - exclude_uids: List[int] = None) -> Dict: +async def scoring_completions( + self, prompt: str, scoring_template: str, responses: List[bt.DendriteCall], exclude_uids: List[int] = None +) -> Dict: """Using the prompt and call responses, outsource prompt-based scoring to network, return scoring average for each response. @@ -211,7 +213,7 @@ def reward_completions(self, prompt: str, responses: List[bt.DendriteCall]) -> t ).to(self.device) # Fill scores with zeros for non successful responses. - successful_rewards = successful_rewards.softmax( 0 ) + successful_rewards = successful_rewards.softmax(0) filled_rewards = torch.zeros(len(responses), dtype=torch.float32) for idx, reward in zip(successful_completions_indices, successful_rewards): filled_rewards[idx] = reward @@ -298,9 +300,13 @@ async def forward(self): # Prompt-based scoring via network. Prohibits self-scoring. if self.config.neuron.outsource_scoring: - followup_scoring = await scoring_completions(self, prompt=bootstrap_prompt, - scoring_template=followup_scoring_template, - responses=followup_responses, exclude_uids=followup_uids) + followup_scoring = await scoring_completions( + self, + prompt=bootstrap_prompt, + scoring_template=followup_scoring_template, + responses=followup_responses, + exclude_uids=followup_uids, + ) # Backward call sends reward info back to followup_uids. _followup_backward = await self.dendrite_pool.async_backward( @@ -329,8 +335,13 @@ async def forward(self): # Prompt-based scoring via network. Prohibits self-scoring. if self.config.neuron.outsource_scoring: - answer_scoring = await scoring_completions(self, prompt=answer_prompt, scoring_template=answer_scoring_template, - responses=answer_responses, exclude_uids=answer_uids) + answer_scoring = await scoring_completions( + self, + prompt=answer_prompt, + scoring_template=answer_scoring_template, + responses=answer_responses, + exclude_uids=answer_uids, + ) # Backward call sends reward info back to answer_uids. _answer_backward = await self.dendrite_pool.async_backward( @@ -384,8 +395,8 @@ async def forward(self): ) if self.config.neuron.outsource_scoring: - event.update({f'followup_{k}': v for k, v in followup_scoring.items()}) - event.update({f'answer_{k}': v for k, v in answer_scoring.items()}) + event.update({f"followup_{k}": v for k, v in followup_scoring.items()}) + event.update({f"answer_{k}": v for k, v in answer_scoring.items()}) bt.logging.debug("step:", str(event)) # Log to wandb. @@ -400,3 +411,5 @@ async def forward(self): # Log locally if not self.config.neuron.dont_save_events: logger.log("EVENTS", "events", **event) + + return event diff --git a/openvalidators/mock.py b/openvalidators/mock.py index 83991a9..1df572d 100644 --- a/openvalidators/mock.py +++ b/openvalidators/mock.py @@ -105,3 +105,12 @@ async def query(): def resync(self, metagraph): pass + + async def async_backward( + self, uids: List[int], roles: List[str], messages: List[str], completions: List[str], rewards: List[float] + ): + async def query(): + await asyncio.sleep(0.01) + return [MockDendriteResponse(messages[0]) for _ in uids] + + return await query() diff --git a/openvalidators/neuron.py b/openvalidators/neuron.py index ab0bb39..512abfe 100644 --- a/openvalidators/neuron.py +++ b/openvalidators/neuron.py @@ -73,7 +73,8 @@ def __init__(self): bt.logging.debug("loading", "wallet") self.wallet = bt.wallet(config=self.config) self.wallet.create_if_non_existent() - self.wallet.reregister(subtensor=self.subtensor, netuid=self.config.netuid) + if not self.config.wallet._mock: + self.wallet.reregister(subtensor=self.subtensor, netuid=self.config.netuid) bt.logging.debug(str(self.wallet)) # Init metagraph. diff --git a/tests/test_weights.py b/tests/test_weights.py new file mode 100644 index 0000000..fd8dec1 --- /dev/null +++ b/tests/test_weights.py @@ -0,0 +1,57 @@ +# The MIT License (MIT) +# Copyright © 2021 Yuma Rao + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +import torch +import copy +import asyncio +import sys +from openvalidators.neuron import neuron as Neuron +from openvalidators.forward import forward + +CLI_ARGS_STR = "validators/openvalidators/neuron.py --mock --wallet._mock --wandb.off --neuron.followup_sample_size 10 --neuron.answer_sample_size 10" + +SYS_ARGV = sys.argv.copy() + + +def test_uid_weights_unchanged_unless_queried(n_steps=10, n_concurrent=1): + """Test that the weights of unqueried uids do not over the course of a forward pass.""" + + sys.argv = CLI_ARGS_STR.split(" ") + neuron = Neuron() + + for _ in range(n_steps): + + prev_scores = copy.deepcopy(neuron.moving_averaged_scores) + + # run concurrent forward passes + async def run_forward(): + coroutines = [forward(neuron) for _ in range(n_concurrent)] + return await asyncio.gather(*coroutines) + + events = neuron.loop.run_until_complete(run_forward()) + # moving_averaged_scores updates are not thread safe, so I don't think we can run concurrent forwards + for event in events: + + # get current scores + next_scores = copy.deepcopy(neuron.moving_averaged_scores) + + queried_uids = sorted(set(event["followup_uids"] + event["answer_uids"])) + ignored_uids = [uid for uid in torch.arange(neuron.metagraph.n.item()) if uid not in queried_uids] + + # ther is a floating point difference (~1e-10) between the scores, so we can't use exact equality + assert next_scores[ignored_uids].allclose(prev_scores[ignored_uids]), "Unqueried uids should not change" + + sys.argv = SYS_ARGV.copy()