From a26b51a5289b16b887b327b164686dadc4173be1 Mon Sep 17 00:00:00 2001 From: Steffen Cruz Date: Fri, 9 Jun 2023 10:36:02 -0600 Subject: [PATCH 1/2] Adds unit test for weight setting, adds mock dendrite async backward, enables reregister to be skipped if wallet is mock --- openvalidators/forward.py | 35 +++++++++++++++------- openvalidators/mock.py | 9 ++++++ openvalidators/neuron.py | 3 +- tests/test_weights.py | 63 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 tests/test_weights.py diff --git a/openvalidators/forward.py b/openvalidators/forward.py index 19aa0cb..e24198a 100644 --- a/openvalidators/forward.py +++ b/openvalidators/forward.py @@ -34,6 +34,7 @@ ) from openvalidators.utils import check_uid_availability + def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor: """Returns k available random uids from the metagraph. Args: @@ -48,7 +49,7 @@ def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor for uid in range(self.metagraph.n.item()): uid_is_available = check_uid_availability(self.metagraph, uid, self.config.neuron.vpermit_tao_limit) - uid_is_not_excluded = (exclude is None or uid not in exclude) + uid_is_not_excluded = exclude is None or uid not in exclude if uid_is_available and uid_is_not_excluded: candidate_uids.append(uid) @@ -76,8 +77,9 @@ def is_successful_completion(self, response: bt.DendriteCall, min_len: int = 10, return len_check and filter_check -async def scoring_completions(self, prompt: str, scoring_template: str, responses: List[bt.DendriteCall], - exclude_uids: List[int] = None) -> Dict: +async def scoring_completions( + self, prompt: str, scoring_template: str, responses: List[bt.DendriteCall], exclude_uids: List[int] = None +) -> Dict: """Using the prompt and call responses, outsource prompt-based scoring to network, return scoring average for each response. @@ -211,7 +213,7 @@ def reward_completions(self, prompt: str, responses: List[bt.DendriteCall]) -> t ).to(self.device) # Fill scores with zeros for non successful responses. - successful_rewards = successful_rewards.softmax( 0 ) + successful_rewards = successful_rewards.softmax(0) filled_rewards = torch.zeros(len(responses), dtype=torch.float32) for idx, reward in zip(successful_completions_indices, successful_rewards): filled_rewards[idx] = reward @@ -298,9 +300,13 @@ async def forward(self): # Prompt-based scoring via network. Prohibits self-scoring. if self.config.neuron.outsource_scoring: - followup_scoring = await scoring_completions(self, prompt=bootstrap_prompt, - scoring_template=followup_scoring_template, - responses=followup_responses, exclude_uids=followup_uids) + followup_scoring = await scoring_completions( + self, + prompt=bootstrap_prompt, + scoring_template=followup_scoring_template, + responses=followup_responses, + exclude_uids=followup_uids, + ) # Backward call sends reward info back to followup_uids. _followup_backward = await self.dendrite_pool.async_backward( @@ -329,8 +335,13 @@ async def forward(self): # Prompt-based scoring via network. Prohibits self-scoring. if self.config.neuron.outsource_scoring: - answer_scoring = await scoring_completions(self, prompt=answer_prompt, scoring_template=answer_scoring_template, - responses=answer_responses, exclude_uids=answer_uids) + answer_scoring = await scoring_completions( + self, + prompt=answer_prompt, + scoring_template=answer_scoring_template, + responses=answer_responses, + exclude_uids=answer_uids, + ) # Backward call sends reward info back to answer_uids. _answer_backward = await self.dendrite_pool.async_backward( @@ -384,8 +395,8 @@ async def forward(self): ) if self.config.neuron.outsource_scoring: - event.update({f'followup_{k}': v for k, v in followup_scoring.items()}) - event.update({f'answer_{k}': v for k, v in answer_scoring.items()}) + event.update({f"followup_{k}": v for k, v in followup_scoring.items()}) + event.update({f"answer_{k}": v for k, v in answer_scoring.items()}) bt.logging.debug("step:", str(event)) # Log to wandb. @@ -400,3 +411,5 @@ async def forward(self): # Log locally if not self.config.neuron.dont_save_events: logger.log("EVENTS", "events", **event) + + return event diff --git a/openvalidators/mock.py b/openvalidators/mock.py index 83991a9..1df572d 100644 --- a/openvalidators/mock.py +++ b/openvalidators/mock.py @@ -105,3 +105,12 @@ async def query(): def resync(self, metagraph): pass + + async def async_backward( + self, uids: List[int], roles: List[str], messages: List[str], completions: List[str], rewards: List[float] + ): + async def query(): + await asyncio.sleep(0.01) + return [MockDendriteResponse(messages[0]) for _ in uids] + + return await query() diff --git a/openvalidators/neuron.py b/openvalidators/neuron.py index ab0bb39..512abfe 100644 --- a/openvalidators/neuron.py +++ b/openvalidators/neuron.py @@ -73,7 +73,8 @@ def __init__(self): bt.logging.debug("loading", "wallet") self.wallet = bt.wallet(config=self.config) self.wallet.create_if_non_existent() - self.wallet.reregister(subtensor=self.subtensor, netuid=self.config.netuid) + if not self.config.wallet._mock: + self.wallet.reregister(subtensor=self.subtensor, netuid=self.config.netuid) bt.logging.debug(str(self.wallet)) # Init metagraph. diff --git a/tests/test_weights.py b/tests/test_weights.py new file mode 100644 index 0000000..351dc97 --- /dev/null +++ b/tests/test_weights.py @@ -0,0 +1,63 @@ +# The MIT License (MIT) +# Copyright © 2021 Yuma Rao + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +import torch +import copy +import asyncio +import sys +from openvalidators.neuron import neuron as Neuron +from openvalidators.forward import forward + +CLI_ARGS = ( + "validators/openvalidators/neuron.py --mock --wallet._mock --wallet.reregister False --wandb.off --logging.trace".split( + " " + ) +) +argv = sys.argv.copy() + + +def test_uid_weights_unchanged_unless_queried(n_steps=10, n_concurrent=1): + """Test that the weights of unqueried uids do not over the course of a forward pass.""" + + sys.argv = CLI_ARGS + neuron = Neuron() + for _ in range(n_steps): + prev_scores = copy.deepcopy(neuron.moving_averaged_scores) + + # run concurrent forward passes + async def run_forward(): + coroutines = [forward(neuron) for _ in range(n_concurrent)] + return await asyncio.gather(*coroutines) + + events = neuron.loop.run_until_complete(run_forward()) + + # get current scores + next_scores = neuron.moving_averaged_scores + + print(f"Events has length {len(events)}") + # moving_averaged_scores updates are not thread safe, so I don't think we can run concurrent forwards + for event in events: + + queried_uids = list(set(event["followup_uids"] + event["answer_uids"])) + ignored_uids = [uid for uid in torch.arange(neuron.metagraph.n.item()) if uid not in queried_uids] + + print(f"Queried uids: {queried_uids}") + print(f"Previous weights of queried uids: {next_scores[queried_uids]}") + print(f"New weights of queried uids: {prev_scores[queried_uids]}") + + assert all(next_scores[ignored_uids] == prev_scores[ignored_uids]), "Unqueried uids should not change" + + sys.argv = argv.copy() From 8f08fd6567cf46e83469e991776e2b03b97c34f2 Mon Sep 17 00:00:00 2001 From: Steffen Cruz Date: Fri, 9 Jun 2023 11:29:45 -0600 Subject: [PATCH 2/2] Change exact equality condition to allclose, because of floating point diffs --- tests/test_weights.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/tests/test_weights.py b/tests/test_weights.py index 351dc97..fd8dec1 100644 --- a/tests/test_weights.py +++ b/tests/test_weights.py @@ -21,20 +21,19 @@ from openvalidators.neuron import neuron as Neuron from openvalidators.forward import forward -CLI_ARGS = ( - "validators/openvalidators/neuron.py --mock --wallet._mock --wallet.reregister False --wandb.off --logging.trace".split( - " " - ) -) -argv = sys.argv.copy() +CLI_ARGS_STR = "validators/openvalidators/neuron.py --mock --wallet._mock --wandb.off --neuron.followup_sample_size 10 --neuron.answer_sample_size 10" + +SYS_ARGV = sys.argv.copy() def test_uid_weights_unchanged_unless_queried(n_steps=10, n_concurrent=1): """Test that the weights of unqueried uids do not over the course of a forward pass.""" - sys.argv = CLI_ARGS + sys.argv = CLI_ARGS_STR.split(" ") neuron = Neuron() + for _ in range(n_steps): + prev_scores = copy.deepcopy(neuron.moving_averaged_scores) # run concurrent forward passes @@ -43,21 +42,16 @@ async def run_forward(): return await asyncio.gather(*coroutines) events = neuron.loop.run_until_complete(run_forward()) - - # get current scores - next_scores = neuron.moving_averaged_scores - - print(f"Events has length {len(events)}") # moving_averaged_scores updates are not thread safe, so I don't think we can run concurrent forwards for event in events: - queried_uids = list(set(event["followup_uids"] + event["answer_uids"])) - ignored_uids = [uid for uid in torch.arange(neuron.metagraph.n.item()) if uid not in queried_uids] + # get current scores + next_scores = copy.deepcopy(neuron.moving_averaged_scores) - print(f"Queried uids: {queried_uids}") - print(f"Previous weights of queried uids: {next_scores[queried_uids]}") - print(f"New weights of queried uids: {prev_scores[queried_uids]}") + queried_uids = sorted(set(event["followup_uids"] + event["answer_uids"])) + ignored_uids = [uid for uid in torch.arange(neuron.metagraph.n.item()) if uid not in queried_uids] - assert all(next_scores[ignored_uids] == prev_scores[ignored_uids]), "Unqueried uids should not change" + # ther is a floating point difference (~1e-10) between the scores, so we can't use exact equality + assert next_scores[ignored_uids].allclose(prev_scores[ignored_uids]), "Unqueried uids should not change" - sys.argv = argv.copy() + sys.argv = SYS_ARGV.copy()