Test weight updates

openvalidators/forward.py

@@ -34,6 +34,7 @@
 )
 from openvalidators.utils import check_uid_availability
 
+
 def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor:
     """Returns k available random uids from the metagraph.
     Args:
@@ -48,7 +49,7 @@ def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor
 
     for uid in range(self.metagraph.n.item()):
         uid_is_available = check_uid_availability(self.metagraph, uid, self.config.neuron.vpermit_tao_limit)
-        uid_is_not_excluded = (exclude is None or uid not in exclude)
+        uid_is_not_excluded = exclude is None or uid not in exclude
 
         if uid_is_available and uid_is_not_excluded:
             candidate_uids.append(uid)
@@ -76,8 +77,9 @@ def is_successful_completion(self, response: bt.DendriteCall, min_len: int = 10,
     return len_check and filter_check
 
 
-async def scoring_completions(self, prompt: str, scoring_template: str, responses: List[bt.DendriteCall],
-                              exclude_uids: List[int] = None) -> Dict:
+async def scoring_completions(
+    self, prompt: str, scoring_template: str, responses: List[bt.DendriteCall], exclude_uids: List[int] = None
+) -> Dict:
     """Using the prompt and call responses, outsource prompt-based scoring to network,
        return scoring average for each response.
 
@@ -211,7 +213,7 @@ def reward_completions(self, prompt: str, responses: List[bt.DendriteCall]) -> t
     ).to(self.device)
 
     # Fill scores with zeros for non successful responses.
-    successful_rewards = successful_rewards.softmax( 0 )
+    successful_rewards = successful_rewards.softmax(0)
     filled_rewards = torch.zeros(len(responses), dtype=torch.float32)
     for idx, reward in zip(successful_completions_indices, successful_rewards):
         filled_rewards[idx] = reward
@@ -298,9 +300,13 @@ async def forward(self):
 
     # Prompt-based scoring via network. Prohibits self-scoring.
     if self.config.neuron.outsource_scoring:
-        followup_scoring = await scoring_completions(self, prompt=bootstrap_prompt,
-                                                     scoring_template=followup_scoring_template,
-                                                     responses=followup_responses, exclude_uids=followup_uids)
+        followup_scoring = await scoring_completions(
+            self,
+            prompt=bootstrap_prompt,
+            scoring_template=followup_scoring_template,
+            responses=followup_responses,
+            exclude_uids=followup_uids,
+        )
 
     # Backward call sends reward info back to followup_uids.
     _followup_backward = await self.dendrite_pool.async_backward(
@@ -329,8 +335,13 @@ async def forward(self):
 
     # Prompt-based scoring via network. Prohibits self-scoring.
     if self.config.neuron.outsource_scoring:
-        answer_scoring = await scoring_completions(self, prompt=answer_prompt, scoring_template=answer_scoring_template,
-                                                   responses=answer_responses, exclude_uids=answer_uids)
+        answer_scoring = await scoring_completions(
+            self,
+            prompt=answer_prompt,
+            scoring_template=answer_scoring_template,
+            responses=answer_responses,
+            exclude_uids=answer_uids,
+        )
 
     # Backward call sends reward info back to answer_uids.
     _answer_backward = await self.dendrite_pool.async_backward(
@@ -384,8 +395,8 @@ async def forward(self):
         )
 
     if self.config.neuron.outsource_scoring:
-        event.update({f'followup_{k}': v for k, v in followup_scoring.items()})
-        event.update({f'answer_{k}': v for k, v in answer_scoring.items()})
+        event.update({f"followup_{k}": v for k, v in followup_scoring.items()})
+        event.update({f"answer_{k}": v for k, v in answer_scoring.items()})
 
     bt.logging.debug("step:", str(event))
     # Log to wandb.
@@ -400,3 +411,5 @@ async def forward(self):
     # Log locally
     if not self.config.neuron.dont_save_events:
         logger.log("EVENTS", "events", **event)
+
+    return event

openvalidators/mock.py

@@ -105,3 +105,12 @@ async def query():
 
     def resync(self, metagraph):
         pass
+
+    async def async_backward(
+        self, uids: List[int], roles: List[str], messages: List[str], completions: List[str], rewards: List[float]
+    ):
+        async def query():
+            await asyncio.sleep(0.01)
+            return [MockDendriteResponse(messages[0]) for _ in uids]
+
+        return await query()

openvalidators/neuron.py

@@ -73,7 +73,8 @@ def __init__(self):
         bt.logging.debug("loading", "wallet")
         self.wallet = bt.wallet(config=self.config)
         self.wallet.create_if_non_existent()
-        self.wallet.reregister(subtensor=self.subtensor, netuid=self.config.netuid)
+        if not self.config.wallet._mock:
+            self.wallet.reregister(subtensor=self.subtensor, netuid=self.config.netuid)
         bt.logging.debug(str(self.wallet))
 
         # Init metagraph.

tests/test_weights.py

@@ -0,0 +1,57 @@
+# The MIT License (MIT)
+# Copyright © 2021 Yuma Rao
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+import torch
+import copy
+import asyncio
+import sys
+from openvalidators.neuron import neuron as Neuron
+from openvalidators.forward import forward
+
+CLI_ARGS_STR = "validators/openvalidators/neuron.py --mock --wallet._mock --wandb.off --neuron.followup_sample_size 10 --neuron.answer_sample_size 10"
+
+SYS_ARGV = sys.argv.copy()
+
+
+def test_uid_weights_unchanged_unless_queried(n_steps=10, n_concurrent=1):
+    """Test that the weights of unqueried uids do not over the course of a forward pass."""
+
+    sys.argv = CLI_ARGS_STR.split(" ")
+    neuron = Neuron()
+
+    for _ in range(n_steps):
+
+        prev_scores = copy.deepcopy(neuron.moving_averaged_scores)
+
+        # run concurrent forward passes
+        async def run_forward():
+            coroutines = [forward(neuron) for _ in range(n_concurrent)]
+            return await asyncio.gather(*coroutines)
+
+        events = neuron.loop.run_until_complete(run_forward())
+        # moving_averaged_scores updates are not thread safe, so I don't think we can run concurrent forwards
+        for event in events:
+
+            # get current scores
+            next_scores = copy.deepcopy(neuron.moving_averaged_scores)
+
+            queried_uids = sorted(set(event["followup_uids"] + event["answer_uids"]))
+            ignored_uids = [uid for uid in torch.arange(neuron.metagraph.n.item()) if uid not in queried_uids]
+
+            # ther is a floating point difference (~1e-10) between the scores, so we can't use exact equality
+            assert next_scores[ignored_uids].allclose(prev_scores[ignored_uids]), "Unqueried uids should not change"
+
+    sys.argv = SYS_ARGV.copy()