Overall improvements for data extraction scripts + unit tests for check_uid_availability

analysis/utils.py

@@ -24,7 +24,7 @@
 from typing import List, Dict, Any, Union
 
 
-def get_runs(project: str = "openvalidators", filters: Dict[str, Any] = None, return_paths: bool = False) -> List:
+def get_runs(project: str = "opentensor-dev/openvalidators", filters: Dict[str, Any] = None, return_paths: bool = False) -> List:
     """Download runs from wandb.
 
     Args:
@@ -36,7 +36,7 @@ def get_runs(project: str = "openvalidators", filters: Dict[str, Any] = None, re
         List[wandb.apis.public.Run]: List of runs or run paths (List[str]).
     """
     api = wandb.Api()
-    wandb.login()
+    wandb.login(anonymous="allow")
 
     runs = api.runs(project, filters=filters)
     if return_paths:
@@ -56,7 +56,7 @@ def download_data(run_path: Union[str, List] = None, timeout: float = 600) -> pd
         pd.DataFrame: Dataframe of event log.
     """
     api = wandb.Api(timeout=timeout)
-    wandb.login()
+    wandb.login(anonymous="allow")
 
     if isinstance(run_path, str):
         run_path = [run_path]

openvalidators/forward.py

@@ -44,12 +44,14 @@ def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor
     Notes:
         If `k` is larger than the number of available `uids`, set `k` to the number of available `uids`.
     """
-    candidate_uids = [
-        uid
-        for uid in range(self.metagraph.n.item())
-        if check_uid_availability(self.metagraph, uid, self.config.neuron.vpermit_tao_limit)
-        and (exclude is None or uid not in exclude)
-    ]
+    candidate_uids = []
+
+    for uid in range(self.metagraph.n.item()):
+        uid_is_available = check_uid_availability(self.metagraph, uid, self.config.neuron.vpermit_tao_limit)
+        uid_is_not_excluded = (exclude is None or uid not in exclude)
+
+        if uid_is_available and uid_is_not_excluded:
+            candidate_uids.append(uid)
 
     available_uids = torch.tensor(candidate_uids, dtype=torch.int64).to(self.device)
     uids = torch.tensor(random.sample(available_uids.tolist(), k), dtype=torch.int64)

scripts/README.md

@@ -14,6 +14,24 @@ The current implementation requires you to be logged in wandb. You can do so by
 wandb login
 ```
 
+You will also need to install `make` to profit from the predefined targets in the [Makefile](Makefile). 
+To install `make` on Ubuntu, run the following command:
+```bash
+sudo apt install make
+```
+
+If you are on Mac, you can install it by running:
+```bash
+brew install make
+```
+
+
+**Note:** All the data collection scripts are designed to be run from the `scripts` directory, 
+so ensure that you are in the right place before starting:
+```bash
+ cd scripts # From root folder
+```
+
 ## Usage
 This repository provides a convenient way to collect data from the
 [WandB Openvalidators platform](https://wandb.ai/opentensor-dev/openvalidators) using the [data_collector.py](data_collector.py)

scripts/data_collector.py

@@ -26,7 +26,7 @@
 from traceback import print_exc
 
 
-DEFAULT_PROJECT = 'openvalidators'
+DEFAULT_PROJECT = 'opentensor-dev/openvalidators'
 DEFAULT_FILTERS = {"tags": {"$in": [openvalidators.__version__]}}
 
 

tests/test_utils.py

@@ -19,17 +19,22 @@
 import copy
 import unittest
 from unittest.mock import MagicMock
-from openvalidators.utils import resync_linear_layer
+from openvalidators.utils import resync_linear_layer, check_uid_availability
 
 
 class UtilsTestCase(unittest.TestCase):
     def setUp(self):
         """
-        Creates a mock metagraph with 1024 mock axons.
+        Creates a mock metagraph with 1024 mock axons before each test.
         """
         mock_metagraph = MagicMock(spec=bt.metagraph)
         mock_metagraph.uids = torch.tensor(range(0, 1024))
+        mock_metagraph.S = torch.zeros(1024)
         mock_metagraph.hotkeys = list(map(str, range(0, 1024)))
+        mock_metagraph.validator_permit = [False] * 1024
+        mock_metagraph.axons = [
+            MagicMock(spec=bt.axon_info, hotkey=str(num), ip="0.0.0.0/0", port=12345) for num in range(0, 1024)
+        ]
 
         self.metagraph = mock_metagraph
         self.keypair = "test"
@@ -63,6 +68,59 @@ def test_resync_linear_layer_multiple_updates(self):
                 self.assertEqual(linear_layer.bias[index].item(), 1)
                 self.assertTrue(torch.all(linear_layer.weight[index] == torch.ones(linear_layer.weight[index].shape)))
 
+    def test_check_uid_availability_not_serving_axon(self):
+        # Arrange: Create a non serving axon
+        uid = 1
+        self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=False)
+
+        # Act: Call the function to check if uid is available
+        result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=0)
+
+        # Assert: Ensure that the result is False (uid is available) when node doesn't have a serving axon
+        self.assertFalse(result)
+
+    def test_check_uid_availability_node_without_validator_permit(self):
+        # Arrange: Create a serving axon without validator permit
+        uid = 1
+        self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=True)
+        self.metagraph.validator_permit[uid] = False
+
+        # Act: Call the function to check if uid is available
+        result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=0)
+
+        # Assert: Ensure that the result is True (uid is available) when node does not have a validator permit
+        self.assertTrue(result)
+
+    def test_check_uid_availability_validator_with_stake_less_than_vpermit_tao_limit(self):
+        # Arrange: Create a serving axon with validator permit and stake less than vpermit_tao_limit
+        uid = 1
+        self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=True)
+        self.metagraph.validator_permit[uid] = True
+        self.metagraph.S[uid] = 1
+        v_permit_tao_limit = 2
+
+        # Act: Call the function to check if uid is available
+        result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=v_permit_tao_limit)
+
+        # Assert: Ensure that the result is True (uid is available) when node validator
+        # has stake less than vpermit_tao_limit
+        self.assertTrue(result)
+
+    def test_check_uid_availability_validator_with_stake_greater_than_vpermit_tao_limit(self):
+        # Arrange: Create a serving axon with validator permit and stake greater than vpermit_tao_limit
+        uid = 1
+        self.metagraph.axons[uid] = MagicMock(spec=bt.axon_info, is_serving=True)
+        self.metagraph.validator_permit[uid] = True
+        self.metagraph.S[uid] = 2
+        v_permit_tao_limit = 1
+
+        # Act: Call the function to check if uid is available
+        result = check_uid_availability(self.metagraph, uid, vpermit_tao_limit=v_permit_tao_limit)
+
+        # Assert: Ensure that the result is False (uid is available) when validator node
+        # has stake greater than vpermit_tao_limit
+        self.assertFalse(result)
+
 
 if __name__ == "__main__":
     unittest.main()