From f8845de01acadb294f876f8f42a40834569e11f8 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 15 Aug 2022 20:44:19 +0200 Subject: [PATCH 01/12] Update rich>=12.5.1 to fix /[ delimitation --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1c7f9b8fd6..c96f9a27e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ pytest-rerunfailures coveralls pytest-cov pyyaml -rich +rich>=12.5.1 retry requests>=2.25.0 scalecodec>=1.0.35 From c95aeef1822b56ad9b710e555c1faecc107c38a9 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 15 Aug 2022 20:23:33 +0200 Subject: [PATCH 02/12] Count responsive uid when has synapse_keys not nan --- bittensor/_neuron/text/core_validator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 4329f9265b..d6c066110a 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -543,6 +543,7 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): zkey = key + '!' # zeroing key stats.setdefault(zkey, 0.) # initialize zkey val to zero to gradually increase with observations if key in _stats and not math.isnan(_stats[key]): + responsive_uids += [_uid] stats[zkey] = (1 - self.alpha) * stats[zkey] + self.alpha * _stats[key] else: stats[zkey] = (1 - self.alpha) * stats[zkey] # + self.alpha * 0 @@ -555,7 +556,6 @@ def neuron_stats_update(self, neuron_stats: Dict[int, Dict[str, Any]]): updates = 'updates_' + key if updates in stats: stats[updates] += 1 # increment number of normal EMA updates made - responsive_uids += [_uid] else: stats.setdefault(updates, 1) # add updates fields for new uid entries From 28b63a9b06430ec87001e687a13f2ef28191b099 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 13:31:47 +0200 Subject: [PATCH 03/12] Ensure each UID is queried once in validator nucleus Persist object variable self.permute_uids across forward calls, select num_endpoints uids each forward call, reset to new permutation of all UIDs once empty. Removes factors of variability between validators by ensuring each UID is queried the same number of times. --- .../_neuron/text/core_validator/__init__.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index d6c066110a..3c0e9d113d 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -660,6 +660,7 @@ def __init__( self, config, device, subtensor ): self.config = config self.device = device self.max_n = subtensor.max_n + self.permute_uids = [] # iterable of next UIDs to query, reset to permuted UIDs when empty tokenizer = bittensor.tokenizer() self.pad_token = tokenizer(tokenizer.pad_token)['input_ids'][0] @@ -794,18 +795,26 @@ def forward( # Ensure number of queried neurons does not exceed metagraph.n num_endpoints = min([self.config.nucleus.topk, metagraph.n]) - logger.info(f'Forward \t| Routing forward [{time.time() - start_time:.3g}s]') - logger.info(f'Dendrite \t| Request {num_endpoints} x {list(inputs_seq.shape)}') - request_start_time = time.time() + # === Ensure each UID is queried once === + # Persist object variable self.permute_uids across forward calls. + # Reset to new permutation of all UIDs once empty. + if len(self.permute_uids) == 0: # no more UIDs to query + self.permute_uids = torch.randperm(metagraph.n) # reset to new permutation of all UIDs # === Randomly select num_endpoints UIDs === - random_uids = torch.randperm(metagraph.n)[:num_endpoints] + random_uids = self.permute_uids[:num_endpoints] # newest selection of UIDs to query + self.permute_uids = self.permute_uids[num_endpoints:] # slice out remaining selection # === Get endpoint information for the selected UIDs === # We index into the metagraph's endpoints and return a list of the filtered set of endpoints we wish to query. # random_endpoints: List[bittensor.endpoints]: endpoint information for filtered uids. # len(neurons) == self.config.nucleus.topk random_endpoints = [metagraph.endpoints[uid] for uid in random_uids] + num_endpoints = len(random_endpoints) # in case len(self.permute_uids) < num_endpoints during random_uids select + + logger.info(f'Forward \t| Routing forward [{time.time() - start_time:.3g}s]') + logger.info(f'Dendrite \t| Request {num_endpoints} x {list(inputs_seq.shape)}') + request_start_time = time.time() # === Define which synapse we want to use === # The synapse defines the task we are sending to the neurons From ab8e18588f174bf49d2723eb0e726514e3cf49c2 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 14:11:28 +0200 Subject: [PATCH 04/12] Ensure each UID is queried at least once in validator neuron Assumes nucleus samples without replacement by permuting range(metagraph.n). Removes another factor of variability between validators, namely how many UIDs are sampled during each validator epoch, which is influenced by the validator speed. --- bittensor/_neuron/text/core_validator/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 3c0e9d113d..9bf37d3a32 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -372,7 +372,8 @@ def run_epoch( self ): epoch_queried_uids = set() start_block = self.subtensor.block - while self.subtensor.block < start_block + blocks_per_epoch: + while (self.subtensor.block < start_block + blocks_per_epoch or + len(epoch_queried_uids) < self.metagraph.n): # ensure each UID is queried at least once - assumes nucleus samples without replacement start_time = time.time() # === Forward === From 960426dcd59bb10b7360c4ba2573cd6d3c659568 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 14:13:28 +0200 Subject: [PATCH 05/12] Skip repeat metagraph_sync in validator run start --- bittensor/_neuron/text/core_validator/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 9bf37d3a32..a33e9f346f 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -365,7 +365,8 @@ def run_epoch( self ): # Each block length lasts blocks_per_epoch blocks. # This gives us a consistent network wide timer. # Here we run until blocks_per_epochs have progressed. - self.metagraph_sync() # Reset metagraph. + if self.epoch > 0: # skip first epoch: already synced at start of run + self.metagraph_sync() # Reset metagraph. epoch_steps = 0 epoch_responsive_uids = set() From ab0424e0a718340fc0d32d000348b8f56417ad62 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 14:43:32 +0200 Subject: [PATCH 06/12] Add set weights console message in validator --- bittensor/_neuron/text/core_validator/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index a33e9f346f..7dd2789e56 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -372,6 +372,7 @@ def run_epoch( self ): epoch_responsive_uids = set() epoch_queried_uids = set() + epoch_start_time = time.time() start_block = self.subtensor.block while (self.subtensor.block < start_block + blocks_per_epoch or len(epoch_queried_uids) < self.metagraph.n): # ensure each UID is queried at least once - assumes nucleus samples without replacement @@ -436,8 +437,9 @@ def run_epoch( self ): f'[dim] Epoch {self.epoch}[/dim] | ' f'[bright_green not bold]{len(responsive_uids)}[/bright_green not bold]/' f'[white]{len(queried_uids)}[/white] ' - f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' - f'[[yellow]{step_time:.3g}[/yellow]s]') + f'[[yellow]{step_time:.3g}[/yellow]s] ' + f'[dim white not bold][green]{len(epoch_responsive_uids)}[/green]/' + f'{len(epoch_queried_uids)}[/dim white not bold]') if self.config.logging.debug or self.config.logging.trace: # === Print stats update (table) === @@ -487,6 +489,15 @@ def run_epoch( self ): if self.config.logging.debug or self.config.logging.trace: self.weights_table(sample_uids, sample_weights) # print weights table + # set weights console message (every epoch) + print(f"[white not bold]{datetime.datetime.now():%Y-%m-%d %H:%M:%S}[/white not bold]{' ' * 4} | " + f"{f'[bright_white]Set weights[/bright_white]'.center(16 + len('[bright_white][/bright_white]'))} | " + f'[bright_green not bold]{len(sample_weights)}[/bright_green not bold] weights | ' + f'[bright_green not bold]{len(epoch_responsive_uids)}[/bright_green not bold]/' + f'[white]{len(epoch_queried_uids)}[/white] ' + f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' + f'[[yellow]{epoch_start_time - time.time():.3g}[/yellow]s]') + self.subtensor.set_weights( uids=sample_uids.detach().to('cpu'), weights=sample_weights.detach().to('cpu'), From 32c7df9f83ec7f08f1b67c917a3fdedf8494bdaf Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 15:10:51 +0200 Subject: [PATCH 07/12] Average synergy over responsives in validation query set --- bittensor/_neuron/text/core_validator/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 7dd2789e56..a2dac75153 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -1234,6 +1234,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens # Synergy = measured performance above expected performance # Measured in effective number of model parameters, just like base Shapley values. syn_loss_diff = {} # expected_loss - measured_loss (where > 0) + responsives = [uid for uid, stat in stats.items() if 'loss' + ext in stat] for _first, first in stats.items(): if 'loss' + ext not in first: continue @@ -1251,6 +1252,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens measured_loss = synergy(first, second, target, ext) loss_diff_share = torch.clamp(expected_loss - measured_loss, 0) / 2 # record direct loss diff + loss_diff_share /= len(responsives) # average over responsives first['synergy_loss_diff' + ext] += loss_diff_share second['synergy_loss_diff' + ext] += loss_diff_share @@ -1266,6 +1268,7 @@ def shapley_synergy(stats: Dict, synergy: Callable, ext: str, target: torch.Tens pow_expected_params = torch.pow(expected_params, scaling_law_power) synergy_share = torch.clamp(pow_measured_params - pow_expected_params, 0) / 2 + synergy_share /= len(responsives) # average over responsives first['synergy' + ext] += synergy_share # share synergy amongst coalition members second['synergy' + ext] += synergy_share From 55ae9dd98e44bd7a3ffbf1b3681b3f0aedcaecc3 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 15:21:52 +0200 Subject: [PATCH 08/12] Add synergy_scaling_law_power to validator nucleus Set synergy_scaling_law_power independent of scaling_law_power, since synergy likely needs a higher power after synergy averaging. --- .../_neuron/text/core_validator/__init__.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index a2dac75153..e0f73f9828 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -716,6 +716,7 @@ def add_args( cls, parser ): parser.add_argument('--nucleus.noise_multiplier', type=float, help='Standard deviation multipler on weights', default=2 ) parser.add_argument('--nucleus.dendrite_backward', action='store_true', help='Pass backward request to the server side or not', default=False ) parser.add_argument('--nucleus.scaling_law_power', type=float, help='Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) + parser.add_argument('--nucleus.synergy_scaling_law_power', type=float, help='Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) @classmethod def config ( cls ): @@ -869,8 +870,9 @@ def forward( # === Prepare validation parameter set === console_width = self.config.get('width', None) # console width for rich table displays of synapse measures validation_params = (random_uids, query_responses, return_ops, times, routing_score, - inputs, val_len, self.loss_fct, self.config.nucleus.scaling_law_power, console_width, - self.config.logging.debug or self.config.logging.trace) + inputs, val_len, self.loss_fct, + self.config.nucleus.scaling_law_power, self.config.nucleus.synergy_scaling_law_power, + console_width, self.config.logging.debug or self.config.logging.trace) loss = torch.tensor(0.).to(self.device) # to accumulate neuron_loss and routing_loss over synapses neuron_stats = {} # to gather neuron synapse validation measures and statistics @@ -898,7 +900,8 @@ def scaling_law_loss_to_params(loss): def textcausallm(uids: torch.Tensor, query_responses: List[List[torch.FloatTensor]], return_ops: List[torch.LongTensor], times: List[torch.FloatTensor], routing_score: torch.FloatTensor, - inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, scaling_law_power: float, + inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, + scaling_law_power: float, synergy_scaling_law_power: float, console_width: int, logging, synapse: 'bittensor.TextCausalLM' = None, index_s: int = 0 ) -> Tuple[torch.FloatTensor, Dict]: r""" @@ -923,6 +926,8 @@ def textcausallm(uids: torch.Tensor, query_responses: List[List[torch.FloatTenso CrossEntropy loss function to use. scaling_law_power (:obj:`float`, `required`): Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. + synergy_scaling_law_power (:obj:`float`, `required`): + Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. console_width (:obj:`int`, `required`): Config console width for table print. logging (:obj:`bool`, `required`): @@ -979,9 +984,9 @@ def _synergy(first, second, target, _ext): synergy_start_time = time.time() syn_loss_diff = shapley_synergy(stats, _synergy, ext='', target=inputs_seq[:, 1:], - scaling_law_power=scaling_law_power) + scaling_law_power=synergy_scaling_law_power) syn_loss_diff_val = shapley_synergy(stats, _synergy, ext='_val', target=inputs_val, - scaling_law_power=scaling_law_power) + scaling_law_power=synergy_scaling_law_power) # === Shapley value combination === # Combine base values with synergy approximation to get final Shapley values. @@ -1020,7 +1025,8 @@ def _synergy(first, second, target, _ext): def textcausallmnext(uids: torch.Tensor, query_responses: List[List[torch.FloatTensor]], return_ops: List[torch.LongTensor], times: List[torch.FloatTensor], routing_score: torch.FloatTensor, - inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, scaling_law_power: float, + inputs: torch.FloatTensor, validation_len: int, loss_fct: Callable, + scaling_law_power: float, synergy_scaling_law_power: float, console_width: int, logging, synapse: 'bittensor.TextCausalLMNext' = None, index_s: int = 0 ) -> Tuple[torch.FloatTensor, Dict]: r""" @@ -1045,6 +1051,8 @@ def textcausallmnext(uids: torch.Tensor, query_responses: List[List[torch.FloatT CrossEntropy loss function to use. scaling_law_power (:obj:`float`, `required`): Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. + synergy_scaling_law_power (:obj:`float`, `required`): + Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5. console_width (:obj:`int`, `required`): Config console width for table print. logging (:obj:`bool`, `required`): @@ -1097,7 +1105,7 @@ def _synergy(first, second, target, ext): synergy_start_time = time.time() - syn_loss_diff = shapley_synergy(stats, _synergy, '_nxt', scaling_law_power=scaling_law_power) + syn_loss_diff = shapley_synergy(stats, _synergy, '_nxt', scaling_law_power=synergy_scaling_law_power) # === Shapley value combination === # Combine base values with synergy approximation to get final Shapley values. From 746605c12cb1692a92d19b81adea717440a2eb4d Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 15:23:13 +0200 Subject: [PATCH 09/12] Increase synergy_scaling_law_power from 0.5 to 0.6 Synergy averaging now significantly reduces synergy Shapley contribution compared to the base Shapley value, so the power needs to be increased to compensate. --- bittensor/_neuron/text/core_validator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index e0f73f9828..90932f5a0f 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -716,7 +716,7 @@ def add_args( cls, parser ): parser.add_argument('--nucleus.noise_multiplier', type=float, help='Standard deviation multipler on weights', default=2 ) parser.add_argument('--nucleus.dendrite_backward', action='store_true', help='Pass backward request to the server side or not', default=False ) parser.add_argument('--nucleus.scaling_law_power', type=float, help='Power for modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) - parser.add_argument('--nucleus.synergy_scaling_law_power', type=float, help='Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.5) + parser.add_argument('--nucleus.synergy_scaling_law_power', type=float, help='Power for synergy modified scaling law, powered down to improve dynamic range, e.g. 3 → 6 nats for 0.5.', default=0.6) @classmethod def config ( cls ): From 0967462b8749dec07fd370d4bc8a7c8ff170fb0d Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 17:53:35 +0200 Subject: [PATCH 10/12] Update set weights console message --- bittensor/_neuron/text/core_validator/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 90932f5a0f..597eb93542 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -492,11 +492,16 @@ def run_epoch( self ): # set weights console message (every epoch) print(f"[white not bold]{datetime.datetime.now():%Y-%m-%d %H:%M:%S}[/white not bold]{' ' * 4} | " f"{f'[bright_white]Set weights[/bright_white]'.center(16 + len('[bright_white][/bright_white]'))} | " - f'[bright_green not bold]{len(sample_weights)}[/bright_green not bold] weights | ' + f'[bright_green not bold]{len(sample_weights)}[/bright_green not bold] [dim]weights set[/dim] | ' f'[bright_green not bold]{len(epoch_responsive_uids)}[/bright_green not bold]/' f'[white]{len(epoch_queried_uids)}[/white] ' f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' - f'[[yellow]{epoch_start_time - time.time():.3g}[/yellow]s]') + f'[[yellow]{epoch_start_time - time.time():.0f}[/yellow]s] | ' + f'weights sum:{sample_weights.sum().item():.2g} ' + f'[white] max:[bold]{sample_weights.max().item():.4g}[/bold] / ' + f'min:[bold]{sample_weights.min().item():.4g}[/bold] [/white] ' + f'\[{sample_weights.max().item() / sample_weights.min().item():.1f}:1] ' + f'({max_allowed_ratio} allowed)') self.subtensor.set_weights( uids=sample_uids.detach().to('cpu'), From 7f9d0a7e10d229407c0b027155fb78dbd30b5949 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 18:54:49 +0200 Subject: [PATCH 11/12] Update set weights console message --- bittensor/_neuron/text/core_validator/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 597eb93542..4451848a86 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -496,8 +496,8 @@ def run_epoch( self ): f'[bright_green not bold]{len(epoch_responsive_uids)}[/bright_green not bold]/' f'[white]{len(epoch_queried_uids)}[/white] ' f'[dim white not bold][green]responsive[/green]/queried[/dim white not bold] ' - f'[[yellow]{epoch_start_time - time.time():.0f}[/yellow]s] | ' - f'weights sum:{sample_weights.sum().item():.2g} ' + f'[[yellow]{time.time() - epoch_start_time:.0f}[/yellow]s] | ' + f'[dim]weights[/dim] sum:{sample_weights.sum().item():.2g} ' f'[white] max:[bold]{sample_weights.max().item():.4g}[/bold] / ' f'min:[bold]{sample_weights.min().item():.4g}[/bold] [/white] ' f'\[{sample_weights.max().item() / sample_weights.min().item():.1f}:1] ' From ddd0c51a647a939b74b5de81d80b0481a8ae2bb0 Mon Sep 17 00:00:00 2001 From: opentaco Date: Mon, 29 Aug 2022 19:34:30 +0200 Subject: [PATCH 12/12] Clear validator nucleus UID permutation before epoch --- bittensor/_neuron/text/core_validator/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 4451848a86..ab9b86acf3 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -368,11 +368,13 @@ def run_epoch( self ): if self.epoch > 0: # skip first epoch: already synced at start of run self.metagraph_sync() # Reset metagraph. + self.nucleus.permute_uids = [] # clear nucleus permutation before epoch + epoch_steps = 0 epoch_responsive_uids = set() epoch_queried_uids = set() - epoch_start_time = time.time() + start_block = self.subtensor.block while (self.subtensor.block < start_block + blocks_per_epoch or len(epoch_queried_uids) < self.metagraph.n): # ensure each UID is queried at least once - assumes nucleus samples without replacement