From 77c3f23bf1ba596f64bb190ad89cdab0465bad07 Mon Sep 17 00:00:00 2001 From: opentaco Date: Wed, 5 Apr 2023 13:50:05 +0200 Subject: [PATCH 1/7] Add math: max_upscale, max_limited --- pallets/subtensor/src/math.rs | 203 ++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/pallets/subtensor/src/math.rs b/pallets/subtensor/src/math.rs index c2fb364213..437f5a7653 100644 --- a/pallets/subtensor/src/math.rs +++ b/pallets/subtensor/src/math.rs @@ -36,6 +36,53 @@ pub fn vec_u16_proportions_to_fixed( vec: Vec ) -> Vec { vec.into_i #[allow(dead_code)] pub fn vec_fixed_proportions_to_u16( vec: Vec ) -> Vec { vec.into_iter().map(|e| fixed_proportion_to_u16(e) ).collect() } +#[allow(dead_code)] +// Max-upscale vector and convert to u16 so max_value = u16::MAX. Assumes non-negative normalized input. +pub fn vec_max_upscale_to_u16( vec: &Vec ) -> Vec { + let u16_max: I32F32 = I32F32::from_num( u16::MAX ); + let threshold: I32F32 = I32F32::from_num( 32768 ); + let max_value: Option<&I32F32> = vec.iter().max(); + match max_value { + Some(val) => { + if *val == I32F32::from_num( 0 ) { + return vec.iter().map(|e: &I32F32| (e * u16_max).to_num::() ).collect() + } + if *val > threshold { + return vec.iter().map(|e: &I32F32| (e * (u16_max / *val)).round().to_num::() ).collect() + } + return vec.iter().map(|e: &I32F32| ((e * u16_max) / *val).round().to_num::() ).collect() + }, + None => { + let sum: I32F32 = vec.iter().sum(); + return vec.iter().map(|e: &I32F32| ((e * u16_max) / sum).to_num::() ).collect() + } + } +} + +#[allow(dead_code)] +// Max-upscale u16 vector and convert to u16 so max_value = u16::MAX. Assumes u16 vector input. +pub fn vec_u16_max_upscale_to_u16( vec: &Vec ) -> Vec { + let vec_fixed: Vec = vec.iter().map(|e: &u16| I32F32::from_num( *e ) ).collect(); + vec_max_upscale_to_u16( &vec_fixed ) +} + +#[allow(dead_code)] +// Checks if u16 vector, when normalized, has a max value not greater than a u16 ratio max_limit. +pub fn check_vec_max_limited( vec: &Vec, max_limit: u16 ) -> bool { + let max_limit_fixed: I32F32 = I32F32::from_num( max_limit ) / I32F32::from_num( u16::MAX ); + let mut vec_fixed: Vec = vec.iter().map(|e: &u16| I32F32::from_num( *e ) ).collect(); + inplace_normalize( &mut vec_fixed ); + let max_value: Option<&I32F32> = vec_fixed.iter().max(); + match max_value { + Some(val) => { + return *val <= max_limit_fixed; + }, + None => { + return true; + } + } +} + #[allow(dead_code)] pub fn sum( x: &Vec ) -> I32F32 { x.iter().sum() } @@ -781,6 +828,13 @@ mod tests { assert_float_compare_64(va[i], vb[i], epsilon); } } + + fn assert_vec_compare_u16(va: &Vec, vb: &Vec) { + assert!(va.len() == vb.len()); + for i in 0..va.len(){ + assert_eq!(va[i], vb[i]); + } + } fn assert_mat_compare(ma: &Vec>, mb: &Vec>, epsilon: I32F32) { assert!(ma.len() == mb.len()); @@ -807,6 +861,155 @@ mod tests { vector.iter().map( | x | I32F32::from_num( *x ) ).collect() } + #[test] + fn test_vec_max_upscale_to_u16() { + let vector: Vec = vec_to_fixed( &vec![] ); + let target: Vec = vec![]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0. ] ); + let target: Vec = vec![ 0 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0., 0. ] ); + let target: Vec = vec![ 0, 0 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0., 1. ] ); + let target: Vec = vec![ 0, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0., 0.000000001 ] ); + let target: Vec = vec![ 0, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0., 0.000016, 1. ] ); + let target: Vec = vec![ 0, 1, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0.000000001, 0.000000001 ] ); + let target: Vec = vec![ 65535, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0.000001, 0.000006, 0.000007, 0.0001, 0.001, 0.01, 0.1, 0.2, 0.3, 0.4] ); + let target: Vec = vec![ 0, 1, 1, 16, 164, 1638, 16384, 32768, 49151, 65535]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ I32F32::from_num(16384) ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ I32F32::from_num(32768) ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ I32F32::from_num(32769) ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ I32F32::from_num(65535) ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ I32F32::max_value() ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0., 1., 65535. ] ); + let target: Vec = vec![ 0, 1, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0., 0.5, 1., 1.5, 2., 32768. ] ); + let target: Vec = vec![ 0, 1, 2, 3, 4, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec_to_fixed( &vec![ 0., 0.5, 1., 1.5, 2., 32768., 32769. ] ); + let target: Vec = vec![ 0, 1, 2, 3, 4, 65533, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ I32F32::from_num(0), I32F32::from_num(1), I32F32::from_num(32768), I32F32::from_num(32769), I32F32::max_value() ]; + let target: Vec = vec![ 0, 0, 1, 1, 65535 ]; + let result: Vec = vec_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + } + + #[test] + fn test_vec_u16_max_upscale_to_u16() { + let vector: Vec = vec![]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &vector); + let vector: Vec = vec![ 0 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &vector); + let vector: Vec = vec![ 0, 0 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &vector); + let vector: Vec = vec![ 1 ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ 0, 1 ]; + let target: Vec = vec![ 0, 65535 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ 65534 ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ 65535 ]; + let target: Vec = vec![ 65535 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ 65535, 65535 ]; + let target: Vec = vec![ 65535, 65535 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ 0, 1, 65534 ]; + let target: Vec = vec![ 0, 1, 65535 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &target); + let vector: Vec = vec![ 0, 1, 2, 3, 4, 65533, 65535 ]; + let result: Vec = vec_u16_max_upscale_to_u16( &vector ); + assert_vec_compare_u16(&result, &vector); + } + + #[test] + fn test_check_vec_max_limited() { + let vector: Vec = vec![]; + let max_limit: u16 = 0; + assert!( check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![]; + let max_limit: u16 = u16::MAX; + assert!( check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ u16::MAX ]; + let max_limit: u16 = u16::MAX; + assert!( check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ u16::MAX ]; + let max_limit: u16 = u16::MAX - 1; + assert!( !check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ u16::MAX ]; + let max_limit: u16 = 0; + assert!( !check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ 0 ]; + let max_limit: u16 = u16::MAX; + assert!( check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ 0, u16::MAX ]; + let max_limit: u16 = u16::MAX; + assert!( check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ 0, u16::MAX, u16::MAX ]; + let max_limit: u16 = u16::MAX / 2; + assert!( !check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ 0, u16::MAX, u16::MAX ]; + let max_limit: u16 = u16::MAX / 2 + 1; + assert!( check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ 0, u16::MAX, u16::MAX, u16::MAX ]; + let max_limit: u16 = u16::MAX / 3 - 1; + assert!( !check_vec_max_limited( &vector, max_limit ) ); + let vector: Vec = vec![ 0, u16::MAX, u16::MAX, u16::MAX ]; + let max_limit: u16 = u16::MAX / 3; + assert!( check_vec_max_limited( &vector, max_limit ) ); + } + #[test] fn test_math_fixed_overflow() { let max_32: I32F32 = I32F32::max_value(); From 15e43bf9e328ce92ff3d019c0c2f8fffca3ffcea Mon Sep 17 00:00:00 2001 From: opentaco Date: Wed, 5 Apr 2023 13:50:47 +0200 Subject: [PATCH 2/7] Replace weight normalization with max-upscale --- pallets/subtensor/src/weights.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pallets/subtensor/src/weights.rs b/pallets/subtensor/src/weights.rs index 143cf3a7d6..8df9f6ef45 100644 --- a/pallets/subtensor/src/weights.rs +++ b/pallets/subtensor/src/weights.rs @@ -1,4 +1,5 @@ use super::*; +use crate::math::*; use frame_support::sp_std::vec; use sp_std::vec::Vec; @@ -102,15 +103,15 @@ impl Pallet { // --- 12. Ensure that the weights have the required length. ensure!( Self::check_length( netuid, neuron_uid, &uids, &values ), Error::::NotSettingEnoughWeights ); - // --- 13. Normalize the weights. - let normalized_values = Self::normalize_weights( values ); + // --- 13. Max-upscale the weights. + let max_upscaled_weights: Vec = vec_u16_max_upscale_to_u16( &values ); // --- 14. Ensure the weights are max weight limited - ensure!( Self::max_weight_limited( netuid, neuron_uid, &uids, &normalized_values ), Error::::MaxWeightExceeded ); + ensure!( Self::max_weight_limited( netuid, neuron_uid, &uids, &max_upscaled_weights ), Error::::MaxWeightExceeded ); // --- 15. Zip weights for sinking to storage map. let mut zipped_weights: Vec<( u16, u16 )> = vec![]; - for ( uid, val ) in uids.iter().zip(normalized_values.iter()) { zipped_weights.push((*uid, *val)) } + for ( uid, val ) in uids.iter().zip(max_upscaled_weights.iter()) { zipped_weights.push((*uid, *val)) } // --- 16. Set weights under netuid, uid double map entry. Weights::::insert( netuid, neuron_uid, zipped_weights ); @@ -221,11 +222,7 @@ impl Pallet { if max_weight_limit == u16::MAX { return true; } // Check if the weights max value is less than or equal to the limit. - let max: u16 = *weights.iter().max().unwrap(); - if max <= max_weight_limit { return true; } - - // The check has failed. - return false; + check_vec_max_limited( weights, max_weight_limit) } // Returns true if the uids and weights correspond to a self weight on the uid. From 74550689c1e9df1b46a0e7b8a8c0da601413b74e Mon Sep 17 00:00:00 2001 From: opentaco Date: Wed, 5 Apr 2023 13:51:07 +0200 Subject: [PATCH 3/7] Update weights tests --- pallets/subtensor/tests/weights.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pallets/subtensor/tests/weights.rs b/pallets/subtensor/tests/weights.rs index a0334cd10e..03c3cd3859 100644 --- a/pallets/subtensor/tests/weights.rs +++ b/pallets/subtensor/tests/weights.rs @@ -383,12 +383,8 @@ fn test_set_weights_sum_larger_than_u16_max() { assert_ok!(result); let all_weights: Vec> = SubtensorModule::get_weights(netuid); - let weights_set: Vec = all_weights[neuron_uid as usize].iter().map(|x| x.to_bits() as u16).collect(); - - // Should sum less than u16 max. - assert!(weights_set.iter().map(|x| *x as u64).sum::() <= (u16::MAX as u64) ); - - // Should be normalized to 50% each. - assert_eq!(weights_set, vec![u16::MAX/2, u16::MAX/2]); + let weights_set: &Vec = &all_weights[neuron_uid as usize]; + assert_eq!( weights_set[0], I32F32::from_num(1) ); + assert_eq!( weights_set[1], I32F32::from_num(1) ); }); } \ No newline at end of file From 34a0b18b5cc56a77c1c2c40bb48455ada689f7fe Mon Sep 17 00:00:00 2001 From: opentaco Date: Wed, 5 Apr 2023 13:51:31 +0200 Subject: [PATCH 4/7] Fix epoch tests --- pallets/subtensor/tests/epoch.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pallets/subtensor/tests/epoch.rs b/pallets/subtensor/tests/epoch.rs index 1af2dc352f..3bce819002 100644 --- a/pallets/subtensor/tests/epoch.rs +++ b/pallets/subtensor/tests/epoch.rs @@ -110,7 +110,7 @@ fn init_run_epochs(netuid: u16, n: u16, validators: &Vec, servers: &Vec Date: Wed, 5 Apr 2023 16:49:06 +0200 Subject: [PATCH 5/7] [BIT-638] Max-upscale pruning_scores (#95) * Add vec_max_upscale_to_u16 * Do max-upscale pruning_scores * Add epoch test log details for u16 pruning_scores * Norm when no max * Added max-upscaling unnormalized handling * Update spec_version to 116 --- pallets/subtensor/src/epoch.rs | 4 ++-- runtime/src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pallets/subtensor/src/epoch.rs b/pallets/subtensor/src/epoch.rs index 8b405baf16..6acb9e23f9 100644 --- a/pallets/subtensor/src/epoch.rs +++ b/pallets/subtensor/src/epoch.rs @@ -215,7 +215,7 @@ impl Pallet { let cloned_consensus: Vec = consensus.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); let cloned_incentive: Vec = incentive.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); let cloned_dividends: Vec = dividends.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); - let cloned_pruning_scores: Vec = pruning_scores.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); + let cloned_pruning_scores: Vec = vec_max_upscale_to_u16(&pruning_scores); let cloned_validator_trust: Vec = validator_trust.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); Active::::insert( netuid, active.clone() ); Emission::::insert( netuid, cloned_emission ); @@ -485,7 +485,7 @@ impl Pallet { let cloned_consensus: Vec = consensus.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); let cloned_incentive: Vec = incentive.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); let cloned_dividends: Vec = dividends.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); - let cloned_pruning_scores: Vec = pruning_scores.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); + let cloned_pruning_scores: Vec = vec_max_upscale_to_u16(&pruning_scores); let cloned_validator_trust: Vec = validator_trust.iter().map(|xi| fixed_proportion_to_u16(*xi)).collect::>(); Active::::insert( netuid, active.clone() ); Emission::::insert( netuid, cloned_emission ); diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 6bc8f22eb7..c752be7504 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -111,7 +111,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // `spec_version`, and `authoring_version` are the same between Wasm and native. // This value is set to 100 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use // the compatible custom types. - spec_version: 115, + spec_version: 116, impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1, From a2cc9836f4a774f1d5d928ef06591549e9992e2a Mon Sep 17 00:00:00 2001 From: opentaco Date: Wed, 5 Apr 2023 17:26:43 +0200 Subject: [PATCH 6/7] Remove redefine --- pallets/subtensor/src/math.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pallets/subtensor/src/math.rs b/pallets/subtensor/src/math.rs index b98ed91e4a..437f5a7653 100644 --- a/pallets/subtensor/src/math.rs +++ b/pallets/subtensor/src/math.rs @@ -836,13 +836,6 @@ mod tests { } } - fn assert_vec_compare_u16(va: &Vec, vb: &Vec) { - assert!(va.len() == vb.len()); - for i in 0..va.len(){ - assert_eq!(va[i], vb[i]); - } - } - fn assert_mat_compare(ma: &Vec>, mb: &Vec>, epsilon: I32F32) { assert!(ma.len() == mb.len()); for row in 0..ma.len() { From 8fb254d8ce33d4e7600d2a9dbf50f95682614b15 Mon Sep 17 00:00:00 2001 From: opentaco Date: Wed, 5 Apr 2023 17:56:48 +0200 Subject: [PATCH 7/7] Upgrade spec_version to 117 --- runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index c752be7504..6bc502d057 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -111,7 +111,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // `spec_version`, and `authoring_version` are the same between Wasm and native. // This value is set to 100 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use // the compatible custom types. - spec_version: 116, + spec_version: 117, impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1,