From 24a3cca3ca1c2802138fcb4f7be8a7f6d937758e Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 31 Mar 2025 14:18:51 -0500 Subject: [PATCH 1/3] add initial impl --- Cargo.lock | 2 +- Cargo.toml | 2 +- pyproject.toml | 2 +- src/lib.rs | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9653c95..b98bd50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,7 +22,7 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "bt_decode" -version = "0.5.0" +version = "0.6.0" dependencies = [ "custom_derive", "frame-metadata 16.0.0", diff --git a/Cargo.toml b/Cargo.toml index 3c6c61e..86c4e5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bt_decode" -version = "0.5.0" +version = "0.6.0" edition = "2021" [lib] diff --git a/pyproject.toml b/pyproject.toml index 866806f..96f777b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bt-decode" -version = "0.5.0" +version = "0.6.0" description = "A wrapper around the scale-codec crate for fast scale-decoding of Bittensor data structures." readme = "README.md" license = {file = "LICENSE"} diff --git a/src/lib.rs b/src/lib.rs index 4b6738d..54d166b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1070,6 +1070,45 @@ mod bt_decode { value_to_pyobject(py, decoded) } + #[pyfunction(name = "decode_list")] + fn py_decode_list( + py: Python, + type_strings: Vec, + portable_registry: &PyPortableRegistry, + encoded: Vec>, + ) -> PyResult>> { + // Create a memoization table for the type string to type id conversion + let mut memo = HashMap::::new(); + + let mut curr_registry = portable_registry.registry.clone(); + + fill_memo_using_well_known_types(&mut memo, &curr_registry); + + let mut decoded_list = Vec::>::new(); + + for (type_string, encoded) in type_strings.iter().zip(encoded.iter()) { + let type_id: u32 = + get_type_id_from_type_string(&mut memo, type_string, &mut curr_registry).ok_or( + PyErr::new::(format!( + "Failed to get type id from type string: {:?}", + type_string + )), + )?; + + let decoded = + decode_as_type(&mut &encoded[..], type_id, &curr_registry).map_err(|_e| { + PyErr::new::(format!( + "Failed to decode type: {:?} with type id: {:?}", + type_string, type_id + )) + })?; + + decoded_list.push(value_to_pyobject(py, decoded)?); + } + + Ok(decoded_list) + } + #[pyfunction(name = "encode")] fn py_encode( py: Python, From 59b2279266724f989a63b58a5839924851f58dcd Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 31 Mar 2025 14:23:12 -0500 Subject: [PATCH 2/3] fmt and clpy --- src/lib.rs | 226 ++++++++++++++++++++++++++--------------------------- 1 file changed, 110 insertions(+), 116 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 54d166b..43925bd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ mod dyndecoder; #[pymodule(name = "bt_decode")] mod bt_decode { - use std::{collections::HashMap, u128}; + use std::collections::HashMap; use dyndecoder::{fill_memo_using_well_known_types, get_type_id_from_type_string}; use frame_metadata::v15::RuntimeMetadataV15; @@ -460,7 +460,7 @@ mod bt_decode { locals.set_item("value", value)?; py.run_bound( - &*format!("ret = isinstance(value, {})", type_name), + &format!("ret = isinstance(value, {})", type_name), None, Some(&locals), ) @@ -520,33 +520,33 @@ mod bt_decode { scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::U128) => { let value = Value::with_context(ValueDef::Primitive(Primitive::U128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::U64) => { let value = Value::with_context(ValueDef::Primitive(Primitive::U128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::U32) => { let value = Value::with_context(ValueDef::Primitive(Primitive::U128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::U16) => { let value = Value::with_context(ValueDef::Primitive(Primitive::U128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::U8) => { let value = Value::with_context(ValueDef::Primitive(Primitive::U128(value)), type_id); - return Ok(value); + Ok(value) } _ => { - return Err(PyErr::new::(format!( + Err(PyErr::new::(format!( "Invalid type for u128 data: {}", value - ))); + ))) } } } else { @@ -555,33 +555,33 @@ mod bt_decode { scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::I128) => { let value = Value::with_context(ValueDef::Primitive(Primitive::I128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::I64) => { let value = Value::with_context(ValueDef::Primitive(Primitive::I128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::I32) => { let value = Value::with_context(ValueDef::Primitive(Primitive::I128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::I16) => { let value = Value::with_context(ValueDef::Primitive(Primitive::I128(value)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::I8) => { let value = Value::with_context(ValueDef::Primitive(Primitive::I128(value)), type_id); - return Ok(value); + Ok(value) } _ => { - return Err(PyErr::new::(format!( + Err(PyErr::new::(format!( "Invalid type for i128 data: {}", value - ))); + ))) } } } @@ -603,7 +603,7 @@ mod bt_decode { let ty_ = portable_registry .registry .resolve(ty_param_id) - .expect(&format!("Failed to resolve type (1): {:?}", ty_param)); + .unwrap_or_else(|| panic!("Failed to resolve type (1): {:?}", ty_param)); log::debug!(target: "btdecode", "ty_param: {:?}", ty_param); log::debug!(target: "btdecode", "ty_param_id: {:?}", ty_param_id); log::debug!(target: "btdecode", "ty_: {:?}", ty_); @@ -614,7 +614,7 @@ mod bt_decode { pyobject_to_value( py, item.as_any().as_unbound(), - &ty_, + ty_, ty_param_id, portable_registry, ) @@ -623,7 +623,7 @@ mod bt_decode { let value = Value::with_context(ValueDef::Composite(Composite::Unnamed(items)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Tuple(_inner) => { dbg!(_inner, py_list); @@ -635,11 +635,11 @@ mod bt_decode { let ty_ = portable_registry .registry .resolve(ty_id) - .expect(&format!("Failed to resolve type (1): {:?}", ty_)); + .unwrap_or_else(|| panic!("Failed to resolve type (1): {:?}", ty_)); pyobject_to_value( py, item.as_any().as_unbound(), - &ty_, + ty_, ty_id, portable_registry, ) @@ -648,7 +648,7 @@ mod bt_decode { let value = Value::with_context(ValueDef::Composite(Composite::Unnamed(items)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Sequence(inner) => { let ty_param = inner.type_param; @@ -656,7 +656,7 @@ mod bt_decode { let ty_ = portable_registry .registry .resolve(ty_param_id) - .expect(&format!("Failed to resolve type (1): {:?}", ty_param)); + .unwrap_or_else(|| panic!("Failed to resolve type (1): {:?}", ty_param)); let items = py_list .iter() @@ -664,7 +664,7 @@ mod bt_decode { pyobject_to_value( py, item.as_any().as_unbound(), - &ty_, + ty_, ty_param_id, portable_registry, ) @@ -673,10 +673,10 @@ mod bt_decode { let value = Value::with_context(ValueDef::Composite(Composite::Unnamed(items)), type_id); - return Ok(value); + Ok(value) } scale_info::TypeDef::Composite(TypeDefComposite { fields }) => { - if fields.len() == 0 { + if fields.is_empty() { return Err(PyErr::new::(format!( "Unexpected 0 fields for unnamed composite type: {:?}", ty @@ -690,7 +690,9 @@ mod bt_decode { let ty_ = portable_registry .registry .resolve(field.ty.id) - .expect(&format!("Failed to resolve type for field: {:?}", field)); + .unwrap_or_else(|| { + panic!("Failed to resolve type for field: {:?}", field) + }); pyobject_to_value( py, @@ -705,14 +707,12 @@ mod bt_decode { let value = Value::with_context(ValueDef::Composite(Composite::Unnamed(vals)), type_id); - return Ok(value); - } - _ => { - return Err(PyErr::new::(format!( - "Invalid type for a list of data: {}", - py_list - ))); + Ok(value) } + _ => Err(PyErr::new::(format!( + "Invalid type for a list of data: {}", + py_list + ))), } } @@ -742,14 +742,12 @@ mod bt_decode { scale_info::TypeDef::Primitive(scale_info::TypeDefPrimitive::Bool) => { let value = Value::with_context(ValueDef::Primitive(Primitive::Bool(value)), type_id); - return Ok(value); - } - _ => { - return Err(PyErr::new::(format!( - "Invalid type for bool data: {}", - value - ))); + Ok(value) } + _ => Err(PyErr::new::(format!( + "Invalid type for bool data: {}", + value + ))), } } else if py_isinstance(py, to_encode, "str")? { log::debug!(target: "btdecode", "encoding to str"); @@ -800,18 +798,15 @@ mod bt_decode { // Must be a Compact int let as_py_int = to_encode.downcast_bound::(py)?.as_unbound(); - match &ty.type_def { - scale_info::TypeDef::Compact(inner) => { - let inner_type_id = inner.type_param.id; - let inner_type_ = portable_registry.registry.resolve(inner_type_id); - if let Some(inner_type) = inner_type_ { - let mut inner_value = - int_type_def_to_value(py, as_py_int, inner_type, inner_type_id)?; - inner_value.context = type_id; - return Ok(inner_value); - } + if let scale_info::TypeDef::Compact(inner) = &ty.type_def { + let inner_type_id = inner.type_param.id; + let inner_type_ = portable_registry.registry.resolve(inner_type_id); + if let Some(inner_type) = inner_type_ { + let mut inner_value = + int_type_def_to_value(py, as_py_int, inner_type, inner_type_id)?; + inner_value.context = type_id; + return Ok(inner_value); } - _ => {} } return Err(PyErr::new::(format!( @@ -833,7 +828,7 @@ mod bt_decode { log::debug!(target: "btdecode", "encoding as list"); let as_list = to_encode.downcast_bound::(py)?; - pylist_to_value(py, &as_list, ty, type_id, portable_registry).map_err(|_e| { + pylist_to_value(py, as_list, ty, type_id, portable_registry).map_err(|_e| { PyErr::new::(format!( "Invalid type for list data: {}", as_list @@ -870,14 +865,15 @@ mod bt_decode { ) ))?; - let inner_type = - portable_registry - .registry - .resolve(field.ty.id) - .expect(&format!( + let inner_type = portable_registry + .registry + .resolve(field.ty.id) + .unwrap_or_else(|| { + panic!( "Inner type: {:?} was not in registry after being registered", field.ty - )); + ) + }); let as_value = pyobject_to_value( py, @@ -934,14 +930,15 @@ mod bt_decode { ) ))?; - let inner_type = - portable_registry - .registry - .resolve(field.ty.id) - .expect(&format!( + let inner_type = portable_registry + .registry + .resolve(field.ty.id) + .unwrap_or_else(|| { + panic!( "Inner type: {:?} was not in registry after being registered", field.ty - )); + ) + }); let as_value = pyobject_to_value( py, @@ -983,61 +980,58 @@ mod bt_decode { portable_registry: &PyPortableRegistry, ) -> PyResult> { // Check if the expected type is an option - match ty.type_def.clone() { - scale_info::TypeDef::Variant(inner) => { - let is_option: bool = if inner.variants.len() == 2 { - let variant_names = inner - .variants - .iter() - .map(|v| &*v.name) - .collect::>(); - variant_names.contains(&"Some") && variant_names.contains(&"None") + if let scale_info::TypeDef::Variant(inner) = &ty.type_def { + let is_option: bool = if inner.variants.len() == 2 { + let variant_names = inner + .variants + .iter() + .map(|v| &*v.name) + .collect::>(); + variant_names.contains(&"Some") && variant_names.contains(&"None") + } else { + false + }; + + if is_option { + if to_encode.is_none(py) { + // None + let none_variant: scale_value::Variant = + Variant::unnamed_fields("None", vec![]); // No fields because it's None + + return Ok(Value::with_context( + ValueDef::Variant(none_variant), + type_id, + )); } else { - false - }; - - if is_option { - if to_encode.is_none(py) { - // None - let none_variant: scale_value::Variant = - Variant::unnamed_fields("None", vec![]); // No fields because it's None - - return Ok(Value::with_context( - ValueDef::Variant(none_variant), - type_id, - )); - } else { - // Some - // Get inner type - let inner_type_id: u32 = inner.variants[1].fields[0].ty.id; - let inner_type: &scale_info::Type = portable_registry - .registry - .resolve(inner_type_id) - .ok_or(PyErr::new::(format!( - "Could not find inner_type: {:?} for Option: {:?}", - ty.type_def, inner_type_id - )))?; - let inner_value: Value = pyobject_to_value_no_option_check( - py, - to_encode, - inner_type, - inner_type_id, - portable_registry, - )?; - let some_variant: scale_value::Variant = - Variant::unnamed_fields("Some", vec![inner_value]); // No fields because it's None + // Some + // Get inner type + let inner_type_id: u32 = inner.variants[1].fields[0].ty.id; + let inner_type: &scale_info::Type = portable_registry + .registry + .resolve(inner_type_id) + .ok_or(PyErr::new::(format!( + "Could not find inner_type: {:?} for Option: {:?}", + ty.type_def, inner_type_id + )))?; + let inner_value: Value = pyobject_to_value_no_option_check( + py, + to_encode, + inner_type, + inner_type_id, + portable_registry, + )?; + let some_variant: scale_value::Variant = + Variant::unnamed_fields("Some", vec![inner_value]); // No fields because it's None - return Ok(Value::with_context( - ValueDef::Variant(some_variant), - type_id, - )); - } - } // else: Regular conversion - } - _ => {} // Regular conversion + return Ok(Value::with_context( + ValueDef::Variant(some_variant), + type_id, + )); + } + } // else: Regular conversion } - return pyobject_to_value_no_option_check(py, to_encode, ty, type_id, portable_registry); + pyobject_to_value_no_option_check(py, to_encode, ty, type_id, portable_registry) } #[pyfunction(name = "decode")] From 76e13dab205b3b08eb03fa8c8d8e125b92d20692 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 31 Mar 2025 15:57:30 -0500 Subject: [PATCH 3/3] rename args, fmt, and add to stub --- bt_decode.pyi | 16 ++++++++++++++++ src/lib.rs | 26 +++++++++++--------------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/bt_decode.pyi b/bt_decode.pyi index ebf41e4..a0de935 100644 --- a/bt_decode.pyi +++ b/bt_decode.pyi @@ -350,6 +350,22 @@ def decode( ) -> Any: pass +def decode_list( + list_type_strings: list[str], + portable_registry: PortableRegistry, + list_encoded: list[bytes], +) -> list[Any]: + """ + Decode a list of SCALE-encoded types using a list of their type-strings. + + Note: the type-strings are potentially all different. + Note: the order of `list_type_strings` and `list_encoded` must match. + + Returns a list of the decoded values as python objects, in the order they were + provided to the function. + """ + pass + def encode( type_string: str, portable_registry: PortableRegistry, to_encode: Any ) -> list[int]: diff --git a/src/lib.rs b/src/lib.rs index 43925bd..71fba14 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -542,12 +542,10 @@ mod bt_decode { Value::with_context(ValueDef::Primitive(Primitive::U128(value)), type_id); Ok(value) } - _ => { - Err(PyErr::new::(format!( - "Invalid type for u128 data: {}", - value - ))) - } + _ => Err(PyErr::new::(format!( + "Invalid type for u128 data: {}", + value + ))), } } else { let value = py_int.extract::(py)?; @@ -577,12 +575,10 @@ mod bt_decode { Value::with_context(ValueDef::Primitive(Primitive::I128(value)), type_id); Ok(value) } - _ => { - Err(PyErr::new::(format!( - "Invalid type for i128 data: {}", - value - ))) - } + _ => Err(PyErr::new::(format!( + "Invalid type for i128 data: {}", + value + ))), } } } @@ -1067,9 +1063,9 @@ mod bt_decode { #[pyfunction(name = "decode_list")] fn py_decode_list( py: Python, - type_strings: Vec, + list_type_strings: Vec, portable_registry: &PyPortableRegistry, - encoded: Vec>, + list_encoded: Vec>, ) -> PyResult>> { // Create a memoization table for the type string to type id conversion let mut memo = HashMap::::new(); @@ -1080,7 +1076,7 @@ mod bt_decode { let mut decoded_list = Vec::>::new(); - for (type_string, encoded) in type_strings.iter().zip(encoded.iter()) { + for (type_string, encoded) in list_type_strings.iter().zip(list_encoded.iter()) { let type_id: u32 = get_type_id_from_type_string(&mut memo, type_string, &mut curr_registry).ok_or( PyErr::new::(format!(