From 9ec8095eb13a7b2b44fc02c66b94046e8c6bf019 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 28 Jan 2025 22:44:48 +0200 Subject: [PATCH 01/14] Update project name for PyPI --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ef6e883..d70afd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "async-substrate-interface" +name = "async_substrate_interface" version = "1.0.0rc4" description = "Asyncio library for interacting with substrate. Mostly API-compatible with py-substrate-interface" readme = "README.md" From 3e245c4bd270d1346551c88a653201ca86818573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Tue, 28 Jan 2025 21:38:14 +0100 Subject: [PATCH 02/14] feat: use bt_decode in runtime_call --- async_substrate_interface/async_substrate.py | 143 +++++++++++------- async_substrate_interface/sync_substrate.py | 139 ++++++++++------- .../asyncio/test_substrate_interface.py | 51 +++++++ tests/unit_tests/sync/__init__.py | 0 .../sync/test_substrate_interface.py | 55 +++++++ 5 files changed, 282 insertions(+), 106 deletions(-) create mode 100644 tests/unit_tests/sync/__init__.py create mode 100644 tests/unit_tests/sync/test_substrate_interface.py diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index b1f33a6..413bd9a 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -23,9 +23,21 @@ import asyncstdlib as a from bittensor_wallet.keypair import Keypair -from bt_decode import PortableRegistry, decode as decode_by_type_string, MetadataV15 +from bittensor_wallet.utils import SS58_FORMAT +from bt_decode import ( + MetadataV15, + PortableRegistry, + decode as decode_by_type_string, + encode as encode_by_type_string, +) from scalecodec.base import ScaleBytes, ScaleType, RuntimeConfigurationObject -from scalecodec.types import GenericCall, GenericRuntimeCallDefinition, GenericExtrinsic +from scalecodec.types import ( + Bytes, + GenericCall, + GenericExtrinsic, + GenericRuntimeCallDefinition, + ss58_decode, +) from websockets.asyncio.client import connect from websockets.exceptions import ConnectionClosed @@ -788,8 +800,10 @@ async def load_registry(self): ) metadata_option_hex_str = metadata_rpc_result["result"] metadata_option_bytes = bytes.fromhex(metadata_option_hex_str[2:]) - metadata_v15 = MetadataV15.decode_from_metadata_option(metadata_option_bytes) - self.registry = PortableRegistry.from_metadata_v15(metadata_v15) + self.metadata_v15 = MetadataV15.decode_from_metadata_option( + metadata_option_bytes + ) + self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15) async def decode_scale( self, @@ -822,6 +836,9 @@ async def _wait_for_registry(): if scale_bytes == b"\x00": obj = None + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) else: try: if not self.registry: @@ -850,25 +867,55 @@ async def _wait_for_registry(): else: return obj - async def encode_scale(self, type_string, value, block_hash=None) -> ScaleBytes: + async def encode_scale(self, type_string, value: Any) -> bytes: """ Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string Args: type_string: the type string of the SCALE object for decoding value: value to encode - block_hash: the hash of the blockchain block whose metadata to use for encoding Returns: - ScaleBytes encoded value + encoded SCALE bytes """ - if not self._metadata or block_hash: - await self.init_runtime(block_hash=block_hash) + if value is None: + result = b"\x00" + else: + if type_string == "scale_info::0": # Is an AccountId + # encode string into AccountId + ## AccountId is a composite type with one, unnamed field + return bytes.fromhex(ss58_decode(value, SS58_FORMAT)) + + elif type_string == "scale_info::151": # Vec + if not isinstance(value, (list, tuple)): + value = [value] + + # Encode length + length = len(value) + if length < 64: + result = bytes([length << 2]) # Single byte mode + else: + raise ValueError("Vector length too large") - obj = self.runtime_config.create_scale_object( - type_string=type_string, metadata=self._metadata - ) - return obj.encode(value) + # Encode each AccountId + for account in value: + if isinstance(account, bytes): + result += account # Already encoded + else: + result += bytes.fromhex( + ss58_decode(value, SS58_FORMAT) + ) # SS58 string + return result + + if isinstance(value, ScaleType): + if value.data.data is not None: + # Already encoded + return bytes(value.data.data) + else: + value = value.value # Unwrap the value of the type + + result = bytes(encode_by_type_string(type_string, self.registry, value)) + return result async def _first_initialize_runtime(self): """ @@ -2164,16 +2211,17 @@ async def query_multi( storage_key = storage_key_map[change_storage_key] if change_data is None: change_data = b"\x00" + obj = None else: change_data = bytes.fromhex(change_data[2:]) - result.append( - ( - storage_key, - await self.decode_scale( + if change_data == b"\x00": + obj = None + else: + obj = await self.decode_scale( storage_key.value_scale_type, change_data - ), - ) - ) + ) + + result.append((storage_key, obj)) return result @@ -2502,56 +2550,44 @@ async def runtime_call( params = {} try: - runtime_call_def = self.runtime_config.type_registry["runtime_api"][api][ - "methods" - ][method] - runtime_api_types = self.runtime_config.type_registry["runtime_api"][ - api - ].get("types", {}) + metadata_v15 = self.metadata_v15.value() + apis = {entry["name"]: entry for entry in metadata_v15["apis"]} + api_entry = apis[api] + methods = {entry["name"]: entry for entry in api_entry["methods"]} + runtime_call_def = methods[method] except KeyError: raise ValueError(f"Runtime API Call '{api}.{method}' not found in registry") - if isinstance(params, list) and len(params) != len(runtime_call_def["params"]): + if isinstance(params, list) and len(params) != len(runtime_call_def["inputs"]): raise ValueError( f"Number of parameter provided ({len(params)}) does not " - f"match definition {len(runtime_call_def['params'])}" + f"match definition {len(runtime_call_def['inputs'])}" ) - # Add runtime API types to registry - self.runtime_config.update_type_registry_types(runtime_api_types) - runtime = Runtime( - self.chain, - self.runtime_config, - self._metadata, - self.type_registry, - ) - # Encode params - param_data = ScaleBytes(bytes()) - for idx, param in enumerate(runtime_call_def["params"]): - scale_obj = runtime.runtime_config.create_scale_object(param["type"]) + param_data = b"" + for idx, param in enumerate(runtime_call_def["inputs"]): + param_type_string = f"scale_info::{param['ty']}" if isinstance(params, list): - param_data += scale_obj.encode(params[idx]) + param_data += await self.encode_scale(param_type_string, params[idx]) else: if param["name"] not in params: raise ValueError(f"Runtime Call param '{param['name']}' is missing") - param_data += scale_obj.encode(params[param["name"]]) + param_data += await self.encode_scale( + param_type_string, params[param["name"]] + ) # RPC request result_data = await self.rpc_request( - "state_call", [f"{api}_{method}", str(param_data), block_hash] + "state_call", [f"{api}_{method}", param_data.hex(), block_hash] ) + output_type_string = f"scale_info::{runtime_call_def['output']}" # Decode result - # TODO update this to use bt-decode - result_obj = runtime.runtime_config.create_scale_object( - runtime_call_def["type"] - ) - result_obj.decode( - ScaleBytes(result_data["result"]), - check_remaining=self.config.get("strict_scale_decode"), - ) + result_bytes = hex_to_bytes(result_data["result"]) + result_obj = Bytes(ScaleBytes(result_bytes)) + result_obj.value = await self.decode_scale(output_type_string, result_bytes) return result_obj @@ -2678,14 +2714,13 @@ async def get_payment_info( extrinsic = await self.create_signed_extrinsic( call=call, keypair=keypair, signature=signature ) - extrinsic_len = self.runtime_config.create_scale_object("u32") - extrinsic_len.encode(len(extrinsic.data)) + extrinsic_len = len(extrinsic.data) result = await self.runtime_call( "TransactionPaymentApi", "query_info", [extrinsic, extrinsic_len] ) - return result.value + return result.result async def get_type_registry( self, block_hash: str = None, max_recursion: int = 4 diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index c344e48..36b8305 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -5,8 +5,20 @@ from typing import Optional, Union, Callable, Any from bittensor_wallet.keypair import Keypair -from bt_decode import PortableRegistry, decode as decode_by_type_string, MetadataV15 -from scalecodec import GenericExtrinsic, GenericCall, GenericRuntimeCallDefinition +from bittensor_wallet.utils import SS58_FORMAT +from bt_decode import ( + MetadataV15, + PortableRegistry, + decode as decode_by_type_string, + encode as encode_by_type_string, +) +from scalecodec import ( + Bytes, + GenericCall, + GenericExtrinsic, + GenericRuntimeCallDefinition, + ss58_decode, +) from scalecodec.base import RuntimeConfigurationObject, ScaleBytes, ScaleType from websockets.sync.client import connect @@ -582,8 +594,10 @@ def load_registry(self): ) metadata_option_hex_str = metadata_rpc_result["result"] metadata_option_bytes = bytes.fromhex(metadata_option_hex_str[2:]) - metadata_v15 = MetadataV15.decode_from_metadata_option(metadata_option_bytes) - self.registry = PortableRegistry.from_metadata_v15(metadata_v15) + self.metadata_v15 = MetadataV15.decode_from_metadata_option( + metadata_option_bytes + ) + self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15) def decode_scale( self, @@ -607,6 +621,9 @@ def decode_scale( if scale_bytes == b"\x00": obj = None + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) else: obj = decode_by_type_string(type_string, self.registry, scale_bytes) if return_scale_obj: @@ -614,25 +631,55 @@ def decode_scale( else: return obj - def encode_scale(self, type_string, value, block_hash=None) -> ScaleBytes: + def encode_scale(self, type_string, value: Any) -> bytes: """ Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string Args: type_string: the type string of the SCALE object for decoding value: value to encode - block_hash: the hash of the blockchain block whose metadata to use for encoding Returns: - ScaleBytes encoded value + encoded SCALE bytes """ - if not self._metadata or block_hash: - self.init_runtime(block_hash=block_hash) + if value is None: + result = b"\x00" + else: + if type_string == "scale_info::0": # Is an AccountId + # encode string into AccountId + ## AccountId is a composite type with one, unnamed field + return bytes.fromhex(ss58_decode(value, SS58_FORMAT)) + + elif type_string == "scale_info::151": # Vec + if not isinstance(value, (list, tuple)): + value = [value] + + # Encode length + length = len(value) + if length < 64: + result = bytes([length << 2]) # Single byte mode + else: + raise ValueError("Vector length too large") - obj = self.runtime_config.create_scale_object( - type_string=type_string, metadata=self._metadata - ) - return obj.encode(value) + # Encode each AccountId + for account in value: + if isinstance(account, bytes): + result += account # Already encoded + else: + result += bytes.fromhex( + ss58_decode(value, SS58_FORMAT) + ) # SS58 string + return result + + if isinstance(value, ScaleType): + if value.data.data is not None: + # Already encoded + return bytes(value.data.data) + else: + value = value.value # Unwrap the value of the type + + result = bytes(encode_by_type_string(type_string, self.registry, value)) + return result def _first_initialize_runtime(self): """ @@ -1906,12 +1953,13 @@ def query_multi( change_data = b"\x00" else: change_data = bytes.fromhex(change_data[2:]) - result.append( - ( - storage_key, - self.decode_scale(storage_key.value_scale_type, change_data), - ) - ) + if change_data == b"\x00": + obj = None + else: + obj = self.decode_scale( + storage_key.value_scale_type, change_data + ) + result.append((storage_key, obj)) return result @@ -2236,56 +2284,44 @@ def runtime_call( params = {} try: - runtime_call_def = self.runtime_config.type_registry["runtime_api"][api][ - "methods" - ][method] - runtime_api_types = self.runtime_config.type_registry["runtime_api"][ - api - ].get("types", {}) + metadata_v15 = self.metadata_v15.value() + apis = {entry["name"]: entry for entry in metadata_v15["apis"]} + api_entry = apis[api] + methods = {entry["name"]: entry for entry in api_entry["methods"]} + runtime_call_def = methods[method] except KeyError: raise ValueError(f"Runtime API Call '{api}.{method}' not found in registry") - if isinstance(params, list) and len(params) != len(runtime_call_def["params"]): + if isinstance(params, list) and len(params) != len(runtime_call_def["inputs"]): raise ValueError( f"Number of parameter provided ({len(params)}) does not " - f"match definition {len(runtime_call_def['params'])}" + f"match definition {len(runtime_call_def['inputs'])}" ) - # Add runtime API types to registry - self.runtime_config.update_type_registry_types(runtime_api_types) - runtime = Runtime( - self.chain, - self.runtime_config, - self._metadata, - self.type_registry, - ) - # Encode params - param_data = ScaleBytes(bytes()) - for idx, param in enumerate(runtime_call_def["params"]): - scale_obj = runtime.runtime_config.create_scale_object(param["type"]) + param_data = b"" + for idx, param in enumerate(runtime_call_def["inputs"]): + param_type_string = f"scale_info::{param['ty']}" if isinstance(params, list): - param_data += scale_obj.encode(params[idx]) + param_data += self.encode_scale(param_type_string, params[idx]) else: if param["name"] not in params: raise ValueError(f"Runtime Call param '{param['name']}' is missing") - param_data += scale_obj.encode(params[param["name"]]) + param_data += self.encode_scale( + param_type_string, params[param["name"]] + ) # RPC request result_data = self.rpc_request( - "state_call", [f"{api}_{method}", str(param_data), block_hash] + "state_call", [f"{api}_{method}", param_data.hex(), block_hash] ) + output_type_string = f"scale_info::{runtime_call_def['output']}" # Decode result - # TODO update this to use bt-decode - result_obj = runtime.runtime_config.create_scale_object( - runtime_call_def["type"] - ) - result_obj.decode( - ScaleBytes(result_data["result"]), - check_remaining=self.config.get("strict_scale_decode"), - ) + result_bytes = hex_to_bytes(result_data["result"]) + result_obj = Bytes(ScaleBytes(result_bytes)) + result_obj.value = self.decode_scale(output_type_string, result_bytes) return result_obj @@ -2410,8 +2446,7 @@ def get_payment_info(self, call: GenericCall, keypair: Keypair) -> dict[str, Any extrinsic = self.create_signed_extrinsic( call=call, keypair=keypair, signature=signature ) - extrinsic_len = self.runtime_config.create_scale_object("u32") - extrinsic_len.encode(len(extrinsic.data)) + extrinsic_len = len(extrinsic.data) result = self.runtime_call( "TransactionPaymentApi", "query_info", [extrinsic, extrinsic_len] diff --git a/tests/unit_tests/asyncio/test_substrate_interface.py b/tests/unit_tests/asyncio/test_substrate_interface.py index 04914cb..c9d28b8 100644 --- a/tests/unit_tests/asyncio/test_substrate_interface.py +++ b/tests/unit_tests/asyncio/test_substrate_interface.py @@ -1,4 +1,7 @@ +import unittest.mock + import pytest +import scalecodec.base from websockets.exceptions import InvalidURI from async_substrate_interface.async_substrate import AsyncSubstrateInterface @@ -16,3 +19,51 @@ async def test_invalid_url_raises_exception(): "non_existent_entry_point" ) as async_substrate: pass + + +@pytest.mark.asyncio +async def test_runtime_call(monkeypatch): + monkeypatch.setattr( + "async_substrate_interface.async_substrate.Websocket", unittest.mock.Mock() + ) + + substrate = AsyncSubstrateInterface("ws://localhost") + substrate._metadata = unittest.mock.Mock() + substrate.metadata_v15 = unittest.mock.Mock( + **{ + "value.return_value": { + "apis": [ + { + "name": "SubstrateApi", + "methods": [ + { + "name": "SubstrateMethod", + "inputs": [], + "output": "1", + }, + ], + }, + ], + }, + } + ) + substrate.rpc_request = unittest.mock.AsyncMock( + return_value={ + "result": "0x00", + }, + ) + substrate.decode_scale = unittest.mock.AsyncMock() + + result = await substrate.runtime_call( + "SubstrateApi", + "SubstrateMethod", + ) + + assert isinstance(result, scalecodec.base.ScaleType) + assert result.value is substrate.decode_scale.return_value + + substrate.rpc_request.assert_called_once_with( + "state_call", + ["SubstrateApi_SubstrateMethod", "", None], + ) + substrate.decode_scale.assert_called_once_with("scale_info::1", b"\x00") diff --git a/tests/unit_tests/sync/__init__.py b/tests/unit_tests/sync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/sync/test_substrate_interface.py b/tests/unit_tests/sync/test_substrate_interface.py new file mode 100644 index 0000000..0373c6a --- /dev/null +++ b/tests/unit_tests/sync/test_substrate_interface.py @@ -0,0 +1,55 @@ +import unittest.mock + +import scalecodec.base + +from async_substrate_interface.sync_substrate import SubstrateInterface + + +def test_runtime_call(monkeypatch): + monkeypatch.setattr( + "async_substrate_interface.sync_substrate.connect", unittest.mock.MagicMock() + ) + + substrate = SubstrateInterface( + "ws://localhost", + _mock=True, + ) + substrate._metadata = unittest.mock.Mock() + substrate.metadata_v15 = unittest.mock.Mock( + **{ + "value.return_value": { + "apis": [ + { + "name": "SubstrateApi", + "methods": [ + { + "name": "SubstrateMethod", + "inputs": [], + "output": "1", + }, + ], + }, + ], + }, + } + ) + substrate.rpc_request = unittest.mock.Mock( + return_value={ + "result": "0x00", + }, + ) + substrate.decode_scale = unittest.mock.Mock() + + result = substrate.runtime_call( + "SubstrateApi", + "SubstrateMethod", + ) + + assert isinstance(result, scalecodec.base.ScaleType) + assert result.value is substrate.decode_scale.return_value + + substrate.rpc_request.assert_called_once_with( + "state_call", + ["SubstrateApi_SubstrateMethod", "", None], + ) + substrate.decode_scale.assert_called_once_with("scale_info::1", b"\x00") From 24f3041e249fc0eae6531e25f1d17750c264d0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Wed, 29 Jan 2025 17:24:50 +0100 Subject: [PATCH 03/14] fix: return ScaleObj --- async_substrate_interface/async_substrate.py | 3 +-- async_substrate_interface/sync_substrate.py | 3 +-- async_substrate_interface/types.py | 5 ++++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 413bd9a..3b1aed4 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -2586,8 +2586,7 @@ async def runtime_call( # Decode result result_bytes = hex_to_bytes(result_data["result"]) - result_obj = Bytes(ScaleBytes(result_bytes)) - result_obj.value = await self.decode_scale(output_type_string, result_bytes) + result_obj = ScaleObj(await self.decode_scale(output_type_string, result_bytes)) return result_obj diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 36b8305..49f1ed2 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -2320,8 +2320,7 @@ def runtime_call( # Decode result result_bytes = hex_to_bytes(result_data["result"]) - result_obj = Bytes(ScaleBytes(result_bytes)) - result_obj.value = self.decode_scale(output_type_string, result_bytes) + result_obj = ScaleObj(self.decode_scale(output_type_string, result_bytes)) return result_obj diff --git a/async_substrate_interface/types.py b/async_substrate_interface/types.py index 3f96bc6..ae2c894 100644 --- a/async_substrate_interface/types.py +++ b/async_substrate_interface/types.py @@ -209,7 +209,7 @@ class Preprocessed: storage_item: ScaleType -class ScaleObj: +class ScaleObj(ScaleType): """Bittensor representation of Scale Object.""" def __init__(self, value): @@ -316,6 +316,9 @@ def __iter__(self): def __len__(self): return len(self.value) + def process(self): + pass + def serialize(self): return self.value From c6597dd91f6011179d4a4eee31cae33d87771e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Wed, 29 Jan 2025 17:34:59 +0100 Subject: [PATCH 04/14] fix: unneeded condition (already covered in decode_scale) --- async_substrate_interface/async_substrate.py | 15 +++++++-------- async_substrate_interface/sync_substrate.py | 13 +++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 3b1aed4..3f9db6c 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -2211,17 +2211,16 @@ async def query_multi( storage_key = storage_key_map[change_storage_key] if change_data is None: change_data = b"\x00" - obj = None else: change_data = bytes.fromhex(change_data[2:]) - if change_data == b"\x00": - obj = None - else: - obj = await self.decode_scale( + result.append( + ( + storage_key, + await self.decode_scale( storage_key.value_scale_type, change_data - ) - - result.append((storage_key, obj)) + ), + ), + ) return result diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 49f1ed2..6eb9dde 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -1953,13 +1953,14 @@ def query_multi( change_data = b"\x00" else: change_data = bytes.fromhex(change_data[2:]) - if change_data == b"\x00": - obj = None - else: - obj = self.decode_scale( + result.append( + ( + storage_key, + self.decode_scale( storage_key.value_scale_type, change_data - ) - result.append((storage_key, obj)) + ), + ), + ) return result From 99ac1637f5107c58d20a49b1e43007e058175a30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Wed, 29 Jan 2025 17:36:30 +0100 Subject: [PATCH 05/14] style: ruff formatting --- async_substrate_interface/sync_substrate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 6eb9dde..ca22bbb 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -1956,9 +1956,7 @@ def query_multi( result.append( ( storage_key, - self.decode_scale( - storage_key.value_scale_type, change_data - ), + self.decode_scale(storage_key.value_scale_type, change_data), ), ) From fba32e1a52b3ec46bde8633aa7582cd7d00a9849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Wed, 29 Jan 2025 17:37:54 +0100 Subject: [PATCH 06/14] fix: get ScaleObj.value --- async_substrate_interface/async_substrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 3f9db6c..eb9050f 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -2718,7 +2718,7 @@ async def get_payment_info( "TransactionPaymentApi", "query_info", [extrinsic, extrinsic_len] ) - return result.result + return result.value async def get_type_registry( self, block_hash: str = None, max_recursion: int = 4 From 356eeeaf23347f6c2dfd1fa57f38a54cf253dd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=BBy=C5=BAniewski?= Date: Wed, 29 Jan 2025 17:41:28 +0100 Subject: [PATCH 07/14] fix: remove unused code --- async_substrate_interface/async_substrate.py | 3 +-- async_substrate_interface/sync_substrate.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index eb9050f..719e75d 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -31,8 +31,7 @@ encode as encode_by_type_string, ) from scalecodec.base import ScaleBytes, ScaleType, RuntimeConfigurationObject -from scalecodec.types import ( - Bytes, +from scalecodec.types import ( GenericCall, GenericExtrinsic, GenericRuntimeCallDefinition, diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index ca22bbb..68b0774 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -13,7 +13,6 @@ encode as encode_by_type_string, ) from scalecodec import ( - Bytes, GenericCall, GenericExtrinsic, GenericRuntimeCallDefinition, From c294eee47a8dba161e42cb326621df4c208fa494 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 29 Jan 2025 21:41:32 +0200 Subject: [PATCH 08/14] Removed superclass of ScaleObj --- async_substrate_interface/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async_substrate_interface/types.py b/async_substrate_interface/types.py index ae2c894..93ba00b 100644 --- a/async_substrate_interface/types.py +++ b/async_substrate_interface/types.py @@ -209,7 +209,7 @@ class Preprocessed: storage_item: ScaleType -class ScaleObj(ScaleType): +class ScaleObj: """Bittensor representation of Scale Object.""" def __init__(self, value): From aa7ad25f9ba83f2be72edb86049ea3c808fc6455 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 29 Jan 2025 22:46:54 +0200 Subject: [PATCH 09/14] Ruff + _metadata_cache fix --- async_substrate_interface/async_substrate.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 96df385..53de792 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -31,7 +31,7 @@ encode as encode_by_type_string, ) from scalecodec.base import ScaleBytes, ScaleType, RuntimeConfigurationObject -from scalecodec.types import ( +from scalecodec.types import ( GenericCall, GenericExtrinsic, GenericRuntimeCallDefinition, @@ -710,7 +710,7 @@ def __init__( self.runtime_config = RuntimeConfigurationObject( ss58_format=self.ss58_format, implements_scale_info=True ) - self.__metadata_cache = {} + self._metadata_cache = {} self._nonces = {} self.metadata_version_hex = "0x0f000000" # v15 self.reload_type_registry() @@ -2614,7 +2614,7 @@ async def get_account_next_index(self, account_address: str) -> int: """ This method maintains a cache of nonces for each account ss58address. Upon subsequent calls, it will return the cached nonce + 1 instead of fetching from the chain. - This allows for correct nonce management in-case of async context when gathering co-routines. + This allows for correct nonce management in-case of async context when gathering co-routines. Args: account_address: SS58 formatted address @@ -2628,7 +2628,9 @@ async def get_account_next_index(self, account_address: str) -> int: async with self._lock: if self._nonces.get(account_address) is None: - nonce_obj = await self.rpc_request("account_nextIndex", [account_address]) + nonce_obj = await self.rpc_request( + "account_nextIndex", [account_address] + ) self._nonces[account_address] = nonce_obj["result"] else: self._nonces[account_address] += 1 From e66a80470ed3a9715b2cd81a9a667260ecfe1147 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 29 Jan 2025 23:20:18 +0200 Subject: [PATCH 10/14] Moved the logic of encode_scale to the Mixin. Fix tests I broke. --- async_substrate_interface/async_substrate.py | 166 +++++++++--------- async_substrate_interface/sync_substrate.py | 59 +------ async_substrate_interface/types.py | 53 +++++- .../asyncio/test_substrate_interface.py | 4 +- .../sync/test_substrate_interface.py | 5 +- 5 files changed, 140 insertions(+), 147 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 53de792..02c2c8a 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -24,12 +24,7 @@ import asyncstdlib as a from bittensor_wallet.keypair import Keypair from bittensor_wallet.utils import SS58_FORMAT -from bt_decode import ( - MetadataV15, - PortableRegistry, - decode as decode_by_type_string, - encode as encode_by_type_string, -) +from bt_decode import MetadataV15, PortableRegistry, decode as decode_by_type_string from scalecodec.base import ScaleBytes, ScaleType, RuntimeConfigurationObject from scalecodec.types import ( GenericCall, @@ -805,6 +800,50 @@ async def load_registry(self): ) self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15) + async def _wait_for_registry(self, _attempt: int = 1, _retries: int = 3) -> None: + async def _waiter(): + while self.registry is None: + await asyncio.sleep(0.1) + return + + try: + if not self.registry: + await asyncio.wait_for(_waiter(), timeout=10) + except TimeoutError: + # indicates that registry was never loaded + if not self._initializing: + raise AttributeError( + "Registry was never loaded. This did not occur during initialization, which usually indicates " + "you must first initialize the AsyncSubstrateInterface object, either with " + "`await AsyncSubstrateInterface.initialize()` or running with `async with`" + ) + elif _attempt < _retries: + await self.load_registry() + return await self._wait_for_registry(_attempt + 1, _retries) + else: + raise AttributeError( + "Registry was never loaded. This occurred during initialization, which usually indicates a " + "connection or node error." + ) + + async def encode_scale( + self, type_string, value: Any, _attempt: int = 1, _retries: int = 3 + ) -> bytes: + """ + Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string + + Args: + type_string: the type string of the SCALE object for decoding + value: value to encode + _attempt: the current number of attempts to load the registry needed to encode the value + _retries: the maximum number of attempts to load the registry needed to encode the value + + Returns: + encoded bytes + """ + await self._wait_for_registry(_attempt, _retries) + return self._encode_scale(type_string, value) + async def decode_scale( self, type_string: str, @@ -812,7 +851,7 @@ async def decode_scale( _attempt=1, _retries=3, return_scale_obj=False, - ) -> Any: + ) -> Union[ScaleObj, Any]: """ Helper function to decode arbitrary SCALE-bytes (e.g. 0x02000000) according to given RUST type_string (e.g. BlockNumber). The relevant versioning information of the type (if defined) will be applied if block_hash @@ -828,95 +867,19 @@ async def decode_scale( Returns: Decoded object """ - - async def _wait_for_registry(): - while self.registry is None: - await asyncio.sleep(0.1) - return - if scale_bytes == b"\x00": obj = None if type_string == "scale_info::0": # Is an AccountId # Decode AccountId bytes to SS58 address return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) else: - try: - if not self.registry: - await asyncio.wait_for(_wait_for_registry(), timeout=10) - obj = decode_by_type_string(type_string, self.registry, scale_bytes) - except TimeoutError: - # indicates that registry was never loaded - if not self._initializing: - raise AttributeError( - "Registry was never loaded. This did not occur during initialization, which usually indicates " - "you must first initialize the AsyncSubstrateInterface object, either with " - "`await AsyncSubstrateInterface.initialize()` or running with `async with`" - ) - elif _attempt < _retries: - await self.load_registry() - return await self.decode_scale( - type_string, scale_bytes, _attempt + 1 - ) - else: - raise AttributeError( - "Registry was never loaded. This occurred during initialization, which usually indicates a " - "connection or node error." - ) + await self._wait_for_registry(_attempt, _retries) + obj = decode_by_type_string(type_string, self.registry, scale_bytes) if return_scale_obj: return ScaleObj(obj) else: return obj - async def encode_scale(self, type_string, value: Any) -> bytes: - """ - Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string - - Args: - type_string: the type string of the SCALE object for decoding - value: value to encode - - Returns: - encoded SCALE bytes - """ - if value is None: - result = b"\x00" - else: - if type_string == "scale_info::0": # Is an AccountId - # encode string into AccountId - ## AccountId is a composite type with one, unnamed field - return bytes.fromhex(ss58_decode(value, SS58_FORMAT)) - - elif type_string == "scale_info::151": # Vec - if not isinstance(value, (list, tuple)): - value = [value] - - # Encode length - length = len(value) - if length < 64: - result = bytes([length << 2]) # Single byte mode - else: - raise ValueError("Vector length too large") - - # Encode each AccountId - for account in value: - if isinstance(account, bytes): - result += account # Already encoded - else: - result += bytes.fromhex( - ss58_decode(value, SS58_FORMAT) - ) # SS58 string - return result - - if isinstance(value, ScaleType): - if value.data.data is not None: - # Already encoded - return bytes(value.data.data) - else: - value = value.value # Unwrap the value of the type - - result = bytes(encode_by_type_string(type_string, self.registry, value)) - return result - async def _first_initialize_runtime(self): """ TODO docstring @@ -2122,6 +2085,39 @@ async def compose_call( return call + # async def new_compose_call( + # self, + # call_module: str, + # call_function: str, + # call_params: Optional[dict] = None, + # block_hash: Optional[str] = None, + # ) -> GenericCall: + # """ + # Composes a call payload which can be used in an extrinsic. + # + # Args: + # call_module: Name of the runtime module e.g. Balances + # call_function: Name of the call function e.g. transfer + # call_params: This is a dict containing the params of the call. e.g. + # `{'dest': 'EaG2CRhJWPb7qmdcJvy3LiWdh26Jreu9Dx6R1rXxPmYXoDk', 'value': 1000000000000}` + # block_hash: Use metadata at given block_hash to compose call + # + # Returns: + # A composed call + # """ + # if call_params is None: + # call_params = {} + # + # encode_data = { + # "call_module": call_module, + # "call_function": call_function, + # "call_args": call_params, + # } + # + # call = await self.encode_scale("scale_info::411", encode_data) + # + # return call + async def query_multiple( self, params: list, diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 68b0774..1aa45db 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -6,12 +6,7 @@ from bittensor_wallet.keypair import Keypair from bittensor_wallet.utils import SS58_FORMAT -from bt_decode import ( - MetadataV15, - PortableRegistry, - decode as decode_by_type_string, - encode as encode_by_type_string, -) +from bt_decode import MetadataV15, PortableRegistry, decode as decode_by_type_string from scalecodec import ( GenericCall, GenericExtrinsic, @@ -630,56 +625,6 @@ def decode_scale( else: return obj - def encode_scale(self, type_string, value: Any) -> bytes: - """ - Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string - - Args: - type_string: the type string of the SCALE object for decoding - value: value to encode - - Returns: - encoded SCALE bytes - """ - if value is None: - result = b"\x00" - else: - if type_string == "scale_info::0": # Is an AccountId - # encode string into AccountId - ## AccountId is a composite type with one, unnamed field - return bytes.fromhex(ss58_decode(value, SS58_FORMAT)) - - elif type_string == "scale_info::151": # Vec - if not isinstance(value, (list, tuple)): - value = [value] - - # Encode length - length = len(value) - if length < 64: - result = bytes([length << 2]) # Single byte mode - else: - raise ValueError("Vector length too large") - - # Encode each AccountId - for account in value: - if isinstance(account, bytes): - result += account # Already encoded - else: - result += bytes.fromhex( - ss58_decode(value, SS58_FORMAT) - ) # SS58 string - return result - - if isinstance(value, ScaleType): - if value.data.data is not None: - # Already encoded - return bytes(value.data.data) - else: - value = value.value # Unwrap the value of the type - - result = bytes(encode_by_type_string(type_string, self.registry, value)) - return result - def _first_initialize_runtime(self): """ TODO docstring @@ -2933,3 +2878,5 @@ def close(self): self.ws.shutdown() except AttributeError: pass + + encode_scale = SubstrateMixin._encode_scale diff --git a/async_substrate_interface/types.py b/async_substrate_interface/types.py index 93ba00b..a2237f3 100644 --- a/async_substrate_interface/types.py +++ b/async_substrate_interface/types.py @@ -6,7 +6,8 @@ from datetime import datetime from typing import Optional, Union, Any -from bt_decode import PortableRegistry +from bt_decode import PortableRegistry, encode as encode_by_type_string +from bittensor_wallet.utils import SS58_FORMAT from scalecodec import ss58_encode, ss58_decode, is_valid_ss58_address from scalecodec.base import RuntimeConfigurationObject, ScaleBytes from scalecodec.type_registry import load_type_registry_preset @@ -705,3 +706,53 @@ def make_payload(id_: str, method: str, params: list) -> dict: "id": id_, "payload": {"jsonrpc": "2.0", "method": method, "params": params}, } + + def _encode_scale(self, type_string, value: Any) -> bytes: + """ + Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string + + Args: + type_string: the type string of the SCALE object for decoding + value: value to encode + + Returns: + encoded bytes + """ + if value is None: + result = b"\x00" + else: + if type_string == "scale_info::0": # Is an AccountId + # encode string into AccountId + ## AccountId is a composite type with one, unnamed field + return bytes.fromhex(ss58_decode(value, SS58_FORMAT)) + + elif type_string == "scale_info::151": # Vec + if not isinstance(value, (list, tuple)): + value = [value] + + # Encode length + length = len(value) + if length < 64: + result = bytes([length << 2]) # Single byte mode + else: + raise ValueError("Vector length too large") + + # Encode each AccountId + for account in value: + if isinstance(account, bytes): + result += account # Already encoded + else: + result += bytes.fromhex( + ss58_decode(value, SS58_FORMAT) + ) # SS58 string + return result + + if isinstance(value, ScaleType): + if value.data.data is not None: + # Already encoded + return bytes(value.data.data) + else: + value = value.value # Unwrap the value of the type + + result = bytes(encode_by_type_string(type_string, self.registry, value)) + return result diff --git a/tests/unit_tests/asyncio/test_substrate_interface.py b/tests/unit_tests/asyncio/test_substrate_interface.py index c9d28b8..b1ee98b 100644 --- a/tests/unit_tests/asyncio/test_substrate_interface.py +++ b/tests/unit_tests/asyncio/test_substrate_interface.py @@ -1,10 +1,10 @@ import unittest.mock import pytest -import scalecodec.base from websockets.exceptions import InvalidURI from async_substrate_interface.async_substrate import AsyncSubstrateInterface +from async_substrate_interface.types import ScaleObj @pytest.mark.asyncio @@ -59,7 +59,7 @@ async def test_runtime_call(monkeypatch): "SubstrateMethod", ) - assert isinstance(result, scalecodec.base.ScaleType) + assert isinstance(result, ScaleObj) assert result.value is substrate.decode_scale.return_value substrate.rpc_request.assert_called_once_with( diff --git a/tests/unit_tests/sync/test_substrate_interface.py b/tests/unit_tests/sync/test_substrate_interface.py index 0373c6a..18e85ea 100644 --- a/tests/unit_tests/sync/test_substrate_interface.py +++ b/tests/unit_tests/sync/test_substrate_interface.py @@ -1,8 +1,7 @@ import unittest.mock -import scalecodec.base - from async_substrate_interface.sync_substrate import SubstrateInterface +from async_substrate_interface.types import ScaleObj def test_runtime_call(monkeypatch): @@ -45,7 +44,7 @@ def test_runtime_call(monkeypatch): "SubstrateMethod", ) - assert isinstance(result, scalecodec.base.ScaleType) + assert isinstance(result, ScaleObj) assert result.value is substrate.decode_scale.return_value substrate.rpc_request.assert_called_once_with( From de9cf6546e454a06173a059f1d22d937c29d9427 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 29 Jan 2025 23:22:20 +0200 Subject: [PATCH 11/14] Remove experimental method. --- async_substrate_interface/async_substrate.py | 33 -------------------- 1 file changed, 33 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 02c2c8a..ecf9360 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -2085,39 +2085,6 @@ async def compose_call( return call - # async def new_compose_call( - # self, - # call_module: str, - # call_function: str, - # call_params: Optional[dict] = None, - # block_hash: Optional[str] = None, - # ) -> GenericCall: - # """ - # Composes a call payload which can be used in an extrinsic. - # - # Args: - # call_module: Name of the runtime module e.g. Balances - # call_function: Name of the call function e.g. transfer - # call_params: This is a dict containing the params of the call. e.g. - # `{'dest': 'EaG2CRhJWPb7qmdcJvy3LiWdh26Jreu9Dx6R1rXxPmYXoDk', 'value': 1000000000000}` - # block_hash: Use metadata at given block_hash to compose call - # - # Returns: - # A composed call - # """ - # if call_params is None: - # call_params = {} - # - # encode_data = { - # "call_module": call_module, - # "call_function": call_function, - # "call_args": call_params, - # } - # - # call = await self.encode_scale("scale_info::411", encode_data) - # - # return call - async def query_multiple( self, params: list, From 72032564ac89132cedb99bd9ce451008b6d30a74 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 29 Jan 2025 23:34:04 +0200 Subject: [PATCH 12/14] Update bt-decode requirement --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d70afd1..7f41d02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "wheel", "asyncstdlib~=3.13.0", "bittensor-wallet>=2.1.3", - "bt-decode==0.4.0", + "bt-decode==v0.5.0-a0", "scalecodec~=1.2.11", "websockets>=14.1", "xxhash" From 2f6774629128e57b74a3378721f230e5816f010e Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 30 Jan 2025 00:25:42 +0200 Subject: [PATCH 13/14] Fixes decode-scale by setting the object to None correctly. --- async_substrate_interface/async_substrate.py | 11 ++++++----- async_substrate_interface/sync_substrate.py | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index ecf9360..01889ac 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -869,12 +869,13 @@ async def decode_scale( """ if scale_bytes == b"\x00": obj = None - if type_string == "scale_info::0": # Is an AccountId - # Decode AccountId bytes to SS58 address - return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) else: - await self._wait_for_registry(_attempt, _retries) - obj = decode_by_type_string(type_string, self.registry, scale_bytes) + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) + else: + await self._wait_for_registry(_attempt, _retries) + obj = decode_by_type_string(type_string, self.registry, scale_bytes) if return_scale_obj: return ScaleObj(obj) else: diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 1aa45db..c018a18 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -615,11 +615,12 @@ def decode_scale( if scale_bytes == b"\x00": obj = None - if type_string == "scale_info::0": # Is an AccountId - # Decode AccountId bytes to SS58 address - return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) else: - obj = decode_by_type_string(type_string, self.registry, scale_bytes) + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) + else: + obj = decode_by_type_string(type_string, self.registry, scale_bytes) if return_scale_obj: return ScaleObj(obj) else: From a4d9ba59af6c76837527168f27154410bf826b6e Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 29 Jan 2025 15:17:06 -0800 Subject: [PATCH 14/14] Bumps version and changelog --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca307c7..f0a92ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 1.0.0rc7 /2025-01-29 + +## What's Changed +* feat: use bt_decode in runtime_call by @zyzniewski-reef in https://github.com/opentensor/async-substrate-interface/pull/15 +* Move logic to mixin + fix tests by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/18 +* Fix decode scale by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/19 +* Backmerge main to staging rc5 by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/20 + ## 1.0.0rc6 /2025-01-28 ## What's Changed diff --git a/pyproject.toml b/pyproject.toml index 4dea09f..4d981ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "async-substrate-interface" -version = "1.0.0rc6" +version = "1.0.0rc7" description = "Asyncio library for interacting with substrate. Mostly API-compatible with py-substrate-interface" readme = "README.md" license = { file = "LICENSE" }