diff --git a/CHANGELOG.md b/CHANGELOG.md index ca307c7..f0a92ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 1.0.0rc7 /2025-01-29 + +## What's Changed +* feat: use bt_decode in runtime_call by @zyzniewski-reef in https://github.com/opentensor/async-substrate-interface/pull/15 +* Move logic to mixin + fix tests by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/18 +* Fix decode scale by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/19 +* Backmerge main to staging rc5 by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/20 + ## 1.0.0rc6 /2025-01-28 ## What's Changed diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index d442b98..01889ac 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -23,9 +23,15 @@ import asyncstdlib as a from bittensor_wallet.keypair import Keypair -from bt_decode import PortableRegistry, decode as decode_by_type_string, MetadataV15 +from bittensor_wallet.utils import SS58_FORMAT +from bt_decode import MetadataV15, PortableRegistry, decode as decode_by_type_string from scalecodec.base import ScaleBytes, ScaleType, RuntimeConfigurationObject -from scalecodec.types import GenericCall, GenericRuntimeCallDefinition, GenericExtrinsic +from scalecodec.types import ( + GenericCall, + GenericExtrinsic, + GenericRuntimeCallDefinition, + ss58_decode, +) from websockets.asyncio.client import connect from websockets.exceptions import ConnectionClosed @@ -789,8 +795,54 @@ async def load_registry(self): ) metadata_option_hex_str = metadata_rpc_result["result"] metadata_option_bytes = bytes.fromhex(metadata_option_hex_str[2:]) - metadata_v15 = MetadataV15.decode_from_metadata_option(metadata_option_bytes) - self.registry = PortableRegistry.from_metadata_v15(metadata_v15) + self.metadata_v15 = MetadataV15.decode_from_metadata_option( + metadata_option_bytes + ) + self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15) + + async def _wait_for_registry(self, _attempt: int = 1, _retries: int = 3) -> None: + async def _waiter(): + while self.registry is None: + await asyncio.sleep(0.1) + return + + try: + if not self.registry: + await asyncio.wait_for(_waiter(), timeout=10) + except TimeoutError: + # indicates that registry was never loaded + if not self._initializing: + raise AttributeError( + "Registry was never loaded. This did not occur during initialization, which usually indicates " + "you must first initialize the AsyncSubstrateInterface object, either with " + "`await AsyncSubstrateInterface.initialize()` or running with `async with`" + ) + elif _attempt < _retries: + await self.load_registry() + return await self._wait_for_registry(_attempt + 1, _retries) + else: + raise AttributeError( + "Registry was never loaded. This occurred during initialization, which usually indicates a " + "connection or node error." + ) + + async def encode_scale( + self, type_string, value: Any, _attempt: int = 1, _retries: int = 3 + ) -> bytes: + """ + Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string + + Args: + type_string: the type string of the SCALE object for decoding + value: value to encode + _attempt: the current number of attempts to load the registry needed to encode the value + _retries: the maximum number of attempts to load the registry needed to encode the value + + Returns: + encoded bytes + """ + await self._wait_for_registry(_attempt, _retries) + return self._encode_scale(type_string, value) async def decode_scale( self, @@ -799,7 +851,7 @@ async def decode_scale( _attempt=1, _retries=3, return_scale_obj=False, - ) -> Any: + ) -> Union[ScaleObj, Any]: """ Helper function to decode arbitrary SCALE-bytes (e.g. 0x02000000) according to given RUST type_string (e.g. BlockNumber). The relevant versioning information of the type (if defined) will be applied if block_hash @@ -815,62 +867,20 @@ async def decode_scale( Returns: Decoded object """ - - async def _wait_for_registry(): - while self.registry is None: - await asyncio.sleep(0.1) - return - if scale_bytes == b"\x00": obj = None else: - try: - if not self.registry: - await asyncio.wait_for(_wait_for_registry(), timeout=10) + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) + else: + await self._wait_for_registry(_attempt, _retries) obj = decode_by_type_string(type_string, self.registry, scale_bytes) - except TimeoutError: - # indicates that registry was never loaded - if not self._initializing: - raise AttributeError( - "Registry was never loaded. This did not occur during initialization, which usually indicates " - "you must first initialize the AsyncSubstrateInterface object, either with " - "`await AsyncSubstrateInterface.initialize()` or running with `async with`" - ) - elif _attempt < _retries: - await self.load_registry() - return await self.decode_scale( - type_string, scale_bytes, _attempt + 1 - ) - else: - raise AttributeError( - "Registry was never loaded. This occurred during initialization, which usually indicates a " - "connection or node error." - ) if return_scale_obj: return ScaleObj(obj) else: return obj - async def encode_scale(self, type_string, value, block_hash=None) -> ScaleBytes: - """ - Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string - - Args: - type_string: the type string of the SCALE object for decoding - value: value to encode - block_hash: the hash of the blockchain block whose metadata to use for encoding - - Returns: - ScaleBytes encoded value - """ - if not self._metadata or block_hash: - await self.init_runtime(block_hash=block_hash) - - obj = self.runtime_config.create_scale_object( - type_string=type_string, metadata=self._metadata - ) - return obj.encode(value) - async def _first_initialize_runtime(self): """ TODO docstring @@ -2173,7 +2183,7 @@ async def query_multi( await self.decode_scale( storage_key.value_scale_type, change_data ), - ) + ), ) return result @@ -2503,56 +2513,43 @@ async def runtime_call( params = {} try: - runtime_call_def = self.runtime_config.type_registry["runtime_api"][api][ - "methods" - ][method] - runtime_api_types = self.runtime_config.type_registry["runtime_api"][ - api - ].get("types", {}) + metadata_v15 = self.metadata_v15.value() + apis = {entry["name"]: entry for entry in metadata_v15["apis"]} + api_entry = apis[api] + methods = {entry["name"]: entry for entry in api_entry["methods"]} + runtime_call_def = methods[method] except KeyError: raise ValueError(f"Runtime API Call '{api}.{method}' not found in registry") - if isinstance(params, list) and len(params) != len(runtime_call_def["params"]): + if isinstance(params, list) and len(params) != len(runtime_call_def["inputs"]): raise ValueError( f"Number of parameter provided ({len(params)}) does not " - f"match definition {len(runtime_call_def['params'])}" + f"match definition {len(runtime_call_def['inputs'])}" ) - # Add runtime API types to registry - self.runtime_config.update_type_registry_types(runtime_api_types) - runtime = Runtime( - self.chain, - self.runtime_config, - self._metadata, - self.type_registry, - ) - # Encode params - param_data = ScaleBytes(bytes()) - for idx, param in enumerate(runtime_call_def["params"]): - scale_obj = runtime.runtime_config.create_scale_object(param["type"]) + param_data = b"" + for idx, param in enumerate(runtime_call_def["inputs"]): + param_type_string = f"scale_info::{param['ty']}" if isinstance(params, list): - param_data += scale_obj.encode(params[idx]) + param_data += await self.encode_scale(param_type_string, params[idx]) else: if param["name"] not in params: raise ValueError(f"Runtime Call param '{param['name']}' is missing") - param_data += scale_obj.encode(params[param["name"]]) + param_data += await self.encode_scale( + param_type_string, params[param["name"]] + ) # RPC request result_data = await self.rpc_request( - "state_call", [f"{api}_{method}", str(param_data), block_hash] + "state_call", [f"{api}_{method}", param_data.hex(), block_hash] ) + output_type_string = f"scale_info::{runtime_call_def['output']}" # Decode result - # TODO update this to use bt-decode - result_obj = runtime.runtime_config.create_scale_object( - runtime_call_def["type"] - ) - result_obj.decode( - ScaleBytes(result_data["result"]), - check_remaining=self.config.get("strict_scale_decode"), - ) + result_bytes = hex_to_bytes(result_data["result"]) + result_obj = ScaleObj(await self.decode_scale(output_type_string, result_bytes)) return result_obj @@ -2581,7 +2578,7 @@ async def get_account_next_index(self, account_address: str) -> int: """ This method maintains a cache of nonces for each account ss58address. Upon subsequent calls, it will return the cached nonce + 1 instead of fetching from the chain. - This allows for correct nonce management in-case of async context when gathering co-routines. + This allows for correct nonce management in-case of async context when gathering co-routines. Args: account_address: SS58 formatted address @@ -2595,7 +2592,9 @@ async def get_account_next_index(self, account_address: str) -> int: async with self._lock: if self._nonces.get(account_address) is None: - nonce_obj = await self.rpc_request("account_nextIndex", [account_address]) + nonce_obj = await self.rpc_request( + "account_nextIndex", [account_address] + ) self._nonces[account_address] = nonce_obj["result"] else: self._nonces[account_address] += 1 @@ -2686,8 +2685,7 @@ async def get_payment_info( extrinsic = await self.create_signed_extrinsic( call=call, keypair=keypair, signature=signature ) - extrinsic_len = self.runtime_config.create_scale_object("u32") - extrinsic_len.encode(len(extrinsic.data)) + extrinsic_len = len(extrinsic.data) result = await self.runtime_call( "TransactionPaymentApi", "query_info", [extrinsic, extrinsic_len] diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index c344e48..c018a18 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -5,8 +5,14 @@ from typing import Optional, Union, Callable, Any from bittensor_wallet.keypair import Keypair -from bt_decode import PortableRegistry, decode as decode_by_type_string, MetadataV15 -from scalecodec import GenericExtrinsic, GenericCall, GenericRuntimeCallDefinition +from bittensor_wallet.utils import SS58_FORMAT +from bt_decode import MetadataV15, PortableRegistry, decode as decode_by_type_string +from scalecodec import ( + GenericCall, + GenericExtrinsic, + GenericRuntimeCallDefinition, + ss58_decode, +) from scalecodec.base import RuntimeConfigurationObject, ScaleBytes, ScaleType from websockets.sync.client import connect @@ -582,8 +588,10 @@ def load_registry(self): ) metadata_option_hex_str = metadata_rpc_result["result"] metadata_option_bytes = bytes.fromhex(metadata_option_hex_str[2:]) - metadata_v15 = MetadataV15.decode_from_metadata_option(metadata_option_bytes) - self.registry = PortableRegistry.from_metadata_v15(metadata_v15) + self.metadata_v15 = MetadataV15.decode_from_metadata_option( + metadata_option_bytes + ) + self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15) def decode_scale( self, @@ -608,32 +616,16 @@ def decode_scale( if scale_bytes == b"\x00": obj = None else: - obj = decode_by_type_string(type_string, self.registry, scale_bytes) + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return bytes.fromhex(ss58_decode(scale_bytes, SS58_FORMAT)) + else: + obj = decode_by_type_string(type_string, self.registry, scale_bytes) if return_scale_obj: return ScaleObj(obj) else: return obj - def encode_scale(self, type_string, value, block_hash=None) -> ScaleBytes: - """ - Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string - - Args: - type_string: the type string of the SCALE object for decoding - value: value to encode - block_hash: the hash of the blockchain block whose metadata to use for encoding - - Returns: - ScaleBytes encoded value - """ - if not self._metadata or block_hash: - self.init_runtime(block_hash=block_hash) - - obj = self.runtime_config.create_scale_object( - type_string=type_string, metadata=self._metadata - ) - return obj.encode(value) - def _first_initialize_runtime(self): """ TODO docstring @@ -1910,7 +1902,7 @@ def query_multi( ( storage_key, self.decode_scale(storage_key.value_scale_type, change_data), - ) + ), ) return result @@ -2236,56 +2228,43 @@ def runtime_call( params = {} try: - runtime_call_def = self.runtime_config.type_registry["runtime_api"][api][ - "methods" - ][method] - runtime_api_types = self.runtime_config.type_registry["runtime_api"][ - api - ].get("types", {}) + metadata_v15 = self.metadata_v15.value() + apis = {entry["name"]: entry for entry in metadata_v15["apis"]} + api_entry = apis[api] + methods = {entry["name"]: entry for entry in api_entry["methods"]} + runtime_call_def = methods[method] except KeyError: raise ValueError(f"Runtime API Call '{api}.{method}' not found in registry") - if isinstance(params, list) and len(params) != len(runtime_call_def["params"]): + if isinstance(params, list) and len(params) != len(runtime_call_def["inputs"]): raise ValueError( f"Number of parameter provided ({len(params)}) does not " - f"match definition {len(runtime_call_def['params'])}" + f"match definition {len(runtime_call_def['inputs'])}" ) - # Add runtime API types to registry - self.runtime_config.update_type_registry_types(runtime_api_types) - runtime = Runtime( - self.chain, - self.runtime_config, - self._metadata, - self.type_registry, - ) - # Encode params - param_data = ScaleBytes(bytes()) - for idx, param in enumerate(runtime_call_def["params"]): - scale_obj = runtime.runtime_config.create_scale_object(param["type"]) + param_data = b"" + for idx, param in enumerate(runtime_call_def["inputs"]): + param_type_string = f"scale_info::{param['ty']}" if isinstance(params, list): - param_data += scale_obj.encode(params[idx]) + param_data += self.encode_scale(param_type_string, params[idx]) else: if param["name"] not in params: raise ValueError(f"Runtime Call param '{param['name']}' is missing") - param_data += scale_obj.encode(params[param["name"]]) + param_data += self.encode_scale( + param_type_string, params[param["name"]] + ) # RPC request result_data = self.rpc_request( - "state_call", [f"{api}_{method}", str(param_data), block_hash] + "state_call", [f"{api}_{method}", param_data.hex(), block_hash] ) + output_type_string = f"scale_info::{runtime_call_def['output']}" # Decode result - # TODO update this to use bt-decode - result_obj = runtime.runtime_config.create_scale_object( - runtime_call_def["type"] - ) - result_obj.decode( - ScaleBytes(result_data["result"]), - check_remaining=self.config.get("strict_scale_decode"), - ) + result_bytes = hex_to_bytes(result_data["result"]) + result_obj = ScaleObj(self.decode_scale(output_type_string, result_bytes)) return result_obj @@ -2410,8 +2389,7 @@ def get_payment_info(self, call: GenericCall, keypair: Keypair) -> dict[str, Any extrinsic = self.create_signed_extrinsic( call=call, keypair=keypair, signature=signature ) - extrinsic_len = self.runtime_config.create_scale_object("u32") - extrinsic_len.encode(len(extrinsic.data)) + extrinsic_len = len(extrinsic.data) result = self.runtime_call( "TransactionPaymentApi", "query_info", [extrinsic, extrinsic_len] @@ -2901,3 +2879,5 @@ def close(self): self.ws.shutdown() except AttributeError: pass + + encode_scale = SubstrateMixin._encode_scale diff --git a/async_substrate_interface/types.py b/async_substrate_interface/types.py index 3f96bc6..a2237f3 100644 --- a/async_substrate_interface/types.py +++ b/async_substrate_interface/types.py @@ -6,7 +6,8 @@ from datetime import datetime from typing import Optional, Union, Any -from bt_decode import PortableRegistry +from bt_decode import PortableRegistry, encode as encode_by_type_string +from bittensor_wallet.utils import SS58_FORMAT from scalecodec import ss58_encode, ss58_decode, is_valid_ss58_address from scalecodec.base import RuntimeConfigurationObject, ScaleBytes from scalecodec.type_registry import load_type_registry_preset @@ -316,6 +317,9 @@ def __iter__(self): def __len__(self): return len(self.value) + def process(self): + pass + def serialize(self): return self.value @@ -702,3 +706,53 @@ def make_payload(id_: str, method: str, params: list) -> dict: "id": id_, "payload": {"jsonrpc": "2.0", "method": method, "params": params}, } + + def _encode_scale(self, type_string, value: Any) -> bytes: + """ + Helper function to encode arbitrary data into SCALE-bytes for given RUST type_string + + Args: + type_string: the type string of the SCALE object for decoding + value: value to encode + + Returns: + encoded bytes + """ + if value is None: + result = b"\x00" + else: + if type_string == "scale_info::0": # Is an AccountId + # encode string into AccountId + ## AccountId is a composite type with one, unnamed field + return bytes.fromhex(ss58_decode(value, SS58_FORMAT)) + + elif type_string == "scale_info::151": # Vec + if not isinstance(value, (list, tuple)): + value = [value] + + # Encode length + length = len(value) + if length < 64: + result = bytes([length << 2]) # Single byte mode + else: + raise ValueError("Vector length too large") + + # Encode each AccountId + for account in value: + if isinstance(account, bytes): + result += account # Already encoded + else: + result += bytes.fromhex( + ss58_decode(value, SS58_FORMAT) + ) # SS58 string + return result + + if isinstance(value, ScaleType): + if value.data.data is not None: + # Already encoded + return bytes(value.data.data) + else: + value = value.value # Unwrap the value of the type + + result = bytes(encode_by_type_string(type_string, self.registry, value)) + return result diff --git a/pyproject.toml b/pyproject.toml index 3ccc4ef..4d981ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "async-substrate-interface" -version = "1.0.0rc6" +version = "1.0.0rc7" description = "Asyncio library for interacting with substrate. Mostly API-compatible with py-substrate-interface" readme = "README.md" license = { file = "LICENSE" } @@ -10,7 +10,7 @@ dependencies = [ "wheel", "asyncstdlib~=3.13.0", "bittensor-wallet>=2.1.3", - "bt-decode==0.4.0", + "bt-decode==v0.5.0-a0", "scalecodec~=1.2.11", "websockets>=14.1", "xxhash" diff --git a/tests/unit_tests/asyncio/test_substrate_interface.py b/tests/unit_tests/asyncio/test_substrate_interface.py index 04914cb..b1ee98b 100644 --- a/tests/unit_tests/asyncio/test_substrate_interface.py +++ b/tests/unit_tests/asyncio/test_substrate_interface.py @@ -1,7 +1,10 @@ +import unittest.mock + import pytest from websockets.exceptions import InvalidURI from async_substrate_interface.async_substrate import AsyncSubstrateInterface +from async_substrate_interface.types import ScaleObj @pytest.mark.asyncio @@ -16,3 +19,51 @@ async def test_invalid_url_raises_exception(): "non_existent_entry_point" ) as async_substrate: pass + + +@pytest.mark.asyncio +async def test_runtime_call(monkeypatch): + monkeypatch.setattr( + "async_substrate_interface.async_substrate.Websocket", unittest.mock.Mock() + ) + + substrate = AsyncSubstrateInterface("ws://localhost") + substrate._metadata = unittest.mock.Mock() + substrate.metadata_v15 = unittest.mock.Mock( + **{ + "value.return_value": { + "apis": [ + { + "name": "SubstrateApi", + "methods": [ + { + "name": "SubstrateMethod", + "inputs": [], + "output": "1", + }, + ], + }, + ], + }, + } + ) + substrate.rpc_request = unittest.mock.AsyncMock( + return_value={ + "result": "0x00", + }, + ) + substrate.decode_scale = unittest.mock.AsyncMock() + + result = await substrate.runtime_call( + "SubstrateApi", + "SubstrateMethod", + ) + + assert isinstance(result, ScaleObj) + assert result.value is substrate.decode_scale.return_value + + substrate.rpc_request.assert_called_once_with( + "state_call", + ["SubstrateApi_SubstrateMethod", "", None], + ) + substrate.decode_scale.assert_called_once_with("scale_info::1", b"\x00") diff --git a/tests/unit_tests/sync/__init__.py b/tests/unit_tests/sync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/sync/test_substrate_interface.py b/tests/unit_tests/sync/test_substrate_interface.py new file mode 100644 index 0000000..18e85ea --- /dev/null +++ b/tests/unit_tests/sync/test_substrate_interface.py @@ -0,0 +1,54 @@ +import unittest.mock + +from async_substrate_interface.sync_substrate import SubstrateInterface +from async_substrate_interface.types import ScaleObj + + +def test_runtime_call(monkeypatch): + monkeypatch.setattr( + "async_substrate_interface.sync_substrate.connect", unittest.mock.MagicMock() + ) + + substrate = SubstrateInterface( + "ws://localhost", + _mock=True, + ) + substrate._metadata = unittest.mock.Mock() + substrate.metadata_v15 = unittest.mock.Mock( + **{ + "value.return_value": { + "apis": [ + { + "name": "SubstrateApi", + "methods": [ + { + "name": "SubstrateMethod", + "inputs": [], + "output": "1", + }, + ], + }, + ], + }, + } + ) + substrate.rpc_request = unittest.mock.Mock( + return_value={ + "result": "0x00", + }, + ) + substrate.decode_scale = unittest.mock.Mock() + + result = substrate.runtime_call( + "SubstrateApi", + "SubstrateMethod", + ) + + assert isinstance(result, ScaleObj) + assert result.value is substrate.decode_scale.return_value + + substrate.rpc_request.assert_called_once_with( + "state_call", + ["SubstrateApi_SubstrateMethod", "", None], + ) + substrate.decode_scale.assert_called_once_with("scale_info::1", b"\x00")