diff --git a/CHANGELOG.md b/CHANGELOG.md index 166e9bf..0d841ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 1.0.2 /2025-02-19 + +## What's Changed +* Closes the connection on the object being garbage-collected by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/51 +* Generate UIDs for websockets by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/50 +* Dynamically pulls the info for Vec from the metadata by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/47 +* Fix readme by @igorsyl in https://github.com/opentensor/async-substrate-interface/pull/46 +* Handle options with bt-decode by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/52 +* Backmerge main to staging 101 by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/53 +* Handles None change_data by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/54 + +## New Contributors +* @igorsyl made their first contribution in https://github.com/opentensor/async-substrate-interface/pull/46 + +**Full Changelog**: https://github.com/opentensor/async-substrate-interface/compare/v1.0.1...v1.0.2 + +## 1.0.1 /2025-02-17 + ## What's Changed * Updates type for vec acc id by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/45 * Backmerge main staging 101 by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/48 diff --git a/README.md b/README.md index aa8f1cd..6b78d0d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,67 @@ -This is a modernised version of the py-substrate-interface library, with the ability to use it asynchronously (as well as synchronously). It aims to be almost fully API-compatible with the original library. +# Async Substrate Interface +This project provides an asynchronous interface for interacting with [Substrate](https://substrate.io/)-based blockchains. It is based on the [py-substrate-interface](https://github.com/polkascan/py-substrate-interface) project. -In addition to it's async nature, it is additionally improved with using bt-decode rather than py-scale-codec for significantly faster SCALE decoding. +Additionally, this project uses [bt-decode](https://github.com/opentensor/bt-decode) instead of [py-scale-codec](https://github.com/polkascan/py-scale-codec) for faster [SCALE](https://docs.substrate.io/reference/scale-codec/) decoding. + +## Installation + +To install the package, use the following command: + +```bash +pip install async-substrate-interface +``` + +## Usage + +Here are examples of how to use the sync and async inferfaces: + +```python +from async_substrate_interface import SubstrateInterface + +def main(): + substrate = SubstrateInterface( + url="wss://rpc.polkadot.io" + ) + with substrate: + result = substrate.query( + module='System', + storage_function='Account', + params=['5CZs3T15Ky4jch1sUpSFwkUbYEnsCfe1WCY51fH3SPV6NFnf'] + ) + + print(result) + +main() +``` + +```python +import asyncio +from async_substrate_interface import AsyncSubstrateInterface + +async def main(): + substrate = AsyncSubstrateInterface( + url="wss://rpc.polkadot.io" + ) + async with substrate: + result = await substrate.query( + module='System', + storage_function='Account', + params=['5CZs3T15Ky4jch1sUpSFwkUbYEnsCfe1WCY51fH3SPV6NFnf'] + ) + + print(result) + +asyncio.run(main()) +``` + +## Contributing + +Contributions are welcome! Please open an issue or submit a pull request to the `staging` branch. + +## License + +This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. + +## Contact + +For any questions or inquiries, please join the Bittensor Development Discord server: [Church of Rao](https://discord.gg/XC7ucQmq2Q). diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 9cad7e0..be112be 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -48,7 +48,7 @@ SubstrateMixin, Preprocessed, ) -from async_substrate_interface.utils import hex_to_bytes, json +from async_substrate_interface.utils import hex_to_bytes, json, generate_unique_id from async_substrate_interface.utils.decoding import ( _determine_if_old_runtime_call, _bt_decode_to_dict_or_list, @@ -507,7 +507,6 @@ def __init__( # TODO reconnection logic self.ws_url = ws_url self.ws: Optional["ClientConnection"] = None - self.id = 0 self.max_subscriptions = max_subscriptions self.max_connections = max_connections self.shutdown_timer = shutdown_timer @@ -543,8 +542,6 @@ async def connect(self, force=False): connect(self.ws_url, **self._options), timeout=10 ) self._receiving_task = asyncio.create_task(self._start_receiving()) - if force: - self.id = 100 async def __aexit__(self, exc_type, exc_val, exc_tb): async with self._lock: # TODO is this actually what I want to happen? @@ -556,7 +553,6 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): except asyncio.CancelledError: pass if self._in_use == 0 and self.ws is not None: - self.id = 0 self._open_subscriptions = 0 self._exit_task = asyncio.create_task(self._exit_with_timer()) @@ -582,7 +578,6 @@ async def shutdown(self): self.ws = None self._initialized = False self._receiving_task = None - self.id = 0 async def _recv(self) -> None: try: @@ -625,8 +620,7 @@ async def send(self, payload: dict) -> int: id: the internal ID of the request (incremented int) """ # async with self._lock: - original_id = self.id - self.id += 1 + original_id = generate_unique_id(json.dumps(payload)) # self._open_subscriptions += 1 try: await self.ws.send(json.dumps({**payload, **{"id": original_id}})) @@ -719,6 +713,8 @@ def __init__( self.metadata_version_hex = "0x0f000000" # v15 self.reload_type_registry() self._initializing = False + self.registry_type_map = {} + self.type_id_to_name = {} async def __aenter__(self): await self.initialize() @@ -735,8 +731,9 @@ async def initialize(self): chain = await self.rpc_request("system_chain", []) self._chain = chain.get("result") init_load = await asyncio.gather( - self.load_registry(), self._first_initialize_runtime(), - return_exceptions=True + self.load_registry(), + self._first_initialize_runtime(), + return_exceptions=True, ) for potential_exception in init_load: if isinstance(potential_exception, Exception): @@ -812,6 +809,7 @@ async def load_registry(self): metadata_option_bytes ) self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15) + self._load_registry_type_map() async def _load_registry_at_block(self, block_hash: str) -> MetadataV15: # Should be called for any block that fails decoding. @@ -894,15 +892,14 @@ async def decode_scale( Returns: Decoded object """ - if scale_bytes == b"\x00": - obj = None + if scale_bytes == b"": + return None + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return ss58_encode(scale_bytes, SS58_FORMAT) else: - if type_string == "scale_info::0": # Is an AccountId - # Decode AccountId bytes to SS58 address - return ss58_encode(scale_bytes, SS58_FORMAT) - else: - await self._wait_for_registry(_attempt, _retries) - obj = decode_by_type_string(type_string, self.registry, scale_bytes) + await self._wait_for_registry(_attempt, _retries) + obj = decode_by_type_string(type_string, self.registry, scale_bytes) if return_scale_obj: return ScaleObj(obj) else: @@ -2235,7 +2232,7 @@ async def query_multi( # Decode result for specified storage_key storage_key = storage_key_map[change_storage_key] if change_data is None: - change_data = b"\x00" + change_data = b"" else: change_data = bytes.fromhex(change_data[2:]) result.append( diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 8bb5211..d489ff8 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -30,7 +30,7 @@ Preprocessed, ScaleObj, ) -from async_substrate_interface.utils import hex_to_bytes, json +from async_substrate_interface.utils import hex_to_bytes, json, generate_unique_id from async_substrate_interface.utils.decoding import ( _determine_if_old_runtime_call, _bt_decode_to_dict_or_list, @@ -518,6 +518,8 @@ def __init__( self.metadata_version_hex = "0x0f000000" # v15 self.reload_type_registry() self.ws = self.connect(init=True) + self.registry_type_map = {} + self.type_id_to_name = {} if not _mock: self.initialize() @@ -525,6 +527,9 @@ def __enter__(self): self.initialize() return self + def __del__(self): + self.close() + def initialize(self): """ Initialize the connection to the chain. @@ -612,6 +617,7 @@ def load_registry(self): metadata_option_bytes ) self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15) + self._load_registry_type_map() def _load_registry_at_block(self, block_hash: str) -> MetadataV15: # Should be called for any block that fails decoding. @@ -646,15 +652,11 @@ def decode_scale( Returns: Decoded object """ - - if scale_bytes == b"\x00": - obj = None + if type_string == "scale_info::0": # Is an AccountId + # Decode AccountId bytes to SS58 address + return ss58_encode(scale_bytes, SS58_FORMAT) else: - if type_string == "scale_info::0": # Is an AccountId - # Decode AccountId bytes to SS58 address - return ss58_encode(scale_bytes, SS58_FORMAT) - else: - obj = decode_by_type_string(type_string, self.registry, scale_bytes) + obj = decode_by_type_string(type_string, self.registry, scale_bytes) if return_scale_obj: return ScaleObj(obj) else: @@ -1681,9 +1683,9 @@ def _make_rpc_request( subscription_added = False ws = self.connect(init=False if attempt == 1 else True) - item_id = 0 for payload in payloads: - item_id += 1 + payload_str = json.dumps(payload["payload"]) + item_id = generate_unique_id(payload_str) ws.send(json.dumps({**payload["payload"], **{"id": item_id}})) request_manager.add_request(item_id, payload["id"]) diff --git a/async_substrate_interface/types.py b/async_substrate_interface/types.py index e192659..6a70db8 100644 --- a/async_substrate_interface/types.py +++ b/async_substrate_interface/types.py @@ -13,6 +13,8 @@ from scalecodec.type_registry import load_type_registry_preset from scalecodec.types import GenericCall, ScaleType +from .utils import json + logger = logging.getLogger("async_substrate_interface") @@ -349,6 +351,9 @@ class SubstrateMixin(ABC): type_registry: Optional[dict] ss58_format: Optional[int] ws_max_size = 2**32 + registry_type_map: dict[str, int] + type_id_to_name: dict[int, str] + metadata_v15 = None @property def chain(self): @@ -604,6 +609,70 @@ def serialize_module_error(module, error, spec_version) -> dict: "spec_version": spec_version, } + def _load_registry_type_map(self): + registry_type_map = {} + type_id_to_name = {} + types = json.loads(self.registry.registry)["types"] + for type_entry in types: + type_type = type_entry["type"] + type_id = type_entry["id"] + type_def = type_type["def"] + type_path = type_type.get("path") + if type_entry.get("params") or type_def.get("variant"): + continue # has generics or is Enum + if type_path: + type_name = type_path[-1] + registry_type_map[type_name] = type_id + type_id_to_name[type_id] = type_name + else: + # probably primitive + if type_def.get("primitive"): + type_name = type_def["primitive"] + registry_type_map[type_name] = type_id + type_id_to_name[type_id] = type_name + for type_entry in types: + type_type = type_entry["type"] + type_id = type_entry["id"] + type_def = type_type["def"] + if type_def.get("sequence"): + sequence_type_id = type_def["sequence"]["type"] + inner_type = type_id_to_name.get(sequence_type_id) + if inner_type: + type_name = f"Vec<{inner_type}>" + type_id_to_name[type_id] = type_name + registry_type_map[type_name] = type_id + elif type_def.get("array"): + array_type_id = type_def["array"]["type"] + inner_type = type_id_to_name.get(array_type_id) + maybe_len = type_def["array"].get("len") + if inner_type: + if maybe_len: + type_name = f"[{inner_type}; {maybe_len}]" + else: + type_name = f"[{inner_type}]" + type_id_to_name[type_id] = type_name + registry_type_map[type_name] = type_id + elif type_def.get("compact"): + compact_type_id = type_def["compact"]["type"] + inner_type = type_id_to_name.get(compact_type_id) + if inner_type: + type_name = f"Compact<{inner_type}>" + type_id_to_name[type_id] = type_name + registry_type_map[type_name] = type_id + elif type_def.get("tuple"): + tuple_type_ids = type_def["tuple"] + type_names = [] + for inner_type_id in tuple_type_ids: + inner_type = type_id_to_name.get(inner_type_id) + if inner_type: + type_names.append(inner_type) + type_name = ", ".join(type_names) + type_name = f"({type_name})" + type_id_to_name[type_id] = type_name + registry_type_map[type_name] = type_id + self.registry_type_map = registry_type_map + self.type_id_to_name = type_id_to_name + def reload_type_registry( self, use_remote_preset: bool = True, auto_discover: bool = True ): @@ -726,12 +795,19 @@ def _encode_scale(self, type_string, value: Any) -> bytes: if value is None: result = b"\x00" else: + try: + vec_acct_id = ( + f"scale_info::{self.registry_type_map['Vec']}" + ) + except KeyError: + vec_acct_id = "scale_info::152" + if type_string == "scale_info::0": # Is an AccountId # encode string into AccountId ## AccountId is a composite type with one, unnamed field return bytes.fromhex(ss58_decode(value, SS58_FORMAT)) - elif type_string == "scale_info::152": # Vec + elif type_string == vec_acct_id: # Vec if not isinstance(value, (list, tuple)): value = [value] diff --git a/async_substrate_interface/utils/__init__.py b/async_substrate_interface/utils/__init__.py index 40e26c5..29b3ced 100644 --- a/async_substrate_interface/utils/__init__.py +++ b/async_substrate_interface/utils/__init__.py @@ -1,4 +1,10 @@ import importlib +import hashlib + + +def generate_unique_id(item: str, length=10): + hashed_value = hashlib.sha256(item.encode()).hexdigest() + return hashed_value[:length] def hex_to_bytes(hex_str: str) -> bytes: diff --git a/pyproject.toml b/pyproject.toml index a70e722..b2547f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "async-substrate-interface" -version = "1.0.1" +version = "1.0.2" description = "Asyncio library for interacting with substrate. Mostly API-compatible with py-substrate-interface" readme = "README.md" license = { file = "LICENSE" }