From b5d7b1aa42980a0fa6d2b5620f950b5d2f91f875 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 4 Mar 2025 23:48:10 +0200 Subject: [PATCH 01/12] [WIP] --- async_substrate_interface/async_substrate.py | 12 +-- async_substrate_interface/sync_substrate.py | 12 +-- async_substrate_interface/utils/cache.py | 91 ++++++++++++++++++++ 3 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 async_substrate_interface/utils/cache.py diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 612670c..4d09b9e 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -21,7 +21,6 @@ TYPE_CHECKING, ) -import asyncstdlib as a from bittensor_wallet.keypair import Keypair from bittensor_wallet.utils import SS58_FORMAT from bt_decode import MetadataV15, PortableRegistry, decode as decode_by_type_string @@ -49,6 +48,7 @@ Preprocessed, ) from async_substrate_interface.utils import hex_to_bytes, json, get_next_id +from async_substrate_interface.utils.cache import async_sql_lru_cache from async_substrate_interface.utils.decoding import ( _determine_if_old_runtime_call, _bt_decode_to_dict_or_list, @@ -1659,7 +1659,7 @@ def convert_event_data(data): events.append(convert_event_data(item)) return events - @a.lru_cache(maxsize=512) # large cache with small items + @async_sql_lru_cache(max_size=512) async def get_parent_block_hash(self, block_hash): block_header = await self.rpc_request("chain_getHeader", [block_hash]) @@ -1672,7 +1672,7 @@ async def get_parent_block_hash(self, block_hash): return block_hash return parent_block_hash - @a.lru_cache(maxsize=16) # small cache with large items + @async_sql_lru_cache(max_size=16) async def get_block_runtime_info(self, block_hash: str) -> dict: """ Retrieve the runtime info of given block_hash @@ -1680,7 +1680,7 @@ async def get_block_runtime_info(self, block_hash: str) -> dict: response = await self.rpc_request("state_getRuntimeVersion", [block_hash]) return response.get("result") - @a.lru_cache(maxsize=512) # large cache with small items + @async_sql_lru_cache(max_size=512) async def get_block_runtime_version_for(self, block_hash: str): """ Retrieve the runtime version of the parent of a given block_hash @@ -1914,7 +1914,7 @@ async def _make_rpc_request( return request_manager.get_results() - @a.lru_cache(maxsize=512) # RPC methods are unlikely to change often + @async_sql_lru_cache(max_size=512) async def supports_rpc_method(self, name: str) -> bool: """ Check if substrate RPC supports given method @@ -1985,7 +1985,7 @@ async def rpc_request( else: raise SubstrateRequestException(result[payload_id][0]) - @a.lru_cache(maxsize=512) # block_id->block_hash does not change + @async_sql_lru_cache(max_size=512) async def get_block_hash(self, block_id: int) -> str: return (await self.rpc_request("chain_getBlockHash", [block_id]))["result"] diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index d327687..edc35d4 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -1,6 +1,5 @@ import logging import random -from functools import lru_cache from hashlib import blake2b from typing import Optional, Union, Callable, Any @@ -31,6 +30,7 @@ ScaleObj, ) from async_substrate_interface.utils import hex_to_bytes, json, get_next_id +from async_substrate_interface.utils.cache import sql_lru_cache from async_substrate_interface.utils.decoding import ( _determine_if_old_runtime_call, _bt_decode_to_dict_or_list, @@ -1406,7 +1406,7 @@ def convert_event_data(data): events.append(convert_event_data(item)) return events - @lru_cache(maxsize=512) # large cache with small items + @sql_lru_cache(max_size=512) def get_parent_block_hash(self, block_hash): block_header = self.rpc_request("chain_getHeader", [block_hash]) @@ -1419,7 +1419,7 @@ def get_parent_block_hash(self, block_hash): return block_hash return parent_block_hash - @lru_cache(maxsize=16) # small cache with large items + @sql_lru_cache(max_size=16) def get_block_runtime_info(self, block_hash: str) -> dict: """ Retrieve the runtime info of given block_hash @@ -1427,7 +1427,7 @@ def get_block_runtime_info(self, block_hash: str) -> dict: response = self.rpc_request("state_getRuntimeVersion", [block_hash]) return response.get("result") - @lru_cache(maxsize=512) # large cache with small items + @sql_lru_cache(max_size=512) def get_block_runtime_version_for(self, block_hash: str): """ Retrieve the runtime version of the parent of a given block_hash @@ -1656,7 +1656,7 @@ def _make_rpc_request( return request_manager.get_results() # TODO change this logic - @lru_cache(maxsize=512) # RPC methods are unlikely to change often + @sql_lru_cache(max_size=512) def supports_rpc_method(self, name: str) -> bool: """ Check if substrate RPC supports given method @@ -1727,7 +1727,7 @@ def rpc_request( else: raise SubstrateRequestException(result[payload_id][0]) - @lru_cache(maxsize=512) # block_id->block_hash does not change + @sql_lru_cache(max_size=512) def get_block_hash(self, block_id: int) -> str: return self.rpc_request("chain_getBlockHash", [block_id])["result"] diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py new file mode 100644 index 0000000..4ab55d8 --- /dev/null +++ b/async_substrate_interface/utils/cache.py @@ -0,0 +1,91 @@ +import functools +import pickle +import sqlite3 +import asyncstdlib as a + + +def _get_table_name(func): + """Convert "ClassName.method_name" to "ClassName_method_name""" + return func.__qualname__.replace(".", "_") + + +def _create_table(conn, table_name): + c = conn.cursor() + c.execute( + f"CREATE TABLE IF NOT EXISTS {table_name} (key BLOB PRIMARY KEY, value BLOB, chain TEXT)" + ) + conn.commit() + + +def _retrieve_from_cache(c, table_name, key, chain): + try: + c.execute( + f"SELECT value FROM {table_name} WHERE key=? AND chain=?", (key, chain) + ) + result = c.fetchone() + if result is not None: + return pickle.loads(result[0]) + except (pickle.PickleError, sqlite3.Error) as e: + print(f"Cache error: {str(e)}") + pass + + +def _insert_into_cache(c, conn, table_name, key, result, chain): + try: + c.execute( + f"INSERT OR REPLACE INTO {table_name} VALUES (?,?,?)", + (key, pickle.dumps(result), chain), + ) + conn.commit() + except (pickle.PickleError, sqlite3.Error) as e: + print(f"Cache error: {str(e)}") + pass + + +def sql_lru_cache(func, max_size=None): + conn = sqlite3.connect("/tmp/cache.db") + + table_name = _get_table_name(func) + _create_table(conn, table_name) + + @functools.lru_cache(maxsize=max_size) + def inner(self, *args, **kwargs): + c = conn.cursor() + key = pickle.dumps((args, kwargs)) + chain = self._chain + + result = _retrieve_from_cache(c, table_name, key, chain) + if result is not None: + return result + + # If not in DB, call func and store in DB + result = func(self, *args, **kwargs) + _insert_into_cache(c, conn, table_name, key, result, chain) + + return result + + return inner + + +def async_sql_lru_cache(func, max_size=None): + conn = sqlite3.connect("/tmp/cache.db") + table_name = _get_table_name(func) + _create_table(conn, table_name) + + @a.lru_cache(maxsize=max_size) + async def inner(self, *args, **kwargs): + c = conn.cursor() + key = pickle.dumps((args, kwargs)) + chain = self._chain + + result = _retrieve_from_cache(c, table_name, key, chain) + if result is not None: + return result + + # If not in DB, call func and store in DB + result = await func(self, *args, **kwargs) + _insert_into_cache(c, conn, table_name, key, result, chain) + + return result + + return inner From 74f0aa7f65a601915e03ea9b5ecf69549a959b39 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Tue, 4 Mar 2025 23:57:50 +0200 Subject: [PATCH 02/12] Cache directory and optional NO_CACHE --- async_substrate_interface/utils/cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index 4ab55d8..e0bf143 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -1,8 +1,11 @@ import functools +import os import pickle import sqlite3 import asyncstdlib as a +CACHE_LOCATION = os.path.expanduser("~/.cache/async-substrate_interface") if os.getenv("NO_CACHE") != "1" else ":memory:" + def _get_table_name(func): """Convert "ClassName.method_name" to "ClassName_method_name""" @@ -43,7 +46,7 @@ def _insert_into_cache(c, conn, table_name, key, result, chain): def sql_lru_cache(func, max_size=None): - conn = sqlite3.connect("/tmp/cache.db") + conn = sqlite3.connect(CACHE_LOCATION) table_name = _get_table_name(func) _create_table(conn, table_name) @@ -68,7 +71,7 @@ def inner(self, *args, **kwargs): def async_sql_lru_cache(func, max_size=None): - conn = sqlite3.connect("/tmp/cache.db") + conn = sqlite3.connect(CACHE_LOCATION) table_name = _get_table_name(func) _create_table(conn, table_name) From 0ee9d9fd6d00aab73b8ee2760260ec72bb44a8ca Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 5 Mar 2025 00:03:10 +0200 Subject: [PATCH 03/12] Store chain by endpoint rather than chain name (i.e. "ws://finney.opentensor.org" vs "Bittensor" --- async_substrate_interface/utils/cache.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index e0bf143..b389f88 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -4,7 +4,11 @@ import sqlite3 import asyncstdlib as a -CACHE_LOCATION = os.path.expanduser("~/.cache/async-substrate_interface") if os.getenv("NO_CACHE") != "1" else ":memory:" +CACHE_LOCATION = ( + os.path.expanduser("~/.cache/async-substrate_interface") + if os.getenv("NO_CACHE") != "1" + else ":memory:" +) def _get_table_name(func): @@ -55,7 +59,7 @@ def sql_lru_cache(func, max_size=None): def inner(self, *args, **kwargs): c = conn.cursor() key = pickle.dumps((args, kwargs)) - chain = self._chain + chain = self.url result = _retrieve_from_cache(c, table_name, key, chain) if result is not None: @@ -79,7 +83,7 @@ def async_sql_lru_cache(func, max_size=None): async def inner(self, *args, **kwargs): c = conn.cursor() key = pickle.dumps((args, kwargs)) - chain = self._chain + chain = self.url result = _retrieve_from_cache(c, table_name, key, chain) if result is not None: From 1ccc34c97d2278556d0752658ab1c32a9c3f5b42 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 5 Mar 2025 00:08:41 +0200 Subject: [PATCH 04/12] Added TODO --- async_substrate_interface/utils/cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index b389f88..dcd2bb8 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -10,6 +10,8 @@ else ":memory:" ) +# TODO do not cache for localnets + def _get_table_name(func): """Convert "ClassName.method_name" to "ClassName_method_name""" From a98ef4c6433da503c1ade9d046fc55a42d9b9f94 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 5 Mar 2025 00:10:39 +0200 Subject: [PATCH 05/12] Don't recreate cursor obj --- async_substrate_interface/utils/cache.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index dcd2bb8..55e2368 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -18,8 +18,7 @@ def _get_table_name(func): return func.__qualname__.replace(".", "_") -def _create_table(conn, table_name): - c = conn.cursor() +def _create_table(c, conn, table_name): c.execute( f"CREATE TABLE IF NOT EXISTS {table_name} (key BLOB PRIMARY KEY, value BLOB, chain TEXT)" ) @@ -53,9 +52,9 @@ def _insert_into_cache(c, conn, table_name, key, result, chain): def sql_lru_cache(func, max_size=None): conn = sqlite3.connect(CACHE_LOCATION) - + c = conn.cursor() table_name = _get_table_name(func) - _create_table(conn, table_name) + _create_table(c, conn, table_name) @functools.lru_cache(maxsize=max_size) def inner(self, *args, **kwargs): @@ -78,8 +77,9 @@ def inner(self, *args, **kwargs): def async_sql_lru_cache(func, max_size=None): conn = sqlite3.connect(CACHE_LOCATION) + c = conn.cursor() table_name = _get_table_name(func) - _create_table(conn, table_name) + _create_table(c, conn, table_name) @a.lru_cache(maxsize=max_size) async def inner(self, *args, **kwargs): From 2b3bec4d410dbb78db1dad9c07e6d6ff8430b274 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 5 Mar 2025 00:30:11 +0200 Subject: [PATCH 06/12] Don't cache local chain --- async_substrate_interface/utils/cache.py | 29 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index 55e2368..536fb79 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -4,9 +4,10 @@ import sqlite3 import asyncstdlib as a +USE_CACHE = True if os.getenv("NO_CACHE") != "1" else False CACHE_LOCATION = ( os.path.expanduser("~/.cache/async-substrate_interface") - if os.getenv("NO_CACHE") != "1" + if USE_CACHE else ":memory:" ) @@ -18,6 +19,10 @@ def _get_table_name(func): return func.__qualname__.replace(".", "_") +def _check_if_local(chain: str) -> bool: + return any([x in chain for x in ["127.0.0.1", "localhost", "0.0.0.0"]]) + + def _create_table(c, conn, table_name): c.execute( f"CREATE TABLE IF NOT EXISTS {table_name} (key BLOB PRIMARY KEY, value BLOB, chain TEXT)" @@ -61,14 +66,16 @@ def inner(self, *args, **kwargs): c = conn.cursor() key = pickle.dumps((args, kwargs)) chain = self.url - - result = _retrieve_from_cache(c, table_name, key, chain) - if result is not None: - return result + if not (local_chain := _check_if_local(chain)) or not USE_CACHE: + result = _retrieve_from_cache(c, table_name, key, chain) + if result is not None: + return result # If not in DB, call func and store in DB result = func(self, *args, **kwargs) - _insert_into_cache(c, conn, table_name, key, result, chain) + + if not local_chain or not USE_CACHE: + _insert_into_cache(c, conn, table_name, key, result, chain) return result @@ -87,13 +94,15 @@ async def inner(self, *args, **kwargs): key = pickle.dumps((args, kwargs)) chain = self.url - result = _retrieve_from_cache(c, table_name, key, chain) - if result is not None: - return result + if not (local_chain := _check_if_local(chain)) or not USE_CACHE: + result = _retrieve_from_cache(c, table_name, key, chain) + if result is not None: + return result # If not in DB, call func and store in DB result = await func(self, *args, **kwargs) - _insert_into_cache(c, conn, table_name, key, result, chain) + if not local_chain or not USE_CACHE: + _insert_into_cache(c, conn, table_name, key, result, chain) return result From 2944e481983f190b1e5a6e5c353e5fcb519de08d Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 5 Mar 2025 00:36:04 +0200 Subject: [PATCH 07/12] Formatting --- async_substrate_interface/utils/cache.py | 84 +++++++++++++----------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index 536fb79..5dc12d0 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -55,55 +55,61 @@ def _insert_into_cache(c, conn, table_name, key, result, chain): pass -def sql_lru_cache(func, max_size=None): - conn = sqlite3.connect(CACHE_LOCATION) - c = conn.cursor() - table_name = _get_table_name(func) - _create_table(c, conn, table_name) - - @functools.lru_cache(maxsize=max_size) - def inner(self, *args, **kwargs): +def sql_lru_cache(max_size=None): + def decorator(func): + conn = sqlite3.connect(CACHE_LOCATION) c = conn.cursor() - key = pickle.dumps((args, kwargs)) - chain = self.url - if not (local_chain := _check_if_local(chain)) or not USE_CACHE: - result = _retrieve_from_cache(c, table_name, key, chain) - if result is not None: - return result + table_name = _get_table_name(func) + _create_table(c, conn, table_name) - # If not in DB, call func and store in DB - result = func(self, *args, **kwargs) + @functools.lru_cache(maxsize=max_size) + def inner(self, *args, **kwargs): + c = conn.cursor() + key = pickle.dumps((args, kwargs)) + chain = self.url + if not (local_chain := _check_if_local(chain)) or not USE_CACHE: + result = _retrieve_from_cache(c, table_name, key, chain) + if result is not None: + return result - if not local_chain or not USE_CACHE: - _insert_into_cache(c, conn, table_name, key, result, chain) + # If not in DB, call func and store in DB + result = func(self, *args, **kwargs) - return result + if not local_chain or not USE_CACHE: + _insert_into_cache(c, conn, table_name, key, result, chain) - return inner + return result + return inner -def async_sql_lru_cache(func, max_size=None): - conn = sqlite3.connect(CACHE_LOCATION) - c = conn.cursor() - table_name = _get_table_name(func) - _create_table(c, conn, table_name) + return decorator - @a.lru_cache(maxsize=max_size) - async def inner(self, *args, **kwargs): + +def async_sql_lru_cache(max_size=None): + def decorator(func): + conn = sqlite3.connect(CACHE_LOCATION) c = conn.cursor() - key = pickle.dumps((args, kwargs)) - chain = self.url + table_name = _get_table_name(func) + _create_table(c, conn, table_name) + + @a.lru_cache(maxsize=max_size) + async def inner(self, *args, **kwargs): + c = conn.cursor() + key = pickle.dumps((args, kwargs)) + chain = self.url + + if not (local_chain := _check_if_local(chain)) or not USE_CACHE: + result = _retrieve_from_cache(c, table_name, key, chain) + if result is not None: + return result - if not (local_chain := _check_if_local(chain)) or not USE_CACHE: - result = _retrieve_from_cache(c, table_name, key, chain) - if result is not None: - return result + # If not in DB, call func and store in DB + result = await func(self, *args, **kwargs) + if not local_chain or not USE_CACHE: + _insert_into_cache(c, conn, table_name, key, result, chain) - # If not in DB, call func and store in DB - result = await func(self, *args, **kwargs) - if not local_chain or not USE_CACHE: - _insert_into_cache(c, conn, table_name, key, result, chain) + return result - return result + return inner - return inner + return decorator From 98ea830a6bfb5ba221cd2708bf7415bba7770a72 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Wed, 5 Mar 2025 00:36:49 +0200 Subject: [PATCH 08/12] Remove TODO --- async_substrate_interface/utils/cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index 5dc12d0..1729774 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -11,8 +11,6 @@ else ":memory:" ) -# TODO do not cache for localnets - def _get_table_name(func): """Convert "ClassName.method_name" to "ClassName_method_name""" From 9722bc4b8e9ef3b743e42cfbc61459fe6b14d42a Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 6 Mar 2025 21:09:03 +0200 Subject: [PATCH 09/12] Remove sql_lru_cache from sync substrate and async substrate, and instead only use it in an experimental new class for just async substrate. --- async_substrate_interface/async_substrate.py | 44 +++++++++++++++++--- async_substrate_interface/sync_substrate.py | 13 +++--- async_substrate_interface/utils/cache.py | 8 ++-- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 4d09b9e..502b743 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -21,6 +21,7 @@ TYPE_CHECKING, ) +import asyncstdlib as a from bittensor_wallet.keypair import Keypair from bittensor_wallet.utils import SS58_FORMAT from bt_decode import MetadataV15, PortableRegistry, decode as decode_by_type_string @@ -1659,8 +1660,11 @@ def convert_event_data(data): events.append(convert_event_data(item)) return events - @async_sql_lru_cache(max_size=512) + @a.lru_cache(maxsize=512) async def get_parent_block_hash(self, block_hash): + return await self._get_parent_block_hash(block_hash) + + async def _get_parent_block_hash(self, block_hash): block_header = await self.rpc_request("chain_getHeader", [block_hash]) if block_header["result"] is None: @@ -1672,16 +1676,22 @@ async def get_parent_block_hash(self, block_hash): return block_hash return parent_block_hash - @async_sql_lru_cache(max_size=16) + @a.lru_cache(maxsize=16) async def get_block_runtime_info(self, block_hash: str) -> dict: + return await self._get_block_runtime_info(block_hash) + + async def _get_block_runtime_info(self, block_hash: str) -> dict: """ Retrieve the runtime info of given block_hash """ response = await self.rpc_request("state_getRuntimeVersion", [block_hash]) return response.get("result") - @async_sql_lru_cache(max_size=512) + @a.lru_cache(maxsize=512) async def get_block_runtime_version_for(self, block_hash: str): + return await self._get_block_runtime_version_for(block_hash) + + async def _get_block_runtime_version_for(self, block_hash: str): """ Retrieve the runtime version of the parent of a given block_hash """ @@ -1914,7 +1924,6 @@ async def _make_rpc_request( return request_manager.get_results() - @async_sql_lru_cache(max_size=512) async def supports_rpc_method(self, name: str) -> bool: """ Check if substrate RPC supports given method @@ -1985,8 +1994,11 @@ async def rpc_request( else: raise SubstrateRequestException(result[payload_id][0]) - @async_sql_lru_cache(max_size=512) + @a.lru_cache(maxsize=512) async def get_block_hash(self, block_id: int) -> str: + return await self._get_block_hash(block_id) + + async def _get_block_hash(self, block_id: int) -> str: return (await self.rpc_request("chain_getBlockHash", [block_id]))["result"] async def get_chain_head(self) -> str: @@ -3230,6 +3242,28 @@ async def _handler(block_data: dict[str, Any]): return await co +class DiskCachedAsyncSubstrateInterface(AsyncSubstrateInterface): + """ + Experimental new class that uses disk-caching in addition to memory-caching for the cached methods + """ + + @async_sql_lru_cache(maxsize=512) + async def get_parent_block_hash(self, block_hash): + return await self._get_parent_block_hash(block_hash) + + @async_sql_lru_cache(maxsize=16) + async def get_block_runtime_info(self, block_hash: str) -> dict: + return await self._get_block_runtime_info(block_hash) + + @async_sql_lru_cache(maxsize=512) + async def get_block_runtime_version_for(self, block_hash: str): + return await self._get_block_runtime_version_for(block_hash) + + @async_sql_lru_cache(maxsize=512) + async def get_block_hash(self, block_id: int) -> str: + return await self._get_block_hash(block_id) + + async def get_async_substrate_interface( url: str, use_remote_preset: bool = False, diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index edc35d4..daad7ce 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -1,3 +1,4 @@ +import functools import logging import random from hashlib import blake2b @@ -30,7 +31,6 @@ ScaleObj, ) from async_substrate_interface.utils import hex_to_bytes, json, get_next_id -from async_substrate_interface.utils.cache import sql_lru_cache from async_substrate_interface.utils.decoding import ( _determine_if_old_runtime_call, _bt_decode_to_dict_or_list, @@ -1406,7 +1406,7 @@ def convert_event_data(data): events.append(convert_event_data(item)) return events - @sql_lru_cache(max_size=512) + @functools.lru_cache(maxsize=512) def get_parent_block_hash(self, block_hash): block_header = self.rpc_request("chain_getHeader", [block_hash]) @@ -1419,7 +1419,7 @@ def get_parent_block_hash(self, block_hash): return block_hash return parent_block_hash - @sql_lru_cache(max_size=16) + @functools.lru_cache(maxsize=16) def get_block_runtime_info(self, block_hash: str) -> dict: """ Retrieve the runtime info of given block_hash @@ -1427,7 +1427,7 @@ def get_block_runtime_info(self, block_hash: str) -> dict: response = self.rpc_request("state_getRuntimeVersion", [block_hash]) return response.get("result") - @sql_lru_cache(max_size=512) + @functools.lru_cache(maxsize=512) def get_block_runtime_version_for(self, block_hash: str): """ Retrieve the runtime version of the parent of a given block_hash @@ -1655,8 +1655,7 @@ def _make_rpc_request( return request_manager.get_results() - # TODO change this logic - @sql_lru_cache(max_size=512) + @functools.lru_cache(maxsize=512) def supports_rpc_method(self, name: str) -> bool: """ Check if substrate RPC supports given method @@ -1727,7 +1726,7 @@ def rpc_request( else: raise SubstrateRequestException(result[payload_id][0]) - @sql_lru_cache(max_size=512) + @functools.lru_cache(maxsize=512) def get_block_hash(self, block_id: int) -> str: return self.rpc_request("chain_getBlockHash", [block_id])["result"] diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index 1729774..554f1d9 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -53,14 +53,14 @@ def _insert_into_cache(c, conn, table_name, key, result, chain): pass -def sql_lru_cache(max_size=None): +def sql_lru_cache(maxsize=None): def decorator(func): conn = sqlite3.connect(CACHE_LOCATION) c = conn.cursor() table_name = _get_table_name(func) _create_table(c, conn, table_name) - @functools.lru_cache(maxsize=max_size) + @functools.lru_cache(maxsize=maxsize) def inner(self, *args, **kwargs): c = conn.cursor() key = pickle.dumps((args, kwargs)) @@ -83,14 +83,14 @@ def inner(self, *args, **kwargs): return decorator -def async_sql_lru_cache(max_size=None): +def async_sql_lru_cache(maxsize=None): def decorator(func): conn = sqlite3.connect(CACHE_LOCATION) c = conn.cursor() table_name = _get_table_name(func) _create_table(c, conn, table_name) - @a.lru_cache(maxsize=max_size) + @a.lru_cache(maxsize=maxsize) async def inner(self, *args, **kwargs): c = conn.cursor() key = pickle.dumps((args, kwargs)) From 375d78f85115b6d56713fc3bc4285978e9b4b906 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 6 Mar 2025 21:12:26 +0200 Subject: [PATCH 10/12] Add optional CACHE_LOCATION env var --- async_substrate_interface/utils/cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index 554f1d9..e634475 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -6,7 +6,9 @@ USE_CACHE = True if os.getenv("NO_CACHE") != "1" else False CACHE_LOCATION = ( - os.path.expanduser("~/.cache/async-substrate_interface") + os.path.expanduser( + os.getenv("CACHE_LOCATION", "~/.cache/async-substrate_interface") + ) if USE_CACHE else ":memory:" ) From a760351b8afe8f6ca8ce6d1911331ca1a72a25b0 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 6 Mar 2025 21:33:05 +0200 Subject: [PATCH 11/12] Fixes path name --- async_substrate_interface/utils/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index e634475..9a01b51 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -7,7 +7,7 @@ USE_CACHE = True if os.getenv("NO_CACHE") != "1" else False CACHE_LOCATION = ( os.path.expanduser( - os.getenv("CACHE_LOCATION", "~/.cache/async-substrate_interface") + os.getenv("CACHE_LOCATION", "~/.cache/async-substrate-interface") ) if USE_CACHE else ":memory:" From b90d49184539c23154a011ccb7c0dfaf9447f90a Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Thu, 6 Mar 2025 21:33:36 +0200 Subject: [PATCH 12/12] Adds trigger to automatically delete rows after 500 --- async_substrate_interface/utils/cache.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/async_substrate_interface/utils/cache.py b/async_substrate_interface/utils/cache.py index 9a01b51..ab4f457 100644 --- a/async_substrate_interface/utils/cache.py +++ b/async_substrate_interface/utils/cache.py @@ -25,7 +25,26 @@ def _check_if_local(chain: str) -> bool: def _create_table(c, conn, table_name): c.execute( - f"CREATE TABLE IF NOT EXISTS {table_name} (key BLOB PRIMARY KEY, value BLOB, chain TEXT)" + f"""CREATE TABLE IF NOT EXISTS {table_name} + ( + rowid INTEGER PRIMARY KEY AUTOINCREMENT, + key BLOB, + value BLOB, + chain TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """ + ) + c.execute( + f"""CREATE TRIGGER IF NOT EXISTS prune_rows_trigger AFTER INSERT ON {table_name} + BEGIN + DELETE FROM {table_name} + WHERE rowid IN ( + SELECT rowid FROM {table_name} + ORDER BY created_at DESC + LIMIT -1 OFFSET 500 + ); + END;""" ) conn.commit() @@ -46,7 +65,7 @@ def _retrieve_from_cache(c, table_name, key, chain): def _insert_into_cache(c, conn, table_name, key, result, chain): try: c.execute( - f"INSERT OR REPLACE INTO {table_name} VALUES (?,?,?)", + f"INSERT OR REPLACE INTO {table_name} (key, value, chain) VALUES (?,?,?)", (key, pickle.dumps(result), chain), ) conn.commit()