From a84e36def9ba53ef05d51515a7014c2ea158be0a Mon Sep 17 00:00:00 2001 From: moose-code Date: Thu, 13 Mar 2025 08:53:25 +0000 Subject: [PATCH 1/7] ALternative wording for rpc fallback --- docs/HyperIndex/Advanced/hypersync.md | 14 ++++++++++---- docs/HyperIndex/supported-networks/hyperliquid.md | 2 +- docs/HyperSync/HyperRPC/hyperrpc-url-endpoints.md | 1 - docs/HyperSync/hypersync-supported-networks.md | 3 +-- supported-networks.json | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/HyperIndex/Advanced/hypersync.md b/docs/HyperIndex/Advanced/hypersync.md index 241de8d7..a997db6b 100644 --- a/docs/HyperIndex/Advanced/hypersync.md +++ b/docs/HyperIndex/Advanced/hypersync.md @@ -75,11 +75,17 @@ This feature eliminates the need to manually determine the deployment block of y HyperSync is maintained and hosted by Envio for all supported networks. We handle the infrastructure, allowing you to focus on building your indexer logic. -### Enhance reliability with the RPC data source +### Improving resilience with RPC fallback -HyperIndex allows you to provide RPC providers specifically for redundancy and failover. This is **recommended** to ensure your indexer has 100% uptime. If something happens with HyperSync, your indexer will automatically switch to RPC. +HyperIndex supports a multi-layered data source architecture through RPC fallback capability. While HyperSync provides excellent performance and reliability, implementing redundant data sources follows best practices for mission-critical applications. -Add it by providing an `rpc` field in your network configuration. It can be a URL or a list of RPC configuration objects: +Adding an RPC fallback provides: + +- **High availability architecture**: Ensures continuous indexing even during unplanned HyperSync outages +- **Multi-provider resilience**: Creates a fault-tolerant system that can gracefully handle any network disruptions +- **Complete control**: Gives you additional options for how your indexer accesses blockchain data + +Configure it by providing an `rpc` field in your network configuration: ```diff name: Greeter @@ -107,7 +113,7 @@ networks: ``` :::info -This feature is available starting from version `2.14.0`. The fallback RPC is used when a primary data source doesn't get a new block for more than 20 seconds. +This feature is available starting from version `2.14.0`. The fallback RPC is activated only when a primary data source doesn't receive a new block for more than 20 seconds. ::: ### Supported Networks diff --git a/docs/HyperIndex/supported-networks/hyperliquid.md b/docs/HyperIndex/supported-networks/hyperliquid.md index d1461a05..0d8066be 100644 --- a/docs/HyperIndex/supported-networks/hyperliquid.md +++ b/docs/HyperIndex/supported-networks/hyperliquid.md @@ -19,7 +19,7 @@ slug: /hyperliquid ### Tier -BRONSE πŸ—οΈ +BRONZE πŸ₯‰ ### Overview diff --git a/docs/HyperSync/HyperRPC/hyperrpc-url-endpoints.md b/docs/HyperSync/HyperRPC/hyperrpc-url-endpoints.md index 043190d8..1e12ddde 100644 --- a/docs/HyperSync/HyperRPC/hyperrpc-url-endpoints.md +++ b/docs/HyperSync/HyperRPC/hyperrpc-url-endpoints.md @@ -48,7 +48,6 @@ Here is a table of the currently supported networks on HyperRPC and their respec | Holesky | 17000 | https://holesky.rpc.hypersync.xyz or https://17000.rpc.hypersync.xyz | | | Holesky Token Test | 17000 | https://holesky-token-test.rpc.hypersync.xyz or https://17000.rpc.hypersync.xyz | | | Hyperliquid | 645749 | https://hyperliquid.rpc.hypersync.xyz or https://645749.rpc.hypersync.xyz | | -| Hyperliquid Temp | 645749 | https://hyperliquid-temp.rpc.hypersync.xyz or https://645749.rpc.hypersync.xyz | | | Ink | 57073 | https://ink.rpc.hypersync.xyz or https://57073.rpc.hypersync.xyz | | | Internal Test Chain | 16858666 | https://internal-test-chain.rpc.hypersync.xyz or https://16858666.rpc.hypersync.xyz | | | Kroma | 255 | https://kroma.rpc.hypersync.xyz or https://255.rpc.hypersync.xyz | | diff --git a/docs/HyperSync/hypersync-supported-networks.md b/docs/HyperSync/hypersync-supported-networks.md index f0bbfac7..15e165ea 100644 --- a/docs/HyperSync/hypersync-supported-networks.md +++ b/docs/HyperSync/hypersync-supported-networks.md @@ -49,8 +49,7 @@ The Tier is the level of support (and therefore reliability) based on the infras | Harmony Shard 0 | 1666600000 | https://harmony-shard-0.hypersync.xyz or https://1666600000.hypersync.xyz | πŸ₯ˆ | | | Holesky | 17000 | https://holesky.hypersync.xyz or https://17000.hypersync.xyz | πŸŽ’ | | | Holesky Token Test | 17000 | https://holesky-token-test.hypersync.xyz or https://17000.hypersync.xyz | πŸ”’ | | -| Hyperliquid | 645749 | https://hyperliquid.hypersync.xyz or https://645749.hypersync.xyz | πŸ—οΈ | | -| Hyperliquid Temp | 645749 | https://hyperliquid-temp.hypersync.xyz or https://645749.hypersync.xyz | πŸ—οΈ | | +| Hyperliquid | 645749 | https://hyperliquid.hypersync.xyz or https://645749.hypersync.xyz | πŸ₯‰ | | | Ink | 57073 | https://ink.hypersync.xyz or https://57073.hypersync.xyz | πŸͺ¨ | | | Internal Test Chain | 16858666 | https://internal-test-chain.hypersync.xyz or https://16858666.hypersync.xyz | πŸ”’ | | | Kroma | 255 | https://kroma.hypersync.xyz or https://255.hypersync.xyz | πŸͺ¨ | | diff --git a/supported-networks.json b/supported-networks.json index e74664b2..0182c6cb 100644 --- a/supported-networks.json +++ b/supported-networks.json @@ -2,4 +2,4 @@ "supported-networks/any-evm-with-rpc", "supported-networks/local-anvil", "supported-networks/local-hardhat", - "supported-networks/0g-newton-testnet","supported-networks/aleph-zero-evm","supported-networks/altlayer-op-demo-testnet","supported-networks/ancient8","supported-networks/arbitrum","supported-networks/arbitrum-blueberry","supported-networks/arbitrum-nova","supported-networks/arbitrum-sepolia","supported-networks/artela-testnet","supported-networks/arthera-mainnet","supported-networks/asset-chain-mainnet","supported-networks/astar-zkevm","supported-networks/astar-zkyoto","supported-networks/aurora","supported-networks/avalanche","supported-networks/b2-hub-testnet","supported-networks/b3","supported-networks/b3-sepolia-testnet","supported-networks/base","supported-networks/base-sepolia","supported-networks/beam","supported-networks/berachain","supported-networks/berachain-artio-testnet","supported-networks/berachain-bartio","supported-networks/bevm-mainnet","supported-networks/bevm-testnet","supported-networks/bitfinity-mainnet","supported-networks/bitfinity-testnet","supported-networks/bitgert-mainnet","supported-networks/bitlayer","supported-networks/blast","supported-networks/blast-sepolia","supported-networks/bnb-smart-chain","supported-networks/bnb-smart-chain-testnet","supported-networks/bob-mainnet","supported-networks/boba","supported-networks/boba-bnb-mainnet","supported-networks/botanix-testnet","supported-networks/bsc","supported-networks/bsc-testnet","supported-networks/canto","supported-networks/canto-testnet","supported-networks/celo","supported-networks/celo-alfajores-testnet","supported-networks/chiliz","supported-networks/chiliz-testnet-spicy","supported-networks/citrea-devnet","supported-networks/citrea-testnet","supported-networks/core","supported-networks/crab","supported-networks/creator-testnet","supported-networks/cronos-zkevm","supported-networks/cronos-zkevm-testnet","supported-networks/crossfi-mainnet","supported-networks/crossfi-mainnet","supported-networks/crossfi-testnet","supported-networks/cyber","supported-networks/darwinia","supported-networks/degen-chain","supported-networks/dfk-chain","supported-networks/dogechain-mainnet","supported-networks/dogechain-testnet","supported-networks/dos-chain","supported-networks/energy-web","supported-networks/eos","supported-networks/eth","supported-networks/etherlink-testnet","supported-networks/exosama","supported-networks/extrabud","supported-networks/fantom","supported-networks/fantom-testnet","supported-networks/flare","supported-networks/flare-songbird","supported-networks/flow","supported-networks/flow-testnet","supported-networks/fraxtal","supported-networks/fuel-mainnet","supported-networks/fuel-testnet","supported-networks/fuji","supported-networks/galadriel-devnet","supported-networks/gnosis","supported-networks/gnosis-chiado","supported-networks/gnosis-traces","supported-networks/gravity-alpha-mainnet","supported-networks/harmony-shard-0","supported-networks/heco-chain","supported-networks/holesky","supported-networks/holesky-token-test","supported-networks/hyperliquid","supported-networks/hyperliquid-temp","supported-networks/immutable-zkevm","supported-networks/immutable-zkevm-testnet","supported-networks/ink","supported-networks/internal-test-chain","supported-networks/iotex-network","supported-networks/japan-open-chain","supported-networks/kaia","supported-networks/kakarot-starknet-sepolia","supported-networks/kroma","supported-networks/layeredge-testnet","supported-networks/lightlink-pegasus-testnet","supported-networks/lightlink-phoenix","supported-networks/linea","supported-networks/lisk","supported-networks/lukso","supported-networks/lukso-testnet","supported-networks/manta","supported-networks/manta-pacific-sepolia","supported-networks/mantle","supported-networks/merlin","supported-networks/metall2","supported-networks/meter-mainnet","supported-networks/meter-testnet","supported-networks/metis","supported-networks/mev-commit","supported-networks/mint-mainnet","supported-networks/mode","supported-networks/monad-testnet","supported-networks/moonbase-alpha","supported-networks/moonbeam","supported-networks/morph","supported-networks/morph-holesky","supported-networks/nautilus","supported-networks/neo-x-testnet","supported-networks/nibiru-testnet","supported-networks/now-chaint","supported-networks/oasis-emerald","supported-networks/oasis-sapphire","supported-networks/onigiri-subnet","supported-networks/onigiri-test-subnet","supported-networks/ontology-mainnet","supported-networks/ontology-testnet","supported-networks/op-celestia-raspberry","supported-networks/opbnb","supported-networks/optimism","supported-networks/optimism-sepolia","supported-networks/optopia","supported-networks/peaq","supported-networks/polygon","supported-networks/polygon-amoy","supported-networks/polygon-zkevm","supported-networks/polygon-zkevm-cardona-testnet","supported-networks/public-goods-network","supported-networks/pulsechain","supported-networks/puppynet-shibarium","supported-networks/ronin","supported-networks/rootstock","supported-networks/saakuru","supported-networks/satoshivm","supported-networks/scroll","supported-networks/scroll-sepolia","supported-networks/sepolia","supported-networks/shibarium","supported-networks/shimmer-evm","supported-networks/skale-europa","supported-networks/soneium","supported-networks/soneium","supported-networks/sophon","supported-networks/sophon-testnet","supported-networks/stratovm-testnet","supported-networks/stratovm-testnet","supported-networks/superseed-sepolia-testnet","supported-networks/taiko","supported-networks/tanssi-demo","supported-networks/telos-evm-mainnet","supported-networks/telos-evm-testnet","supported-networks/torus-mainnet","supported-networks/torus-testnet","supported-networks/unichain","supported-networks/unichain-sepolia","supported-networks/unicorn-ultra-nebulas-testnet","supported-networks/velas-mainnet","supported-networks/viction","supported-networks/x-layer-mainnet","supported-networks/x-layer-testnet","supported-networks/xdc","supported-networks/xdc-apothem-testnet","supported-networks/xdc-network","supported-networks/xdc-testnet","supported-networks/zeta","supported-networks/zeta-testnet","supported-networks/zircuit","supported-networks/zklink-nova-mainnet","supported-networks/zksync","supported-networks/zksync-sepolia-testnet","supported-networks/zora","supported-networks/zora-sepolia"]} \ No newline at end of file + "supported-networks/0g-newton-testnet","supported-networks/aleph-zero-evm","supported-networks/altlayer-op-demo-testnet","supported-networks/ancient8","supported-networks/arbitrum","supported-networks/arbitrum-blueberry","supported-networks/arbitrum-nova","supported-networks/arbitrum-sepolia","supported-networks/artela-testnet","supported-networks/arthera-mainnet","supported-networks/asset-chain-mainnet","supported-networks/astar-zkevm","supported-networks/astar-zkyoto","supported-networks/aurora","supported-networks/avalanche","supported-networks/b2-hub-testnet","supported-networks/b3","supported-networks/b3-sepolia-testnet","supported-networks/base","supported-networks/base-sepolia","supported-networks/beam","supported-networks/berachain","supported-networks/berachain-artio-testnet","supported-networks/berachain-bartio","supported-networks/bevm-mainnet","supported-networks/bevm-testnet","supported-networks/bitfinity-mainnet","supported-networks/bitfinity-testnet","supported-networks/bitgert-mainnet","supported-networks/bitlayer","supported-networks/blast","supported-networks/blast-sepolia","supported-networks/bnb-smart-chain","supported-networks/bnb-smart-chain-testnet","supported-networks/bob-mainnet","supported-networks/boba","supported-networks/boba-bnb-mainnet","supported-networks/botanix-testnet","supported-networks/bsc","supported-networks/bsc-testnet","supported-networks/canto","supported-networks/canto-testnet","supported-networks/celo","supported-networks/celo-alfajores-testnet","supported-networks/chiliz","supported-networks/chiliz-testnet-spicy","supported-networks/citrea-devnet","supported-networks/citrea-testnet","supported-networks/core","supported-networks/crab","supported-networks/creator-testnet","supported-networks/cronos-zkevm","supported-networks/cronos-zkevm-testnet","supported-networks/crossfi-mainnet","supported-networks/crossfi-mainnet","supported-networks/crossfi-testnet","supported-networks/cyber","supported-networks/darwinia","supported-networks/degen-chain","supported-networks/dfk-chain","supported-networks/dogechain-mainnet","supported-networks/dogechain-testnet","supported-networks/dos-chain","supported-networks/energy-web","supported-networks/eos","supported-networks/eth","supported-networks/etherlink-testnet","supported-networks/exosama","supported-networks/extrabud","supported-networks/fantom","supported-networks/fantom-testnet","supported-networks/flare","supported-networks/flare-songbird","supported-networks/flow","supported-networks/flow-testnet","supported-networks/fraxtal","supported-networks/fuel-mainnet","supported-networks/fuel-testnet","supported-networks/fuji","supported-networks/galadriel-devnet","supported-networks/gnosis","supported-networks/gnosis-chiado","supported-networks/gnosis-traces","supported-networks/gravity-alpha-mainnet","supported-networks/harmony-shard-0","supported-networks/heco-chain","supported-networks/holesky","supported-networks/holesky-token-test","supported-networks/hyperliquid","supported-networks/immutable-zkevm","supported-networks/immutable-zkevm-testnet","supported-networks/ink","supported-networks/internal-test-chain","supported-networks/iotex-network","supported-networks/japan-open-chain","supported-networks/kaia","supported-networks/kakarot-starknet-sepolia","supported-networks/kroma","supported-networks/layeredge-testnet","supported-networks/lightlink-pegasus-testnet","supported-networks/lightlink-phoenix","supported-networks/linea","supported-networks/lisk","supported-networks/lukso","supported-networks/lukso-testnet","supported-networks/manta","supported-networks/manta-pacific-sepolia","supported-networks/mantle","supported-networks/merlin","supported-networks/metall2","supported-networks/meter-mainnet","supported-networks/meter-testnet","supported-networks/metis","supported-networks/mev-commit","supported-networks/mint-mainnet","supported-networks/mode","supported-networks/monad-testnet","supported-networks/moonbase-alpha","supported-networks/moonbeam","supported-networks/morph","supported-networks/morph-holesky","supported-networks/nautilus","supported-networks/neo-x-testnet","supported-networks/nibiru-testnet","supported-networks/now-chaint","supported-networks/oasis-emerald","supported-networks/oasis-sapphire","supported-networks/onigiri-subnet","supported-networks/onigiri-test-subnet","supported-networks/ontology-mainnet","supported-networks/ontology-testnet","supported-networks/op-celestia-raspberry","supported-networks/opbnb","supported-networks/optimism","supported-networks/optimism-sepolia","supported-networks/optopia","supported-networks/peaq","supported-networks/polygon","supported-networks/polygon-amoy","supported-networks/polygon-zkevm","supported-networks/polygon-zkevm-cardona-testnet","supported-networks/public-goods-network","supported-networks/pulsechain","supported-networks/puppynet-shibarium","supported-networks/ronin","supported-networks/rootstock","supported-networks/saakuru","supported-networks/satoshivm","supported-networks/scroll","supported-networks/scroll-sepolia","supported-networks/sepolia","supported-networks/shibarium","supported-networks/shimmer-evm","supported-networks/skale-europa","supported-networks/soneium","supported-networks/soneium","supported-networks/sophon","supported-networks/sophon-testnet","supported-networks/stratovm-testnet","supported-networks/stratovm-testnet","supported-networks/superseed-sepolia-testnet","supported-networks/taiko","supported-networks/tanssi-demo","supported-networks/telos-evm-mainnet","supported-networks/telos-evm-testnet","supported-networks/torus-mainnet","supported-networks/torus-testnet","supported-networks/unichain","supported-networks/unichain-sepolia","supported-networks/unicorn-ultra-nebulas-testnet","supported-networks/velas-mainnet","supported-networks/viction","supported-networks/x-layer-mainnet","supported-networks/x-layer-testnet","supported-networks/xdc","supported-networks/xdc-apothem-testnet","supported-networks/xdc-network","supported-networks/xdc-testnet","supported-networks/zeta","supported-networks/zeta-testnet","supported-networks/zircuit","supported-networks/zklink-nova-mainnet","supported-networks/zksync","supported-networks/zksync-sepolia-testnet","supported-networks/zora","supported-networks/zora-sepolia"]} \ No newline at end of file From 6a9f705912820c09096d6749e45809fb855268bf Mon Sep 17 00:00:00 2001 From: moose-code Date: Thu, 13 Mar 2025 10:06:11 +0000 Subject: [PATCH 2/7] Improve hypersync documentation --- docs/HyperSync/HyperRPC/overview-hyperrpc.md | 108 ++- docs/HyperSync/api-tokens.mdx | 62 +- docs/HyperSync/hypersync-clients.md | 187 +++- docs/HyperSync/hypersync-curl-examples.md | 130 ++- docs/HyperSync/hypersync-query.md | 835 ++++++++---------- .../HyperSync/hypersync-supported-networks.md | 16 +- docs/HyperSync/hypersync-usage.md | 275 +++++- docs/HyperSync/overview.md | 57 -- sidebarsHyperSync.js | 18 +- 9 files changed, 1043 insertions(+), 645 deletions(-) diff --git a/docs/HyperSync/HyperRPC/overview-hyperrpc.md b/docs/HyperSync/HyperRPC/overview-hyperrpc.md index b47cadcb..dab9dc7f 100644 --- a/docs/HyperSync/HyperRPC/overview-hyperrpc.md +++ b/docs/HyperSync/HyperRPC/overview-hyperrpc.md @@ -5,34 +5,98 @@ sidebar_label: Overview slug: /overview-hyperrpc --- -HyperRPC is an extremely fast read-only RPC for data-intensive tasks. It is designed to be a drop-in solution with a mind-boggling performance boost. +# HyperRPC: Ultra-Fast Read-Only Blockchain Access -This unique RPC has been designed from the ground up to optimize for performance of read-only calls. Early results show a potential 5x performance boost in data-intensive RPC calls such as `eth_getLogs` compared to traditional nodes (`geth`, `erigon`, `reth` etc). +HyperRPC is an extremely fast read-only RPC designed specifically for data-intensive blockchain tasks. Built from the ground up to optimize performance, it offers a simple drop-in solution with dramatic speed improvements over traditional nodes. -Its optimizations mean that while it is suitable for heavy-lifting data extraction, it cannot be used for regular RPC calls to post transactions. Currently, method support is limited to: +## Table of Contents -> - `eth_chainId` -> - `eth_blockNumber` -> - `eth_getBlockByNumber` -> - `eth_getBlockByHash` -> - `eth_getTransactionByHash` -> - `eth_getTransactionByBlockHashAndIndex` -> - `eth_getTransactionByBlockNumberAndIndex` -> - `eth_getTransactionReceipt` -> - `eth_getBlockReceipts` -> - `eth_getLogs` -> - `trace_block` (only on [select chains](./hyperrpc-supported-networks)) +- [What is HyperRPC?](#what-is-hyperrpc) +- [Performance Advantages](#performance-advantages) +- [Supported Methods](#supported-methods) +- [Supported Networks](#supported-networks) +- [Getting Started](#getting-started) +- [Development Status](#development-status) +## What is HyperRPC? -These methods are already supported on a wide selection of chains (the most up-to-date list can be found [here](./hyperrpc-supported-networks)) +HyperRPC is a specialized JSON-RPC endpoint that focuses exclusively on **read operations** to deliver exceptional performance. By optimizing specifically for data retrieval workflows, HyperRPC can significantly outperform traditional nodes for data-intensive applications. -Testing the endpoint and providing feedback is hugely valuable. To access these endpoints for free please join our [Discord](https://discord.gg/Q9qt8gZ2fX) and reach out or alternatively login via GitHub on the main website to access your endpoints. +Key characteristics: -> ### Disclaimer -> -> - HyperRPC is still under active development to improve performance and stability. -> - It does not support all RPC methods. -> - It has not been audited for security purposes. -> - We appreciate your patience until we get there. Until then, we are happy to answer all questions in our [Discord](https://discord.gg/Q9qt8gZ2fX). +- **Read-only**: Optimized for data retrieval (cannot send transactions) +- **Drop-in compatible**: Works with existing tooling that uses standard RPC methods +- **Specialized**: Designed specifically for data-intensive operations like `eth_getLogs` + +## Performance Advantages + +Early benchmarks show that HyperRPC can deliver up to **5x performance improvement** for data-intensive operations compared to traditional nodes like `geth`, `erigon`, and `reth`. + +This performance boost is particularly noticeable for: + +- Historical data queries +- Log event filtering +- Block and transaction retrievals +- Analytics applications + +## Supported Methods + +HyperRPC currently supports the following Ethereum JSON-RPC methods: + +| Category | Methods | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| **Chain Data** | `eth_chainId`, `eth_blockNumber` | +| **Block Data** | `eth_getBlockByNumber`, `eth_getBlockByHash`, `eth_getBlockReceipts` | +| **Transaction Data** | `eth_getTransactionByHash`, `eth_getTransactionByBlockHashAndIndex`, `eth_getTransactionByBlockNumberAndIndex`, `eth_getTransactionReceipt` | +| **Event Logs** | `eth_getLogs` | +| **Traces** | `trace_block` (only on [select chains](./hyperrpc-supported-networks)) | + +## Supported Networks + +HyperRPC is available across numerous EVM-compatible networks. For the most up-to-date list of supported chains, please see our [Supported Networks](./hyperrpc-supported-networks) page. + +## Getting Started + +To start using HyperRPC: + +1. **Get Access**: + + - Visit our [Supported Networks](./hyperrpc-supported-networks) page to find all available HyperRPC endpoints + - Each network has a ready-to-use URL that you can start using immediately + +2. **Use Like a Standard RPC**: + + ```javascript + // Example: Fetching logs with HyperRPC + const response = await fetch("https://eth.rpc.hypersync.xyz", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "eth_getLogs", + params: [ + { + fromBlock: "0x1000000", + toBlock: "0x1000100", + address: "0xYourContractAddress", + }, + ], + }), + }); + ``` + +3. **Provide Feedback**: + Your testing and feedback are incredibly valuable as we continue to improve HyperRPC. Let us know about your experience in our [Discord](https://discord.gg/Q9qt8gZ2fX). + +## Development Status + +:::caution Important Notice + +- HyperRPC is under active development to further improve performance and stability +- It is designed for read-only operations and does not support all standard RPC methods +- It has not yet undergone formal security audits +- We welcome questions and feedback in our [Discord](https://discord.gg/Q9qt8gZ2fX) + ::: --- diff --git a/docs/HyperSync/api-tokens.mdx b/docs/HyperSync/api-tokens.mdx index 7c1080ae..5c636586 100644 --- a/docs/HyperSync/api-tokens.mdx +++ b/docs/HyperSync/api-tokens.mdx @@ -4,28 +4,53 @@ title: API Tokens sidebar_label: API Tokens slug: /api-tokens --- + import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; -# API Tokens for HyperSync Clients +# API Tokens for HyperSync + +## Overview + +API tokens provide authenticated access to HyperSync services, enabling enhanced capabilities and usage tracking. While currently optional, they will become mandatory in the future and offer significant advantages for serious usage. + +## Table of Contents + +- [Benefits of Using API Tokens](#benefits-of-using-api-tokens) +- [Generating API Tokens](#generating-api-tokens) +- [Implementation Guide](#implementation-guide) +- [Security Best Practices](#security-best-practices) + +## Benefits of Using API Tokens + +Using API tokens with HyperSync provides several advantages: -API tokens can be used with HyperSync clients to access enhanced services and track usage statistics. While not currently mandatory, they will become compulsory in the future. API tokens allow us to provide better service with dedicated capacity, bandwidth, and higher-quality guarantees. +- **Dedicated Resources**: Access to reserved capacity and bandwidth +- **Performance Guarantees**: Higher quality of service with better reliability +- **Usage Analytics**: Track and monitor your consumption patterns +- **Future-Proofing**: Early adoption before tokens become mandatory +- **Enhanced Support**: Better visibility for troubleshooting assistance ## Generating API Tokens -You can generate API tokens at [https://envio.dev/app/api-tokens](https://envio.dev/app/api-tokens). +You can generate API tokens through the Envio Dashboard: -## Using API Tokens +1. Visit [https://envio.dev/app/api-tokens](https://envio.dev/app/api-tokens) +2. Sign in to your account (or create one if you don't have one) +3. Follow the prompts to create a new token +4. Copy and securely store your token -To use an API token, you need to pass it as a `bearer_token` when creating a HyperSync client. Here's how to do it in different languages: +## Implementation Guide + +To use an API token, pass it as a `bearer_token` when creating a HyperSync client: - + ```typescript const client = HypersyncClient.new({ - url: "https://eth.hypersync.xyz", - bearerToken: process.env.HYPERSYNC_BEARER_TOKEN, + url: "https://eth.hypersync.xyz", + bearerToken: process.env.HYPERSYNC_BEARER_TOKEN, }); ``` @@ -35,7 +60,7 @@ const client = HypersyncClient.new({ ```python client = hypersync.HypersyncClient(hypersync.ClientConfig( url="https://eth.hypersync.xyz", - bearer_token=your_api_token + bearer_token=os.environ.get("HYPERSYNC_BEARER_TOKEN") )) ``` @@ -44,7 +69,7 @@ client = hypersync.HypersyncClient(hypersync.ClientConfig( ```rust let client = Client::new(ClientConfig { - bearer_token: Some("your_api_token".to_owned()), + bearer_token: Some(std::env::var("HYPERSYNC_BEARER_TOKEN").unwrap_or_default()), ..Default::default() }) .unwrap(); @@ -53,4 +78,19 @@ let client = Client::new(ClientConfig { -**Note on Security**: Bearer tokens should never be committed to git or shared publicly. We recommending using the environment for them and storing them them in `.env` files that are included in your `.gitignore`. +## Security Best Practices + +When working with API tokens: + +- **Never commit tokens to git repositories** +- **Use environment variables** to store tokens instead of hardcoding +- **Add token files like `.env` to your `.gitignore** +- **Rotate tokens periodically** for enhanced security +- **Limit token sharing** to only those who require access + +```bash +# Example .env file +HYPERSYNC_BEARER_TOKEN=your_secret_token_here +``` + +This approach keeps your tokens secure while making them available to your application at runtime. diff --git a/docs/HyperSync/hypersync-clients.md b/docs/HyperSync/hypersync-clients.md index d02f9969..4e76931c 100644 --- a/docs/HyperSync/hypersync-clients.md +++ b/docs/HyperSync/hypersync-clients.md @@ -1,39 +1,180 @@ --- id: hypersync-clients title: HyperSync Clients -sidebar_label: HyperSync Clients +sidebar_label: Clients ⭐ Recommended slug: /hypersync-clients --- -There are high-performance clients for JavaScript/TypeScript (NodeJS only for now), Python, and Rust now. Please get in touch if there are features that you would appreciate, for example, if you require a browser-based javascript client, additional data, or even usage in another language. +# HyperSync Client Libraries -The clients use a common rust base for high performance and uniform experience across different languages. All clients use our optimized binary transport and purpose-built API for maximum efficiency. +HyperSync provides powerful client libraries that enable you to integrate high-performance blockchain data access into your applications. These libraries handle the communication with HyperSync servers, data serialization/deserialization, and provide convenient APIs for querying blockchain data. -# NodeJS client +## Quick Links -- [Rendered API Docs](https://enviodev.github.io/hypersync-client-node/) -- [NPM page](https://www.npmjs.com/package/@envio-dev/hypersync-client) -- [GitHub repository](https://github.com/enviodev/hypersync-client-node) -- [Examples](https://github.com/enviodev/hypersync-client-node/tree/main/examples) +| Client | Resources | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Node.js** | [πŸ“ API Docs](https://enviodev.github.io/hypersync-client-node/) Β· [πŸ“¦ NPM](https://www.npmjs.com/package/@envio-dev/hypersync-client) Β· [πŸ’» GitHub](https://github.com/enviodev/hypersync-client-node) Β· [πŸ§ͺ Examples](https://github.com/enviodev/hypersync-client-node/tree/main/examples) | +| **Python** | [πŸ“¦ PyPI](https://pypi.org/project/hypersync/) Β· [πŸ’» GitHub](https://github.com/enviodev/hypersync-client-python) Β· [πŸ§ͺ Examples](https://github.com/enviodev/hypersync-client-python/tree/main/examples) | +| **Rust** | [πŸ“¦ Crates.io](https://crates.io/crates/hypersync-client) Β· [πŸ“ API Docs](https://docs.rs/hypersync-client/latest/hypersync_client/) Β· [πŸ’» GitHub](https://github.com/enviodev/hypersync-client-rust) Β· [πŸ§ͺ Examples](https://github.com/enviodev/hypersync-client-rust/tree/main/examples) | +| **Go** _(community)_ | [πŸ’» GitHub](https://github.com/enviodev/hypersync-client-go) Β· [πŸ§ͺ Examples](https://github.com/enviodev/hypersync-client-go/tree/main/examples) | +| **API Tokens** | [πŸ”‘ Get Tokens](./api-tokens.mdx) | -# Python Client +## Client Overview -- [PyPI page](https://pypi.org/project/hypersync/) -- [GitHub repository](https://github.com/enviodev/hypersync-client-python) -- [Examples](https://github.com/enviodev/hypersync-client-python/tree/main/examples) +All HyperSync clients share these key features: -# Rust Client +- **High Performance**: Built on a common Rust foundation for maximum efficiency +- **Optimized Transport**: Uses binary formats to minimize bandwidth and maximize throughput +- **Consistent Experience**: Similar APIs across all language implementations +- **Automatic Pagination**: Handles large data sets efficiently +- **Event Decoding**: Parses binary event data into structured formats -- [crates.io page](https://crates.io/crates/hypersync-client) -- [docs.rs page](https://docs.rs/hypersync-client/latest/hypersync_client/) -- [GitHub page](https://github.com/enviodev/hypersync-client-rust) -- [Examples](https://github.com/enviodev/hypersync-client-rust/tree/main/examples) +Choose the client that best matches your application's technology stack: -## API Tokens +| Feature | Node.js | Python | Rust | Go | +| ----------------- | ------------- | ------------------ | ------------- | ------------- | +| Async Support | βœ… | βœ… | βœ… | βœ… | +| Typing | TypeScript | Type Hints | Native | Native | +| Data Formats | JSON, Parquet | JSON, Parquet, CSV | JSON, Parquet | JSON, Parquet | +| Memory Efficiency | Good | Better | Best | Better | +| Installation | npm | pip | cargo | go get | -For information on how to use API tokens with HyperSync clients, please refer to our [API Tokens](./api-tokens) guide. +## Node.js Client -> ### Disclaimer -> -> - Docs under construction. -> - We appreciate your patience until we get there. Until then, we are happy to answer all questions in our [Discord](https://discord.gg/Q9qt8gZ2fX). +:::info Resources + +- [πŸ“ API Documentation](https://enviodev.github.io/hypersync-client-node/) +- [πŸ“¦ NPM Package](https://www.npmjs.com/package/@envio-dev/hypersync-client) +- [πŸ’» GitHub Repository](https://github.com/enviodev/hypersync-client-node) +- [πŸ§ͺ Example Projects](https://github.com/enviodev/hypersync-client-node/tree/main/examples) + ::: + +The Node.js client provides a TypeScript-first experience for JavaScript developers. + +### Installation + +```bash +# Using npm +npm install @envio-dev/hypersync-client + +# Using yarn +yarn add @envio-dev/hypersync-client + +# Using pnpm +pnpm add @envio-dev/hypersync-client +``` + +## Python Client + +:::info Resources + +- [πŸ“¦ PyPI Package](https://pypi.org/project/hypersync/) +- [πŸ’» GitHub Repository](https://github.com/enviodev/hypersync-client-python) +- [πŸ§ͺ Example Projects](https://github.com/enviodev/hypersync-client-python/tree/main/examples) + ::: + +The Python client provides a Pythonic interface with full type hinting support. + +### Installation + +```bash +pip install hypersync +``` + +## Rust Client + +:::info Resources + +- [πŸ“¦ Crates.io Package](https://crates.io/crates/hypersync-client) +- [πŸ“ API Documentation](https://docs.rs/hypersync-client/latest/hypersync_client/) +- [πŸ’» GitHub Repository](https://github.com/enviodev/hypersync-client-rust) +- [πŸ§ͺ Example Projects](https://github.com/enviodev/hypersync-client-rust/tree/main/examples) + ::: + +The Rust client provides the most efficient and direct access to HyperSync, with all the safety and performance benefits of Rust. + +### Installation + +Add the following to your `Cargo.toml`: + +```toml +[dependencies] +hypersync-client = "0.1" +tokio = { version = "1", features = ["full"] } +``` + +## Go Client + +:::info Resources + +- [πŸ’» GitHub Repository](https://github.com/enviodev/hypersync-client-go) +- [πŸ§ͺ Example Projects](https://github.com/enviodev/hypersync-client-go/tree/main/examples) + ::: + +:::caution Community Maintained +The Go client is community maintained and marked as work-in-progress. For production use, you may want to test thoroughly or consider the officially supported clients. +::: + +The Go client provides a native Go interface for accessing HyperSync, with support for streaming and decoding blockchain data. + +### Installation + +```bash +go get github.com/enviodev/hypersync-client-go +``` + +## Using API Tokens + +:::info Get Your API Token +You'll need an API token to use any HyperSync client. [Get your token here](./api-tokens.mdx). +::: + +All HyperSync clients require an API token for authentication. Tokens are used to manage access and usage limits. + +To get an API token: + +1. Visit the [Envio Dashboard](https://dash.envio.dev) +2. Register or sign in with your account +3. Navigate to the API Tokens section +4. Create a new token with the appropriate permissions + +For detailed instructions, see our [API Tokens](./api-tokens.mdx) guide. + +## Client Selection Guide + +Choose the client that best fits your use case: + +:::tip Node.js Client +**Choose when**: You're building JavaScript/TypeScript applications or if your team is most comfortable with the JavaScript ecosystem. +::: + +:::tip Python Client +**Choose when**: You're doing data science work, need integration with pandas/numpy, or if your team prefers Python's simplicity. +::: + +:::tip Rust Client +**Choose when**: You need maximum performance, are doing systems programming, or building performance-critical applications. +::: + +:::tip Go Client +**Choose when**: You're working in a Go ecosystem and want native integration with Go applications. Note that this client is community maintained. +::: + +## Additional Resources + +- [πŸ“š HyperSync Usage Guide](./hypersync-usage) +- [πŸ“ Query Reference](./hypersync-query) +- [πŸ§ͺ cURL Examples](./hypersync-curl-examples) +- [πŸ“Š Supported Networks](./hypersync-supported-networks) + +## Support + +Need help getting started or have questions about our clients? Connect with our community: + +- [Discord Community](https://discord.gg/Q9qt8gZ2fX) +- [GitHub Issues](https://github.com/enviodev) +- [Documentation Feedback](https://github.com/enviodev/docs/issues) + +``` + +``` diff --git a/docs/HyperSync/hypersync-curl-examples.md b/docs/HyperSync/hypersync-curl-examples.md index dcb39da2..b3f9f011 100644 --- a/docs/HyperSync/hypersync-curl-examples.md +++ b/docs/HyperSync/hypersync-curl-examples.md @@ -1,38 +1,61 @@ --- id: hypersync-curl-examples -title: Using curl requests with HyperSync -sidebar_label: HyperSync curl examples +title: Using curl with HyperSync +sidebar_label: curl Examples ⭐ Recommended slug: /hypersync-curl-examples --- -### When to Use Json API Requests and When Not +# Using curl with HyperSync -Using Json API requests to interact with envio.dev can be highly effective for various scenarios, but it's important to understand when it is appropriate to use them and when other methods might be more suitable. +This guide demonstrates how to interact with HyperSync using direct HTTP requests via curl. These examples provide a quick way to explore HyperSync functionality without installing client libraries. -##### Json API Advantages: +:::info Recommended Approach +We highly recommend trying these curl examples as they're super quick and easy to run directly in your terminal. It's one of the fastest ways to experience HyperSync's performance firsthand and see just how quickly you can retrieve blockchain data without any setup overhead. Simply copy, paste, and be amazed by the response speed! -- Quick Testing and Debugging: curl is ideal for quickly testing and debugging API endpoints without the need for complex setup. -- Automated Scripts: Perfect for shell scripts that automate HTTP requests efficiently. -- Simple Data Retrieval: Efficient and easy to use for straightforward data retrieval tasks. -- Flexibility: The Json API can be used with any programming language that doesn't support the HyperSync Client libraries. +While curl requests are technically slower than our client libraries (since they use HTTP rather than binary data transfer protocols), they're still impressively fast and provide an excellent demonstration of HyperSync's capabilities without any installation requirements. +::: -##### Client libraries advantages: +## Table of Contents -- Complex Workflows: HyperSync client libraries provide greater flexibility and convenient code organization for workflows involving multiple steps and conditional logic. -- Data Compression: HyperSync libraries automatically send data in a compressed format, enhancing throughput for data-intensive queries. -- Query Fragmentation Handling: Client libraries handle subsequent queries automatically if the initial query doesn't reach the to_block or the end of the chain. -- Arrow Support: Data can be returned in Apache Arrow format, facilitating easier data manipulation and analysis. -- Auto Retry: Client libraries automatically retry failed requests, ensuring more reliable data retrieval. +1. [Curl vs. Client Libraries](#curl-vs-client-libraries) +2. [Common Use Cases](#common-use-cases) + - [ERC-20 Transfers for an Address](#get-all-erc-20-transfers-for-an-address) + - [Contract Event Logs](#get-all-logs-for-a-smart-contract) + - [Blob Transactions](#get-blob-data-for-the-optimism-chain) + - [Token Mint Events](#get-mint-usdc-events) + - [Address Transactions](#get-all-transactions-for-an-address) + - [Transaction Status Filtering](#get-successful-or-failed-transactions) -### Curl query examples +## Curl vs. Client Libraries -#### Get all ERC-20 transfers for the EOA address +When deciding whether to use curl commands or client libraries, consider the following comparison: -The following query shows how to filter all ERC-20 transfer events for a specific EOA address. +### When to Use curl (JSON API) -Topics Filter: The topic filter is used to filter logs based on event signatures. In this example, the topic hash 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef is the signature for the ERC-20 Transfer event. The filter specifies that we are interested in logs where either the second or third topic (representing the sender and recipient addresses) matches 0x0000000000000000000000001e037f97d730Cc881e77F01E409D828b0bb14de0. +- **Quick Prototyping**: Test endpoints and explore data structure without setup +- **Simple Scripts**: Perfect for shell scripts and automation +- **Language Independence**: When working with languages without HyperSync client libraries +- **API Exploration**: When learning the HyperSync API capabilities -From/To Addresses: The transactions.from and transactions.to fields filter transactions by the sender (from) and recipient (to) addresses. Here, we filter transactions where the sender or recipient address is 0x1e037f97d730Cc881e77F01E409D828b0bb14de0. +### When to Use Client Libraries + +- **Production Applications**: For stable, maintained codebases +- **Complex Data Processing**: When working with large datasets or complex workflows +- **Performance**: Client libraries offer automatic compression and pagination +- **Error Handling**: Built-in retry mechanisms and better error reporting +- **Data Formats**: Support for efficient formats like Apache Arrow + +## Common Use Cases + +### Get All ERC-20 Transfers for an Address + +This example filters for all ERC-20 transfer events involving a specific address, either as sender or recipient. Feel free to swap your address into the example. + +**What this query does:** + +- Filters logs for the Transfer event signature (topic0) +- Matches when the address appears in either topic1 (sender) or topic2 (recipient) +- Also includes direct transactions to/from the address ```bash curl --request POST \ @@ -103,15 +126,18 @@ curl --request POST \ "input" ] } - }' +}' ``` -#### Get All Logs for a Smart Contract Address +### Get All Logs for a Smart Contract -This query returns all logs for a specified smart contract, starting from the beginning of the blockchain. +This example retrieves all event logs emitted by a specific contract (USDC in this case). -Note that this query might not return all data at once. Instead, it will likely return a `next_block` parameter, -which should be used as the `from_block` for subsequent queries to fetch the next chunk of data. +**Key points:** + +- Sets `from_block: 0` to scan from the beginning of the chain +- Uses `next_block` in the response for pagination to fetch subsequent data +- Includes relevant block, log, and transaction fields ```bash curl --request POST \ @@ -151,14 +177,19 @@ curl --request POST \ "input" ] } - }' +}' ``` -#### Get Blob Data for the Optimism Chain +### Get Blob Data for the Optimism Chain + +This example finds blob transactions used by the Optimism chain for data availability. -This query returns all blob transactions produced by the Optimism chain. After retrieving the transaction data, you can query the Ethereum network to get the relevant blobs of data. +**Key points:** -Note that these blobs are only stored for 18 days. +- Starts at a relatively recent block (20,000,000) +- Filters for transactions from the Optimism sequencer address +- Specifically looks for type 3 (blob) transactions +- Results can be used to retrieve the actual blob data from Ethereum ```bash curl --request POST \ @@ -188,14 +219,19 @@ curl --request POST \ "type" ] } - }' +}' ``` -#### Mint USDC events +### Get Mint USDC Events + +This example identifies USDC token minting events. -This query retrieves all mint events for the USDC token. +**How it works:** -It uses the signature for the transfer event (0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef) and the `from` address 0x0000000000000000000000000000000000000000000000000000000000000000 to identify mint transactions. +- Filters for the USDC contract address +- Looks for Transfer events (topic0) +- Specifically matches when topic1 (from address) is the zero address, indicating a mint +- Returns detailed information about each mint event ```bash curl --request POST \ @@ -244,14 +280,19 @@ curl --request POST \ "input" ] } - }' +}' ``` -#### Get All Transactions From/To an Address +### Get All Transactions for an Address -This query returns all transactions involving a specific EOA address, either as the sender or the recipient. +This example retrieves all transactions where a specific address is either the sender or receiver. -Note that `from_block` is set to a specific block because the beginning of the chain does not contain data related to this address. Multiple queries may be needed to start retrieving data. +**Implementation notes:** + +- Starts from a specific block (15,362,000) for efficiency +- Uses two transaction filters in an OR relationship +- Only includes essential fields in the response +- Multiple queries may be needed for complete history ```bash curl --request POST \ @@ -281,19 +322,24 @@ curl --request POST \ "to" ] } - }' +}' ``` -#### Get All Successful/Failed Transactions for the Last 10 Blocks +### Get Successful or Failed Transactions -First, query the current block height using https://eth.hypersync.xyz/height, subtract 10 from it, and store the result in a variable. +This example shows how to filter transactions based on their status (successful or failed) for recent blocks. -Use this variable as the from_block to get transactions with a matching status. +**How it works:** -This example returns successful transactions. To retrieve failed transactions, change the status to 0. +1. First, query the current chain height +2. Calculate a starting point (current height minus 10) +3. Query transactions with status=1 (successful) or status=0 (failed) ```bash +# Get current height and calculate starting block height=$((`curl https://eth.hypersync.xyz/height | jq .height` - 10)) + +# Query successful transactions (change status to 0 for failed transactions) curl --request POST \ --url https://eth.hypersync.xyz/query \ --header 'Content-Type: application/json' \ diff --git a/docs/HyperSync/hypersync-query.md b/docs/HyperSync/hypersync-query.md index 7a6cf1ab..3bbcdc35 100644 --- a/docs/HyperSync/hypersync-query.md +++ b/docs/HyperSync/hypersync-query.md @@ -1,580 +1,515 @@ --- id: hypersync-query title: HyperSync Query -sidebar_label: HyperSync Query +sidebar_label: Query & Response Structure slug: /hypersync-query --- # HyperSync Query -This page explains how the HyperSync query works and how to get the data you need from HyperSync. +This guide explains how to structure queries for HyperSync to efficiently retrieve blockchain data. You'll learn both the basics and advanced techniques to make the most of HyperSync's powerful querying capabilities. -:::note -Not all of the features that are implemented in HyperSync are implemented in HyperFuel (Fuel implementation of HyperSync). For example, stream and collect functions aren't implemented on the Fuel client as of writing. +:::note HyperFuel Limitations +Not all features implemented in HyperSync are available in HyperFuel (the Fuel implementation of HyperSync). For example, as of this writing, stream and collect functions aren't implemented in the Fuel client. +::: + +## Client Examples + +HyperSync offers client libraries in multiple languages, each with its own comprehensive examples. Instead of providing generic examples here, we recommend exploring the language-specific examples: + +| Client | Example Links | +| ----------- | -------------------------------------------------------------------------------------------- | +| **Node.js** | [Example Repository](https://github.com/enviodev/hypersync-client-node/tree/main/examples) | +| **Python** | [Example Repository](https://github.com/enviodev/hypersync-client-python/tree/main/examples) | +| **Rust** | [Example Repository](https://github.com/enviodev/hypersync-client-rust/tree/main/examples) | +| **Go** | [Example Repository](https://github.com/enviodev/hypersync-client-go/tree/main/examples) | + +Additionally, we maintain a comprehensive collection of real-world examples covering various use cases across different languages: + +- [**30 HyperSync Examples**](https://github.com/enviodev/30-hypersync-examples) - A diverse collection of practical examples demonstrating HyperSync's capabilities in Python, JavaScript, TypeScript, Rust, and more. + +For more details on client libraries, see the [HyperSync Clients](./hypersync-clients) documentation. + +:::tip Developer Tip +Set the `RUST_LOG` environment variable to `trace` for more detailed logs when using client libraries. ::: ## Table of Contents -1. [Introduction](#introduction) (A simple query example for introduction) -2. [Query Execution Explained](#query-execution-explained) (Explanation of how queries work) -3. [Query Structure](#query-fields) (Explanation of all query fields) -4. [Data Schema](#data-schema) (Entire schema for each data table) -5. [Response Structure](#response-structure) (Explanation of all response fields) -6. [Stream and Collect Functions](#stream-and-collect-functions) (Explains the stream and collect functions that are implemented in the client libraries) -7. [Join Modes](#join-modes) (Explains the join modes that are implemented in the client libraries) - -## Introduction - -This section gives a brief introduction using a simple query example. - -Below is an example query that gets some logs and transactions related to a contract at address `0x3f69bb14860f7f3348ac8a5f0d445322143f7fee`. - -We will explain the exact semantics of the query in the next sections but can just go over the basics here for an introduction. - -`logs` is the list of `LogSelection`s we want to pass. The server return all logs that match any of our selections. - -Also the transactions for the logs that our query matches will be returned. This is because transactions are joined after querying logs implicitly in HyperSync query implementation. Blocks are also joined after transactions but we won't get any blocks because we didn't select any block fields in our `field_selection`. - -`field_selection` is for selecting the specific fields we want returned. You can think of it like the part where you list column names in an SQL `SELECT` statement. The available tables are `log`, `transaction`, `block`, and `trace`. We will give an exact list of available columns in later sections. - -We also have a `max_num_logs` parameter here which means the query execution will stop as soon as the server realizes it reached or exceeded ten logs in the response. So this is not an exact limit, but it can be effective when trying to fine tune the query response size or runtime. - -If you want to drive this query until the end of the blockchain height or if you want to build a full data pipeline, it is recommended to use the `collect` and `stream` functions that are implemented in the client libraries. - -TIP: set `RUST_LOG` environment variable to `trace` if you want to see more logs when using the client libraries. - -```json -{ - "from_block": 0, - "logs": [ - { - "address": [ - "0x3f69bb14860f7f3348ac8a5f0d445322143f7fee" - ] - } - ], - "field_selection": { - "log": [ - "address", - "data", - "topic0", - "topic1", - "topic2", - "topic3" - ], - "transaction": [ - "from", - "to", - "value", - "input", - "output", - "gas_used", - "type", - "block_number" - ] - }, - "max_num_logs": 10 -} -``` +1. [Understanding HyperSync Queries](#understanding-hypersync-queries) +2. [Query Execution Process](#query-execution-process) +3. [Query Structure Reference](#query-structure-reference) +4. [Data Schema](#data-schema) +5. [Response Structure](#response-structure) +6. [Stream and Collect Functions](#stream-and-collect-functions) +7. [Working with Join Modes](#working-with-join-modes) +8. [Best Practices](#best-practices) + +## Understanding HyperSync Queries + +A HyperSync query defines what blockchain data you want to retrieve and how you want it returned. Unlike regular RPC calls, HyperSync queries offer: -## Query Execution Explained +- **Flexible filtering** across logs, transactions, traces, and blocks +- **Field selection** to retrieve only the data you need +- **Automatic pagination** to handle large result sets +- **Join capabilities** that link related blockchain data together -This section explains how the query executes on the server, step by step. +### Core Concepts -#### Preliminary +- **Selections**: Define criteria for filtering blockchain data (logs, transactions, traces) +- **Field Selection**: Specify which fields to include in the response +- **Limits**: Control query execution time and response size +- **Joins**: Determine how related data is connected in the response -The data in HyperSync server is split into groups by block number. Each group consists of data for a - contiguous block range. When the query runs, these groups are always queried atomically, meaning if a group - is queried it is never queried in half. This is important when considering limit arguments like `max_num_logs` - or when considering time/response_size limits. If the server realizes it reached a limit then it will terminate the - query after it finishes the current data group it is querying and then return the response to the user. On the - response it will put `next_block` value which is the block the query stopped at. +## Query Execution Process -#### Query limits +### How Data is Organized -- Time: There is a server-configured time limit that applies to all queries. This limit can be exceeded slightly by the server, the reason is explained in the preliminary above. -- Response size: There is a server-configured response size limit that applies to all queries. But the user can also set a lower response size limit in their query via max_num_* parameters. This limit can be exceeded slightly by the server, the reason is explained in the preliminary above. -- to_block: This can be set by the user in a query. The number is exclusive so the query will go up to `to_block` but won't include data from this block. This limit won't be exceeded by the server so the query always will stop at this block number and will not include data from this block number or the later blocks. Also, the next_block will be equal to this block number if none of the other limits are triggered before the server reaches this block number. +HyperSync organizes blockchain data into groups of contiguous blocks. When executing a query: -#### Steps +1. The server identifies which block group contains the starting block +2. It processes data groups sequentially until it hits a limit +3. Results are returned along with a `next_block` value for pagination -- Server receives the query and checks which block the query starts at. -- It finds the data group it should start from. -- It iterates through the data groups using the query until it hits some limit -- When a limit is hit, the response is serialized and sent back to the client. +### Query Limits -## Query Fields +HyperSync enforces several types of limits to ensure efficient query execution: -In this section, we show the explanation of each field in the query. We give the content in `rust` style for - formatting purposes. Field name casing can change based on which language client you are using, e.g. camelCase for nodejs client +| Limit Type | Description | Behavior | +| ----------------- | ------------------------------------------------ | --------------------------------------------------------- | +| **Time** | Server-configured maximum execution time | May slightly exceed limit to complete current block group | +| **Response Size** | Maximum data returned | May slightly exceed limit to complete current block group | +| **to_block** | User-specified ending block (exclusive) | Never exceeds this limit | +| **max*num*\*** | User-specified maximum number of results by type | May slightly exceed limit to complete current block group | + +### Execution Steps + +1. Server receives query and identifies the starting block group +2. It scans each block group, applying selection criteria +3. It joins related data according to the specified join mode +4. When a limit is reached, it finishes processing the current block group +5. It returns results with pagination information + +## Query Structure Reference + +A complete HyperSync query can include the following components: + +### Core Query Parameters ```rust struct Query { /// The block to start the query from from_block: u64, - /// The block to end the query at. If not specified, the query will go until the - /// end of data. Exclusive, the returned range will be [from_block..to_block). - /// - /// The query will return before it reaches this target block if it hits the time limit - /// configured on the server. The user should continue their query by putting the - /// next_block field in the response into from_block field of their next query. This implements - /// pagination. + + /// The block to end the query at (exclusive) + /// If not specified, the query runs until the end of available data to_block: Optional, - /// List of log selections, these have an OR relationship between them, so the query will return logs - /// that match any of these selections. + + /// Log selection criteria (OR relationship between selections) logs: Array, - /// List of transaction selections, the query will return transactions that match any of these selections and - /// it will return transactions that are related to the returned logs. + + /// Transaction selection criteria (OR relationship between selections) transactions: Array, - /// List of trace selections, the query will return traces that match any of these selections and - /// it will re turn traces that are related to the returned logs. + + /// Trace selection criteria (OR relationship between selections) traces: Array, - /// Weather to include all blocks regardless of if they are related to a returned transaction or log. Normally - /// the server will return only the blocks that are related to the transaction or logs in the response. But if this - /// is set to true, the server will return data for all blocks in the requested range [from_block, to_block). + + /// Whether to include all blocks in the requested range + /// Default: only return blocks related to matched transactions/logs include_all_blocks: bool, - /// Field selection. The user can select which fields they are interested in, requesting less fields will improve - /// query execution time and reduce the payload size so the user should always use a minimal number of fields. + + /// Fields to include in the response field_selection: FieldSelection, - /// Maximum number of blocks that should be returned, the server might return more blocks than this number but - /// it won't overshoot by too much. + + /// Maximum results limits (approximate) max_num_blocks: Optional, - /// Maximum number of transactions that should be returned, the server might return more transactions than this number but - /// it won't overshoot by too much. max_num_transactions: Optional, - /// Maximum number of logs that should be returned, the server might return more logs than this number but - /// it won't overshoot by too much. max_num_logs: Optional, - /// Maximum number of traces that should be returned, the server might return more traces than this number but - /// it won't overshoot by too much. max_num_traces: Optional, - /// Selects join mode for the query, - /// Default: join in this order logs -> transactions -> traces -> blocks - /// JoinAll: join everything to everything. For example if logSelection matches log0, we get the - /// associated transaction of log0 and then we get associated logs of that transaction as well. Applites similarly - /// to blocks, traces. - /// JoinNothing: join nothing. - #[serde(default)] - pub join_mode: JoinMode, + + /// Data relationship model (Default, JoinAll, or JoinNothing) + join_mode: JoinMode, } +``` +### Selection Types + +#### Log Selection + +```rust struct LogSelection { - /// Address of the contract, any logs that has any of these addresses will be returned. - /// Empty means match all. + /// Contract addresses to match (empty = match all) address: Array
, - /// Topics to match, each member of the top level array is another array, if the nth topic matches any - /// topic specified in nth element of topics, the log will be returned. Empty means match all. + + /// Topics to match by position (empty = match all) + /// Each array element corresponds to a topic position (0-3) + /// Within each position, any matching value will satisfy the condition topics: Array>, } +``` + +#### Transaction Selection +```rust struct TransactionSelection { - /// Address the transaction should originate from. If transaction.from matches any of these, the transaction - /// will be returned. Keep in mind that this has an and relationship with `to` filter, so each transaction should - /// match both of them. Empty means match all. + /// Sender addresses (empty = match all) + /// Has AND relationship with 'to' field from: Array
, - /// Address the transaction should go to. If transaction.to matches any of these, the transaction will - /// be returned. Keep in mind that this has an and relationship with `from` filter, so each transaction should - /// match both of them. Empty means match all. + + /// Recipient addresses (empty = match all) + /// Has AND relationship with 'from' field to: Array
, - /// If first 4 bytes of transaction input matches any of these, transaction will be returned. Empty means match all. + + /// Method signatures to match (first 4 bytes of input) sighash: Array, - /// If transaction.status matches this value, the transaction will be returned. + + /// Transaction status to match (1 = success, 0 = failure) status: Optional, - /// If transaction.type matches any of these values, the transaction will be returned. - #[rename("type")] + + /// Transaction types to match (e.g., 0 = legacy, 2 = EIP-1559) kind: Array, - // If transaction.contract_address matches any of these values, the transaction will be returned. + + /// Created contract addresses to match contract_address: Array
, } +``` + +#### Trace Selection +```rust struct TraceSelection { - /// Address the trace should originate from. If trace.from matches any of these, the trace - /// will be returned. Keep in mind that this has an and relationship with `to` filter, so each trace should - /// match both of them. Empty means match all. + /// Sender addresses (empty = match all) + /// Has AND relationship with 'to' field from: Array
, - /// Address the trace should go to. If trace.to matches any of these, the trace will - /// be returned. Keep in mind that this has an and relationship with `from` filter, so each trace should - /// match both of them. Empty means match all. + + /// Recipient addresses (empty = match all) + /// Has AND relationship with 'from' field to: Array
, - /// The trace will be returned if the trace is a contract deployment (create) trace - /// and the address of the deployed contract matches any of these addresses. Empty means match all. + + /// Created contract addresses to match address: Array
, - /// If trace.call_type matches any of these values, the trace will be returned. Empty means match all. - /// See ethereum RPC `trace_block` method logs to learn more about this field + + /// Call types to match (e.g., "call", "delegatecall") call_type: Array, - /// If trace.reward_type matches any of these values, the trace will be returned. Empty means match all. - /// See ethereum RPC `trace_block` method logs to learn more about this field + + /// Reward types to match (e.g., "block", "uncle") reward_type: Array, - /// If trace.type matches any of these values, the trace will be returned. Empty means match all. - /// For example it can be `reward` or `create`. - /// See ethereum RPC `trace_block` method logs to learn more about this field - #[rename("type")] + + /// Trace types to match (e.g., "call", "create", "suicide", "reward") kind: Array, - /// If first 4 bytes of trace input matches any of these, trace will be returned. Empty means match all. - /// Sighash is a ethereum style hex encoded string of four bytes, e.g. 0xa2b4c6d8 + + /// Method signatures to match (first 4 bytes of input) sighash: Array, } +``` + +#### Field Selection +```rust struct FieldSelection { - /// List of names of columns to return from block table + /// Block fields to include in response block: Array, - /// List of names of columns to return from transaction table + + /// Transaction fields to include in response transaction: Array, - /// List of names of columns to return from log table + + /// Log fields to include in response log: Array, - /// List of names of columns to return from trace table + + /// Trace fields to include in response trace: Array, } ``` ## Data Schema -This section is an exhaustive list of all columns of all tables. We give the content in `python` style for - formatting purposes. Casing is always snake_case when passing raw strings inside field_selection into a client library. +HyperSync organizes blockchain data into four main tables. Below are the available fields for each table. + +:::info Field Naming +When specifying fields in your query, always use snake_case names (e.g., `block_number`, not `blockNumber`). +::: + +### Block Fields ```python class BlockField(StrEnum): - """Block field enum""" - # A scalar value equal to the number of ancestor blocks. The genesis block has a number of zero; formally Hi. - NUMBER = 'number' - # The Keccak 256-bit hash of the block - HASH = 'hash' - # The Keccak 256-bit hash of the parent block’s header, in its entirety; formally Hp. - PARENT_HASH = 'parent_hash' - # A 64-bit value which, combined with the mixhash, proves that a sufficient amount of computation has been carried - # out on this block; formally Hn. - NONCE = 'nonce' - # The Keccak 256-bit hash of the ommers/uncles list portion of this block; formally Ho. - SHA3_UNCLES = 'sha3_uncles' - # The Bloom filter composed from indexable information (logger address and log topics) - # contained in each log entry from the receipt of each transaction in the transactions list; - # formally Hb. - LOGS_BLOOM = 'logs_bloom' - # The Keccak 256-bit hash of the root node of the trie structure populated with each - # transaction in the transactions list portion of the block; formally Ht. - TRANSACTIONS_ROOT = 'transactions_root' - # The Keccak 256-bit hash of the root node of the state trie, after all transactions are - # executed and finalisations applied; formally Hr. - STATE_ROOT = 'state_root' - # The Keccak 256-bit hash of the root node of the trie structure populated with each transaction in the - # transactions list portion of the block; formally Ht. - RECEIPTS_ROOT = 'receipts_root' - # The 160-bit address to which all fees collected from the successful mining of this block - # be transferred; formally Hc. - MINER = 'miner' - # A scalar value corresponding to the difficulty level of this block. This can be calculated - # from the previous block’s difficulty level and the timestamp; formally Hd. - DIFFICULTY = 'difficulty' - # The cumulative sum of the difficulty of all blocks that have been mined in the Ethereum network since the - # inception of the network It measures the overall security and integrity of the Ethereum network. - TOTAL_DIFFICULTY = 'total_difficulty' - # An arbitrary byte array containing data relevant to this block. This must be 32 bytes or - # fewer; formally Hx. - EXTRA_DATA = 'extra_data' - # The size of this block in bytes as an integer value, encoded as hexadecimal. - SIZE = 'size' - # A scalar value equal to the current limit of gas expenditure per block; formally Hl. - GAS_LIMIT = 'gas_limit' - # A scalar value equal to the total gas used in transactions in this block; formally Hg. - GAS_USED = 'gas_used' - # A scalar value equal to the reasonable output of Unix’s time() at this block’s inception; formally Hs. - TIMESTAMP = 'timestamp' - # Ommers/uncles header. - UNCLES = 'uncles' - # A scalar representing EIP1559 base fee which can move up or down each block according - # to a formula which is a function of gas used in parent block and gas target - # (block gas limit divided by elasticity multiplier) of parent block. - # The algorithm results in the base fee per gas increasing when blocks are - # above the gas target, and decreasing when blocks are below the gas target. The base fee per gas is burned. - BASE_FEE_PER_GAS = 'base_fee_per_gas' - # The total amount of blob gas consumed by the transactions within the block, added in EIP-4844. - BLOB_GAS_USED = 'blob_gas_used' - # A running total of blob gas consumed in excess of the target, prior to the block. Blocks - # with above-target blob gas consumption increase this value, blocks with below-target blob - # gas consumption decrease it (bounded at 0). This was added in EIP-4844. - EXCESS_BLOB_GAS = 'excess_blob_gas' - # The hash of the parent beacon block's root is included in execution blocks, as proposed by - # EIP-4788. - # This enables trust-minimized access to consensus state, supporting staking pools, bridges, and more. - PARENT_BEACON_BLOCK_ROOT = 'parent_beacon_block_root' - # The Keccak 256-bit hash of the withdrawals list portion of this block. - # See [EIP-4895](https://eips.ethereum.org/EIPS/eip-4895). - WITHDRAWALS_ROOT = 'withdrawals_root' - # Withdrawal represents a validator withdrawal from the consensus layer. - WITHDRAWALS = 'withdrawals' - # The L1 block number that would be used for block.number calls. - L1_BLOCK_NUMBER = 'l1_block_number' - # The number of L2 to L1 messages since Nitro genesis. - SEND_COUNT = 'send_count' - # The Merkle root of the outbox tree state. - SEND_ROOT = 'send_root' - # A 256-bit hash which, combined with the nonce, proves that a sufficient amount of computation has - # been carried out on this block; formally Hm. - MIX_HASH = 'mix_hash' + NUMBER = 'number' # Block number + HASH = 'hash' # Block hash + PARENT_HASH = 'parent_hash' # Parent block hash + TIMESTAMP = 'timestamp' # Block timestamp (Unix time) + MINER = 'miner' # Miner/validator address + GAS_LIMIT = 'gas_limit' # Block gas limit + GAS_USED = 'gas_used' # Total gas used in block + BASE_FEE_PER_GAS = 'base_fee_per_gas' # EIP-1559 base fee + # Many more fields available... +``` + +### Transaction Fields +```python class TransactionField(StrEnum): - """Transaction field enum""" - # The Keccak 256-bit hash of the block - BLOCK_HASH = 'block_hash' - # A scalar value equal to the number of ancestor blocks. The genesis block has a number of - # zero; formally Hi. - BLOCK_NUMBER = 'block_number' - # The 160-bit address of the message call’s sender - FROM = 'from' - # A scalar value equal to the maximum amount of gas that should be used in executing - # this transaction. This is paid up-front, before any computation is done and may not be increased later; - # formally Tg. - GAS = 'gas' - # A scalar value equal to the number of Wei to be paid per unit of gas for all computation - # costs incurred as a result of the execution of this transaction; formally Tp. - GAS_PRICE = 'gas_price' - # A transaction hash is a keccak hash of an RLP encoded signed transaction. - HASH = 'hash' - # Input has two uses depending if transaction is Create or Call (if `to` field is None or - # Some). pub init: An unlimited size byte array specifying the - # EVM-code for the account initialisation procedure CREATE, - # data: An unlimited size byte array specifying the - # input data of the message call, formally Td. - INPUT = 'input' - # A scalar value equal to the number of transactions sent by the sender; formally Tn. - NONCE = 'nonce' - # The 160-bit address of the message call’s recipient or, for a contract creation - # transaction, βˆ…, used here to denote the only member of B0 ; formally Tt. - TO = 'to' - # Index of the transaction in the block - TRANSACTION_INDEX = 'transaction_index' - # A scalar value equal to the number of Wei to be transferred to the message call’s recipient or, - # in the case of contract creation, as an endowment to the newly created account; formally Tv. - VALUE = 'value' - # Replay protection value based on chain_id. See EIP-155 for more info. - V = 'v' - # The R field of the signature; the point on the curve. - R = 'r' - # The S field of the signature; the point on the curve. - S = 's' - # yParity: Signature Y parity; formally Ty - Y_PARITY = 'y_parity' - # Max Priority fee that transaction is paying. This is also known as `GasTipCap` - MAX_PRIORITY_FEE_PER_GAS = 'max_priority_fee_per_gas' - # A scalar value equal to the maximum. This is also known as `GasFeeCap` - MAX_FEE_PER_GAS = 'max_fee_per_gas' - # Added as EIP-pub 155: Simple replay attack protection - CHAIN_ID = 'chain_id' - # The accessList specifies a list of addresses and storage keys; - # these addresses and storage keys are added into the `accessed_addresses` - # and `accessed_storage_keys` global sets (introduced in EIP-2929). - # A gas cost is charged, though at a discount relative to the cost of accessing outside the list. - ACCESS_LIST = 'access_list' - # Max fee per data gas aka BlobFeeCap or blobGasFeeCap - MAX_FEE_PER_BLOB_GAS = 'max_fee_per_blob_gas' - # It contains a list of fixed size hash(32 bytes) - BLOB_VERSIONED_HASHES = 'blob_versioned_hashes' - # The total amount of gas used in the block until this transaction was executed. - CUMULATIVE_GAS_USED = 'cumulative_gas_used' - # The sum of the base fee and tip paid per unit of gas. - EFFECTIVE_GAS_PRICE = 'effective_gas_price' - # Gas used by transaction - GAS_USED = 'gas_used' - # Address of created contract if transaction was a contract creation - CONTRACT_ADDRESS = 'contract_address' - # Bloom filter for logs produced by this transaction - LOGS_BLOOM = 'logs_bloom' - # Transaction type. For ethereum: Legacy, Eip2930, Eip1559, Eip4844 - KIND = 'type' - # The Keccak 256-bit hash of the root node of the trie structure populated with each - # transaction in the transactions list portion of the block; formally Ht. - ROOT = 'root' - # If transaction is executed successfully. This is the `statusCode` - STATUS = 'status' - # The fee associated with a transaction on the Layer 1, it is calculated as l1GasPrice multiplied by l1GasUsed - L1_FEE = 'l1_fee' - # The gas price for transactions on the Layer 1 - L1_GAS_PRICE = 'l1_gas_price' - # The amount of gas consumed by a transaction on the Layer 1 - L1_GAS_USED = 'l1_gas_used' - # A multiplier applied to the actual gas usage on Layer 1 to calculate the dynamic costs. - # If set to 1, it has no impact on the L1 gas usage - L1_FEE_SCALAR = 'l1_fee_scalar' - # Amount of gas spent on L1 calldata in units of L2 gas. - GAS_USED_FOR_L1 = 'gas_used_for_l1' + # Block-related fields + BLOCK_HASH = 'block_hash' # The Keccak 256-bit hash of the block + BLOCK_NUMBER = 'block_number' # Block number containing the transaction + + # Transaction identifiers + HASH = 'hash' # Transaction hash (keccak hash of RLP encoded signed transaction) + TRANSACTION_INDEX = 'transaction_index' # Index of the transaction in the block + + # Transaction participants + FROM = 'from' # 160-bit address of the sender + TO = 'to' # 160-bit address of the recipient (null for contract creation) + + # Gas information + GAS = 'gas' # Gas limit set by sender + GAS_PRICE = 'gas_price' # Wei paid per unit of gas + GAS_USED = 'gas_used' # Actual gas used by the transaction + CUMULATIVE_GAS_USED = 'cumulative_gas_used' # Total gas used in the block up to this transaction + EFFECTIVE_GAS_PRICE = 'effective_gas_price' # Sum of base fee and tip paid per unit of gas + + # EIP-1559 fields + MAX_PRIORITY_FEE_PER_GAS = 'max_priority_fee_per_gas' # Max priority fee (a.k.a. GasTipCap) + MAX_FEE_PER_GAS = 'max_fee_per_gas' # Max fee per gas (a.k.a. GasFeeCap) + + # Transaction data + INPUT = 'input' # Transaction input data or contract initialization code + VALUE = 'value' # Amount of ETH transferred in wei + NONCE = 'nonce' # Number of transactions sent by the sender + + # Signature fields + V = 'v' # Replay protection value (based on chain_id) + R = 'r' # The R field of the signature + S = 's' # The S field of the signature + Y_PARITY = 'y_parity' # Signature Y parity + CHAIN_ID = 'chain_id' # Chain ID for replay protection (EIP-155) + + # Contract-related fields + CONTRACT_ADDRESS = 'contract_address' # Address of created contract (for contract creation txs) + + # Transaction result fields + STATUS = 'status' # Success (1) or failure (0) + LOGS_BLOOM = 'logs_bloom' # Bloom filter for logs produced by this transaction + ROOT = 'root' # State root (pre-Byzantium) + + # EIP-2930 fields + ACCESS_LIST = 'access_list' # List of addresses and storage keys to pre-warm + + # EIP-4844 (blob transactions) fields + MAX_FEE_PER_BLOB_GAS = 'max_fee_per_blob_gas' # Max fee per data gas (blob fee cap) + BLOB_VERSIONED_HASHES = 'blob_versioned_hashes' # List of blob versioned hashes + + # Transaction type + KIND = 'type' # Transaction type (0=legacy, 1=EIP-2930, 2=EIP-1559, 3=EIP-4844) + + # L2-specific fields (for rollups) + L1_FEE = 'l1_fee' # Fee for L1 data (L1GasPrice Γ— L1GasUsed) + L1_GAS_PRICE = 'l1_gas_price' # Gas price on L1 + L1_GAS_USED = 'l1_gas_used' # Amount of gas consumed on L1 + L1_FEE_SCALAR = 'l1_fee_scalar' # Multiplier for L1 fee calculation + GAS_USED_FOR_L1 = 'gas_used_for_l1' # Gas spent on L1 calldata in L2 gas units +``` + +### Log Fields +```python class LogField(StrEnum): - """Log filed enum""" - # The boolean value indicating if the event was removed from the blockchain due - # to a chain reorganization. True if the log was removed. False if it is a valid log. - REMOVED = 'removed' - # The integer identifying the index of the event within the block's list of events. - LOG_INDEX = 'log_index' - # The integer index of the transaction within the block's list of transactions. - TRANSACTION_INDEX = 'transaction_index' - # The hash of the transaction that triggered the event. - TRANSACTION_HASH = 'transaction_hash' - # The hash of the block in which the event was included. - BLOCK_HASH = 'block_hash' - # The block number in which the event was included. - BLOCK_NUMBER = 'block_number' - # The contract address from which the event originated. - ADDRESS = 'address' - # The non-indexed data that was emitted along with the event. - DATA = 'data' - # Topic pushed by contract - TOPIC0 = 'topic0' - # Topic pushed by contract - TOPIC1 = 'topic1' - # Topic pushed by contract - TOPIC2 = 'topic2' - # Topic pushed by contract - TOPIC3 = 'topic3' + # Log identification + LOG_INDEX = 'log_index' # Index of the log in the block + TRANSACTION_INDEX = 'transaction_index' # Index of the transaction in the block + + # Transaction information + TRANSACTION_HASH = 'transaction_hash' # Hash of the transaction that created this log + + # Block information + BLOCK_HASH = 'block_hash' # Hash of the block containing this log + BLOCK_NUMBER = 'block_number' # Block number containing this log + + # Log content + ADDRESS = 'address' # Contract address that emitted the event + DATA = 'data' # Non-indexed data from the event + + # Topics (indexed parameters) + TOPIC0 = 'topic0' # Event signature hash + TOPIC1 = 'topic1' # First indexed parameter + TOPIC2 = 'topic2' # Second indexed parameter + TOPIC3 = 'topic3' # Third indexed parameter + + # Reorg information + REMOVED = 'removed' # True if log was removed due to chain reorganization +``` + +### Trace Fields +```python class TraceField(StrEnum): - """Trace field enum""" - # The address of the sender who initiated the transaction. - FROM = 'from' - # The address of the recipient of the transaction if it was a transaction to an address. - # For contract creation transactions, this field is None. - TO = 'to' - # The type of trace, `call` or `delegatecall`, two ways to invoke a function in a smart contract. - # `call` creates a new environment for the function to work in, so changes made in that - # function won't affect the environment where the function was called. - # `delegatecall` doesn't create a new environment. Instead, it runs the function within the - # environment of the caller, so changes made in that function will affect the caller's environment. - CALL_TYPE = 'call_type' - # The units of gas included in the transaction by the sender. - GAS = 'gas' - # The optional input data sent with the transaction, usually used to interact with smart contracts. - INPUT = 'input' - # The init code. - INIT = 'init' - # The value of the native token transferred along with the transaction, in Wei. - VALUE = 'value' - # The address of the receiver for reward transaction. - AUTHOR = 'author' - # Kind of reward. `Block` reward or `Uncle` reward. - REWARD_TYPE = 'reward_type' - # The hash of the block in which the transaction was included. - BLOCK_HASH = 'block_hash' - # The number of the block in which the transaction was included. - BLOCK_NUMBER = 'block_number' - # Destroyed address. - ADDRESS = 'address' - # Contract code. - CODE = 'code' - # The total used gas by the call, encoded as hexadecimal. - GAS_USED = 'gas_used' - # The return value of the call, encoded as a hexadecimal string. - OUTPUT = 'output' - # The number of sub-traces created during execution. When a transaction is executed on the EVM, - # it may trigger additional sub-executions, such as when a smart contract calls another smart - # contract or when an external account is accessed. - SUBTRACES = 'subtraces' - # An array that indicates the position of the transaction in the trace. - TRACE_ADDRESS = 'trace_address' - # The hash of the transaction. - TRANSACTION_HASH = 'transaction_hash' - # The index of the transaction in the block. - TRANSACTION_POSITION = 'transaction_position' - # The type of action taken by the transaction, `call`, `create`, `reward` and `suicide`. - # `call` is the most common type of trace and occurs when a smart contract invokes another contract's function. - # `create` represents the creation of a new smart contract. This type of trace occurs when a smart contract - # is deployed to the blockchain. - TYPE = 'type' - # A string that indicates whether the transaction was successful or not. - # None if successful, Reverted if not. - ERROR = 'error' + # Trace identification + TRANSACTION_HASH = 'transaction_hash' # Hash of the transaction + TRANSACTION_POSITION = 'transaction_position' # Index of the transaction in the block + SUBTRACES = 'subtraces' # Number of sub-traces created during execution + TRACE_ADDRESS = 'trace_address' # Array indicating position in the trace tree + + # Block information + BLOCK_HASH = 'block_hash' # Hash of the block containing this trace + BLOCK_NUMBER = 'block_number' # Block number containing this trace + + # Transaction participants + FROM = 'from' # Address of the sender + TO = 'to' # Address of the recipient (null for contract creation) + + # Value and gas + VALUE = 'value' # ETH value transferred (in wei) + GAS = 'gas' # Gas limit + GAS_USED = 'gas_used' # Gas actually used + + # Call data + INPUT = 'input' # Call data for function calls + INIT = 'init' # Initialization code for contract creation + OUTPUT = 'output' # Return data from the call + + # Contract information + ADDRESS = 'address' # Contract address (for creation/destruction) + CODE = 'code' # Contract code + + # Trace types and categorization + TYPE = 'type' # Trace type (call, create, suicide, reward) + CALL_TYPE = 'call_type' # Call type (call, delegatecall, staticcall, etc.) + REWARD_TYPE = 'reward_type' # Reward type (block, uncle) + + # Other actors + AUTHOR = 'author' # Address of receiver for reward transactions + + # Result information + ERROR = 'error' # Error message if failed ``` +For a complete list of all available fields, refer to the [HyperSync API Reference](https://docs.envio.dev/docs/api). + ## Response Structure -This section gives explanation of all response fields. We give the content in `rust` style for - formatting purposes. Field name casing can change based on which language client you are using, e.g. camelCase for nodejs client +When you execute a HyperSync query, the response includes both metadata and the requested data: ```rust -/// Query response from HyperSync server. struct QueryResponse { - /// Current height of the source HyperSync instance. - /// This number is inclusive. Returns null if the server has no blocks yet. + /// Current height of the blockchain in HyperSync archive_height: Optional, - /// Next block to query for, the responses are paginated so - /// the caller should continue the query from this block if they - /// didn't get responses up to the to_block they specified in the Query. + + /// Block number to use as from_block in your next query for pagination next_block: u64, - /// Total time it took the HyperSync instance to execute the query in milliseconds. + + /// Query execution time in milliseconds total_execution_time: u64, - /// Response data - /// This can be apache arrow formatted data or it can be native struct data based on - /// which client and method you are using. + + /// The actual blockchain data matching your query data: ResponseData, - /// Rollback guard, this structure is intended to make rollback handling easy and efficient on client side. + + /// Information to help handle chain reorganizations rollback_guard: Optional, } +``` + +### Rollback Guard + +The optional `rollback_guard` helps manage chain reorganizations: -/// This structure is intended to make rollback handling easy and efficient on client side. -/// Here "in memory" refers to data that is only kept in memory and not yet flushed to disk. The data that -/// can be rolled back at any time is kept in memory until it reaches enough block depth so we can be sure -/// it won't be rolled back. +```rust struct RollbackGuard { - /// Block number of last block scanned in memory + /// Last block scanned block_number: u64, - /// Block timestamp of last block scanned in memory timestamp: i64, - /// Block hash of last block scanned in memory hash: Hash, - /// Block number of first block scanned in memory + + /// First block scanned first_block_number: u64, - /// Parent hash of first block scanned in memory first_parent_hash: Hash, } ``` ## Stream and Collect Functions -This section explains how the `stream` and `collect` functions work on the client libraries. These functions run the query multiple times internally so they can be harder to understand compared to how a single query works. +For continuous data processing or building data pipelines, client libraries provide `stream` and `collect` functions that wrap the base query functionality. -Disclaimer: These functions are not designed to be used in a rollback context, if you are at the tip of the chain we recommend implementing a loop using one of the `get` functions from the client library you are using and handling the rollbacks manually. +:::caution Tip of Chain Warning +These functions are not designed for use at the blockchain tip where rollbacks may occur. For real-time data near the chain tip, implement a custom loop using the `get` functions and handle rollbacks manually. +::: -#### Stream +### Stream Function -Stream function runs many internal queries concurrently and gives back the results as they come. It returns a stream handle that can be used to receive the results. It pipelines decoding/decompressing etc. of response chunks so the user can get a very efficient experience out of the box. It will stream data until it reaches query.to_block or if to_block was not specified in the query, it runs until it reaches chain height at the time of the stream start (it gets the height of the chain when it is starting the stream and runs until it reaches it). +The `stream` function: -WARNING: If you are opening/closing many streams in your program, it is recommended to explicitly call `close` function on the stream handle so there is no resource leakage. +- Runs multiple queries concurrently +- Returns a stream handle that yields results as they're available +- Optimizes performance through pipelined decoding/decompression +- Continues until reaching either `to_block` or the chain height at stream start -#### Collect +### Collect Functions -The collect function essentially calls `stream` internally and collects all of the data into a single response. The `collect_parquet` function can be used to pipe data into a parquet file, it doesn't accumulate all data in memory so can be used to collect data that doesn't fit in RAM. We still recommend chunking the `collect_parquet` calls so you don't end up with big parquet files. +The `collect` functions: -TIP: set `RUST_LOG` environment variable to `trace` if you want to see more logs when using the client libraries. +- Call `stream` internally and aggregate results +- Offer different output formats (JSON, Parquet) +- Handle data that may not fit in memory -## Join Modes +:::warning Resource Management +Always call `close()` on stream handles when finished to prevent resource leaks, especially if creating multiple streams. +::: + +## Working with Join Modes -NOTE: The word join does not mean the same as in `SQL` in this context. It means the inclusion of relevant data in the response instead of left/outer joining or similar in `SQL`. You can think of it like you do a select on table `A` and then join table `B`. In SQL this would mean each row contains rows from `A` and `B`. But in HyperSync this means you get data for `A` and `B` separately and the rows you get from `A` mean you will get the joined-in rows from `B` as well in the response. +HyperSync "joins" connect related blockchain data automatically. Unlike SQL joins that combine rows from different tables, HyperSync joins determine which related records to include in the response. -In the context of HyperSync, joins refer to the implicit linking of different types of blockchain data (logs, transactions, traces, and blocks) based on certain relationships. Here's an explanation of how these joins work: +### Default Join Mode (logs β†’ transactions β†’ traces β†’ blocks) -### Default Join Mechanism +With the default join mode: -`logs` -> `transactions` -> `traces` -> `blocks` +1. When you query logs, you automatically get their associated transactions +2. Those transactions' traces are also included +3. The blocks containing these transactions are included -1. **Logs to Transactions**: - - When you query logs using `log_selection`, HyperSync automatically retrieves the transactions associated with those logs. This is based on the transaction hash present in each log. - - Example: If your `log_selection` returns some logs, HyperSync includes the transactions that generated these logs in the response as well. +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β” +β”‚ Logs β”‚ ──> β”‚ Transactions β”‚ ──> β”‚ Tracesβ”‚ ──> β”‚ Blocksβ”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ +``` -2. **Transactions to Traces**: - - After fetching the relevant transactions (either directly through `transaction_selection` or via logs), HyperSync retrieves the associated traces. - - Example: If your `transaction_selection` returns transactions or if you get some transactions because of your `log_selection`, HyperSync includes traces related to these transactions in the response as well. +### JoinAll Mode -And this same scheme goes on from traces into blocks as well. We realize this doesn't cover all use cases. For example, the user can't get logs based on their transaction_selection this way. This is why we implemented alternative `join_mode` implementations. +JoinAll creates a more comprehensive network of related data: -### Alternative Join Modes +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ + β–Ό β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β” <──> β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” <──> β”Œβ”€β”€β”€β”€β”€β”€β”€β” <──> β”Œβ”€β”€β”€β”€β”€β”€β”€β” +β”‚ Logs β”‚ β”‚ Transactions β”‚ β”‚ Tracesβ”‚ β”‚ Blocksβ”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ +``` -1. **JoinAll**: +For example, if you query a trace: - This mode first builds up a list of transactions based on the selection of every table. For example, the list will include a transaction if your `log_selection` selected a log generated by this transaction. +1. You get the transaction that created it +2. You get ALL logs from that transaction (not just the ones matching your criteria) +3. You get ALL traces from that transaction +4. You get the block containing the transaction + +### JoinNothing Mode + +JoinNothing is the most restrictive: + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β” +β”‚ Logs β”‚ β”‚ Transactions β”‚ β”‚ Tracesβ”‚ β”‚ Blocksβ”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ +``` - Then it includes every object relating to any of these transactions on the list. For example, it includes all traces that were generated by any of these transactions on the list. +Only data directly matching your selection criteria is returned, with no related records included. - For example, let's say you selected a trace. It will include the transaction that generated this trace. Then it will include everything related to this transaction including all the logs, traces, the block, and the transaction itself. +## Best Practices -3. **JoinNothing**: - This mode completely removes joining so you only get what your selections match. For example if your `log_selection` matches a log then you will only get that log, you will not get the block of the log or the transaction that generated that log. +To get the most out of HyperSync queries: -To use these join modes set `query.join_mode` to the desired value. It defaults to the `Default` mode if it is not specified. +1. **Minimize field selection** - Only request fields you actually need to improve performance +2. **Use appropriate limits** - Set `max_num_*` parameters to control response size +3. **Choose the right join mode** - Use `JoinNothing` for minimal data, `JoinAll` for complete context +4. **Process in chunks** - For large datasets, use pagination or the `stream` function +5. **Consider Parquet** - For analytical workloads, use `collect_parquet` for efficient storage +6. **Handle chain tip carefully** - Near the chain tip, implement custom rollback handling diff --git a/docs/HyperSync/hypersync-supported-networks.md b/docs/HyperSync/hypersync-supported-networks.md index 15e165ea..87ab6de0 100644 --- a/docs/HyperSync/hypersync-supported-networks.md +++ b/docs/HyperSync/hypersync-supported-networks.md @@ -10,7 +10,21 @@ We are rapidly adding new supported networks. If you don't see your network here ::: :::info -The Tier is the level of support (and therefore reliability) based on the infrastructure running the chain. If you would like us to upgrade a chain's level of support, reach out to us in [Discord](https://discord.gg/Q9qt8gZ2fX). +The Tier is the level of support (and therefore reliability) based on the infrastructure running the chain. We are actively working to make the tier distinctions more clear and transparent to our users. + +Currently, tiers relate to various service quality aspects including: + +- Allocated resources and compute power +- Query processing speed +- Infrastructure redundancy +- Backup frequency and retention +- Multi-region availability +- Priority for upgrades and new features +- SLA guarantees + +While detailed tier specifications are still being finalized, we're committed to providing transparent service level information in the near future. + +If you are a network operator or user and would like improved service support or to discuss upgrading a chain's level of support, please reach out to us in [Discord](https://discord.gg/Q9qt8gZ2fX). ::: diff --git a/docs/HyperSync/hypersync-usage.md b/docs/HyperSync/hypersync-usage.md index ee48ad9b..f8857819 100644 --- a/docs/HyperSync/hypersync-usage.md +++ b/docs/HyperSync/hypersync-usage.md @@ -1,51 +1,268 @@ --- id: hypersync-usage -title: Usage -sidebar_label: Usage +title: Using HyperSync +sidebar_label: Getting Started slug: /hypersync-usage --- +# Getting Started with HyperSync + +HyperSync is Envio's high-performance blockchain data engine that provides up to 2000x faster access to blockchain data compared to traditional RPC endpoints. This guide will help you understand how to effectively use HyperSync in your applications. + +## Quick Start Video + +Watch this quick tutorial to see HyperSync in action: + -### Examples +## Core Concepts + +HyperSync revolves around two main concepts: -We've found most developers have enjoyed learning HyperSync by practical example. You will find examples [here](./hypersync-clients.md) in Python, Rust, and NodeJs in each section. +1. **Queries** - Define what blockchain data you want to retrieve +2. **Output Configuration** - Specify how you want that data formatted and delivered -### Queries +Think of queries as your data filter and the output configuration as your data processor. -Using HyperSync primarily revolves around successfully constructing and then executing queries. Queries allow you to essentially filter for blocks, logs, transactions and traces. Hovering over types in your IDE will allow you to see all exhaustive options in order to construct an appropriate query. +## Building Effective Queries + +Queries are the heart of working with HyperSync. They allow you to filter for specific blocks, logs, transactions, and traces. + +### Query Structure + +A basic HyperSync query contains: ```python -class Query( - from_block: int, - field_selection: FieldSelection, - to_block: int | None = None, - logs: list[LogSelection] | None = None, - transactions: list[TransactionSelection] | None = None, - traces: list[TraceSelection] | None = None, - include_all_blocks: bool | None = None, - max_num_blocks: int | None = None, - max_num_transactions: int | None = None, - max_num_logs: int | None = None, - max_num_traces: int | None = None +query = hypersync.Query( + from_block=12345678, # Required: Starting block number + to_block=12345778, # Optional: Ending block number + field_selection=field_selection, # Required: What fields to return + logs=[log_selection], # Optional: Filter for specific logs + transactions=[tx_selection], # Optional: Filter for specific transactions + traces=[trace_selection], # Optional: Filter for specific traces + include_all_blocks=False, # Optional: Include blocks with no matches + max_num_blocks=1000, # Optional: Limit number of blocks processed + max_num_transactions=5000, # Optional: Limit number of transactions processed + max_num_logs=5000, # Optional: Limit number of logs processed + max_num_traces=5000 # Optional: Limit number of traces processed ) ``` -#### Field Selection +### Field Selection + +Field selection allows you to specify exactly which data fields you want to retrieve. This improves performance by only fetching what you need: -You can choose exactly what data you would like to be returned from the request. For example, this is useful when filtering for Logs, but you would also like the block data associated with that log to maybe get the timestamp of when that log was emitted. +```python +field_selection = hypersync.FieldSelection( + # Block fields you want to retrieve + block=[ + BlockField.NUMBER, + BlockField.TIMESTAMP, + BlockField.HASH + ], + + # Transaction fields you want to retrieve + transaction=[ + TransactionField.HASH, + TransactionField.FROM, + TransactionField.TO, + TransactionField.VALUE + ], + + # Log fields you want to retrieve + log=[ + LogField.ADDRESS, + LogField.TOPIC0, + LogField.TOPIC1, + LogField.TOPIC2, + LogField.TOPIC3, + LogField.DATA, + LogField.TRANSACTION_HASH + ], -### Useful tips: + # Trace fields you want to retrieve (if applicable) + trace=[ + TraceField.ACTION_FROM, + TraceField.ACTION_TO, + TraceField.ACTION_VALUE + ] +) +``` -- Run export `export RUST_LOG=trace` to see detailed HyperSync request progress information. -- HyperSync requests have a 5-second time limit. The request will return with the block that it reached during the query allowing you to paginate and make the next query. HyperSync generally scans through more than 10m blocks in 5 seconds. -- Modify `batch_size` and `batch_size` params based on your chain and use case to improve performance in some cases. E.g. +### Filtering for Specific Data + +For most use cases, you'll want to filter for specific logs, transactions, or traces: + +#### Log Selection Example ```python - config = hypersync.ParquetConfig( - path="data", - hex_output=True, - batch_size=1000000, - concurrency=10, +# Filter for Transfer events from USDC contract +log_selection = hypersync.LogSelection( + address=["0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"], # USDC contract + topics=[ + ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"] # Transfer event signature + ] +) +``` + +#### Transaction Selection Example + +```python +# Filter for transactions to the Uniswap V3 router +tx_selection = hypersync.TransactionSelection( + to=["0xE592427A0AEce92De3Edee1F18E0157C05861564"] # Uniswap V3 Router +) +``` + +## Processing the Results + +HyperSync provides multiple ways to process query results: + +### Stream to Parquet Files + +Parquet is the recommended format for large data sets: + +```python +# Configure output format +config = hypersync.StreamConfig( + hex_output=hypersync.HexOutput.PREFIXED, + event_signature="Transfer(address indexed from, address indexed to, uint256 value)" +) + +# Stream results to a Parquet file +await client.collect_parquet("data_directory", query, config) +``` + +### Stream to JSON Files + +For smaller datasets or debugging: + +```python +# Stream results to JSON +await client.collect_json("output.json", query, config) +``` + +### Process Data in Memory + +For immediate processing: + +```python +# Process data directly +async for result in client.stream(query, config): + for log in result.logs: + # Process each log + print(f"Transfer from {log.event_params['from']} to {log.event_params['to']}") +``` + +## Tips and Best Practices + +### Performance Optimization + +- **Use Appropriate Batch Sizes**: Adjust batch size based on your chain and use case: + + ```python + config = hypersync.ParquetConfig( + path="data", + hex_output=hypersync.HexOutput.PREFIXED, + batch_size=1000000, # Process 1M blocks at a time + concurrency=10, # Use 10 concurrent workers + ) + ``` + +- **Enable Trace Logs**: Set `RUST_LOG=trace` to see detailed progress: + + ```bash + export RUST_LOG=trace + ``` + +- **Paginate Large Queries**: HyperSync requests have a 5-second time limit. For large data sets, paginate results: + ```python + current_block = start_block + while current_block < end_block: + query.from_block = current_block + query.to_block = min(current_block + 1000000, end_block) + result = await client.collect_parquet("data", query, config) + current_block = result.end_block + 1 + ``` + +### Network-Specific Considerations + +- **High-Volume Networks**: For networks like Ethereum Mainnet, use smaller block ranges or more specific filters +- **Low-Volume Networks**: For smaller chains, you can process the entire chain in one query + +## Complete Example + +Here's a complete example that fetches all USDC Transfer events: + +```python +import hypersync +from hypersync import ( + LogSelection, + LogField, + BlockField, + FieldSelection, + TransactionField, + HexOutput +) +import asyncio + +async def collect_usdc_transfers(): + # Initialize client + client = hypersync.HypersyncClient( + hypersync.ClientConfig( + url="https://ethereum.hypersync.xyz", + bearer_token="your-token-here", # Get from https://docs.envio.dev/docs/HyperSync/api-tokens + ) + ) + + # Define field selection + field_selection = hypersync.FieldSelection( + block=[BlockField.NUMBER, BlockField.TIMESTAMP], + transaction=[TransactionField.HASH], + log=[ + LogField.ADDRESS, + LogField.TOPIC0, + LogField.TOPIC1, + LogField.TOPIC2, + LogField.DATA, + ] + ) + + # Define query for USDC transfers + query = hypersync.Query( + from_block=12000000, + to_block=12100000, + field_selection=field_selection, + logs=[ + LogSelection( + address=["0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"], # USDC contract + topics=[ + ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"] # Transfer signature + ] + ) + ] ) + + # Configure output + config = hypersync.StreamConfig( + hex_output=HexOutput.PREFIXED, + event_signature="Transfer(address indexed from, address indexed to, uint256 value)" + ) + + # Collect data to a Parquet file + result = await client.collect_parquet("usdc_transfers", query, config) + print(f"Processed blocks {query.from_block} to {result.end_block}") + +asyncio.run(collect_usdc_transfers()) ``` + +## Next Steps + +Now that you understand the basics of using HyperSync: + +- Browse the [Python Client](./hypersync-clients.md) or other language-specific clients +- Learn about [advanced query options](./hypersync-query.md) +- See [example queries for common use cases](./hypersync-curl-examples.md) +- [Get your API token](./api-tokens.mdx) to start building + +For detailed API references and examples in other languages, check our [client documentation](./hypersync-clients.md). diff --git a/docs/HyperSync/overview.md b/docs/HyperSync/overview.md index 5c777540..4182e858 100644 --- a/docs/HyperSync/overview.md +++ b/docs/HyperSync/overview.md @@ -75,63 +75,6 @@ HyperSync powers a wide range of blockchain applications, enabling developers to - **Ultra-fast Account Abstraction (AA) focused block explorer** - **Fast historical data retrieval** with minimal latency -## Getting Started with HyperSync - -HyperSync offers simple yet powerful client libraries. Here's a basic example using the Python client: - -```python -import hypersync -from hypersync import ( - LogSelection, - LogField, - FieldSelection, - TransactionField, -) -import asyncio - -async def collect_events(): - # Initialize client with your network of choice - client = hypersync.HypersyncClient( - hypersync.ClientConfig( - url="https://arbitrum.hypersync.xyz", - bearer_token="your-token-here", # See https://docs.envio.dev/docs/HyperSync/api-tokens - ) - ) - - # Define your query - this example finds all Uniswap v3 pool creation events - query = hypersync.Query( - from_block=0, - logs=[ - LogSelection( - address=["0x1F98431c8aD98523631AE4a59f267346ea31F984"], # Uniswap v3 factory - topics=[["0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118"]], # PoolCreated event - ) - ], - field_selection=FieldSelection( - log=[ - LogField.TOPIC0, - LogField.TOPIC1, - LogField.TOPIC2, - LogField.TOPIC3, - LogField.DATA, - LogField.TRANSACTION_HASH, - ], - transaction=[TransactionField.BLOCK_NUMBER], - ), - ) - - # Configure output format with automatic event decoding - config = hypersync.StreamConfig( - hex_output=hypersync.HexOutput.PREFIXED, - event_signature="PoolCreated(address indexed token0, address indexed token1, uint24 indexed fee, int24 tickSpacing, address pool)", - ) - - # Collect data to a local Parquet file - await client.collect_parquet("data", query, config) - -asyncio.run(collect_events()) -``` - ## See HyperSync in Action diff --git a/sidebarsHyperSync.js b/sidebarsHyperSync.js index ffe51cd5..620f1e43 100644 --- a/sidebarsHyperSync.js +++ b/sidebarsHyperSync.js @@ -4,24 +4,22 @@ module.exports = { "hypersync-usage", "hypersync-clients", "hypersync-query", - "hypersync-supported-networks", - "api-tokens", "hypersync-curl-examples", - { - type: "category", - label: "HyperFuel", - items: [ - "HyperFuel/hyperfuel", - "HyperFuel/hyperfuel-query", - ], - }, + "api-tokens", + "hypersync-supported-networks", { type: "category", label: "HyperRPC", + collapsed: false, items: [ "HyperRPC/overview-hyperrpc", "HyperRPC/hyperrpc-supported-networks", ], }, + { + type: "category", + label: "HyperFuel", + items: ["HyperFuel/hyperfuel", "HyperFuel/hyperfuel-query"], + }, ], }; From 6264f10ca5cf05ad5a3532f7785885e7bddad0f0 Mon Sep 17 00:00:00 2001 From: moose-code Date: Thu, 13 Mar 2025 11:47:32 +0000 Subject: [PATCH 3/7] Make LLM docs version --- LLM_DOCS_README.md | 59 +++++++++++++++++++++++ package.json | 8 +-- scripts/conditionally-update-endpoints.js | 28 +++++++++++ sidebarsHyperIndex.js | 33 +++++++++---- 4 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 LLM_DOCS_README.md create mode 100644 scripts/conditionally-update-endpoints.js diff --git a/LLM_DOCS_README.md b/LLM_DOCS_README.md new file mode 100644 index 00000000..4b456711 --- /dev/null +++ b/LLM_DOCS_README.md @@ -0,0 +1,59 @@ +# LLM-Friendly Documentation Setup + +This project includes a special build configuration for creating LLM-friendly documentation that excludes the large number of auto-generated supported network pages. + +## How It Works + +1. The build process checks for the `DOCS_FOR_LLM` environment variable +2. When `DOCS_FOR_LLM=true`, the build: + - Skips generating all supported network pages + - Creates a minimal list of important networks + - Produces a streamlined documentation site optimized for LLM context windows + +## Vercel Setup Instructions + +To set up the LLM-friendly documentation in Vercel: + +1. **Create a new Environment in Vercel**: + + - Go to your Vercel project dashboard + - Navigate to Settings β†’ Environment Variables + - Add a new Environment Variable: + - Name: `DOCS_FOR_LLM` + - Value: `true` + - In the "Environments" dropdown, select only the environment you want for LLM docs (e.g., "Preview") + +2. **Create a Preview Deployment**: + + - In Vercel, go to Deployments + - Create a new deployment + - Select your repository and branch + - Choose the environment where `DOCS_FOR_LLM=true` + - Deploy + +3. **Alternative: Create a Git Branch**: + - Create a branch named `llm-docs` in your repository + - Configure Vercel to use the `DOCS_FOR_LLM=true` environment variable for this branch + - Vercel will automatically deploy with the LLM-friendly configuration + +## Local Testing + +To test the LLM-friendly build locally: + +```bash +# Run with LLM mode enabled +DOCS_FOR_LLM=true yarn start + +# Run normal mode (full documentation) +yarn start +``` + +## Files Modified for This Feature + +1. `scripts/conditionally-update-endpoints.js` - Conditionally generates network pages +2. `package.json` - Updated build scripts to use the conditional script +3. `sidebarsHyperIndex.js` - Modified to conditionally include networks in the sidebar + +## Maintenance Notes + +When adding new features or sections to the documentation, consider whether they should be included in both versions or just the full version. diff --git a/package.json b/package.json index 265814c4..41a46d86 100644 --- a/package.json +++ b/package.json @@ -4,17 +4,17 @@ "private": true, "scripts": { "docusaurus": "docusaurus", - "prestart": "node ./scripts/update-endpoints.js", + "prestart": "node ./scripts/conditionally-update-endpoints.js", "start": "docusaurus start", - "predev": "node ./scripts/update-endpoints.js", + "predev": "node ./scripts/conditionally-update-endpoints.js", "dev": "yarn start", - "prebuild": "node ./scripts/update-endpoints.js", + "prebuild": "node ./scripts/conditionally-update-endpoints.js", "build": "docusaurus build", "swizzle": "docusaurus swizzle", "update-endpoints": "node ./scripts/update-endpoints.js", "update-cli-help": "node ./scripts/update-cli-help.js", "update-evm-config-schema": "curl https://raw.githubusercontent.com/enviodev/hyperindex/main/codegenerator/cli/npm/envio/evm.schema.json > ./static/schemas/config.evm.json", - "predeploy": "yarn update-endpoints && yarn update-cli-help && yarn update-evm-config-schema", + "predeploy": "yarn prebuild && yarn update-cli-help && yarn update-evm-config-schema", "deploy": "docusaurus deploy", "clear": "docusaurus clear", "serve": "docusaurus serve", diff --git a/scripts/conditionally-update-endpoints.js b/scripts/conditionally-update-endpoints.js new file mode 100644 index 00000000..78256451 --- /dev/null +++ b/scripts/conditionally-update-endpoints.js @@ -0,0 +1,28 @@ +// Check if we're in LLM mode +if (process.env.DOCS_FOR_LLM !== "true") { + // Run the regular endpoint update if not in LLM mode + require("./update-endpoints.js"); + console.log("Generated full documentation with network pages"); +} else { + console.log("Skipping network pages generation for LLM-friendly build"); + + // Create a minimal supported-networks.json with just a few essential networks + const fs = require("fs"); + const minimalNetworks = { + supportedNetworks: [ + "supported-networks/any-evm-with-rpc", + "supported-networks/local-anvil", + "supported-networks/local-hardhat", + "supported-networks/ethereum", + "supported-networks/arbitrum", + "supported-networks/polygon", + "supported-networks/optimism", + // Just a few key networks for LLM context efficiency + ], + }; + + fs.writeFileSync( + "./supported-networks.json", + JSON.stringify(minimalNetworks, null, 2) + ); +} diff --git a/sidebarsHyperIndex.js b/sidebarsHyperIndex.js index 62f9cf2d..675e289b 100644 --- a/sidebarsHyperIndex.js +++ b/sidebarsHyperIndex.js @@ -1,5 +1,28 @@ var { supportedNetworks } = require("./supported-networks.json"); +// Create a conditional networks section based on environment variable +const networksSection = + process.env.DOCS_FOR_LLM === "true" + ? { + type: "category", + label: "Supported Networks", + link: { + type: "doc", + id: "supported-networks/index", + }, + // In LLM mode, we use the minimal list from supported-networks.json + items: supportedNetworks, + } + : { + type: "category", + label: "Supported Networks", + link: { + type: "doc", + id: "supported-networks/index", + }, + items: supportedNetworks, + }; + module.exports = { someSidebar: [ "overview", @@ -101,15 +124,7 @@ module.exports = { "Troubleshoot/reserved-words", ], }, - { - type: "category", - label: "Supported Networks", - link: { - type: "doc", - id: "supported-networks/index", - }, - items: supportedNetworks, - }, + networksSection, "fuel/fuel", "licensing", "terms-of-service", From d2817d12955827f08912c4ca467bc75b1fde3a56 Mon Sep 17 00:00:00 2001 From: moose-code Date: Thu, 13 Mar 2025 12:16:04 +0000 Subject: [PATCH 4/7] Fix llm --- scripts/conditionally-update-endpoints.js | 26 +++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/conditionally-update-endpoints.js b/scripts/conditionally-update-endpoints.js index 78256451..f012506b 100644 --- a/scripts/conditionally-update-endpoints.js +++ b/scripts/conditionally-update-endpoints.js @@ -4,16 +4,17 @@ if (process.env.DOCS_FOR_LLM !== "true") { require("./update-endpoints.js"); console.log("Generated full documentation with network pages"); } else { - console.log("Skipping network pages generation for LLM-friendly build"); + console.log("Creating LLM-friendly build with minimal network pages"); - // Create a minimal supported-networks.json with just a few essential networks const fs = require("fs"); + const path = require("path"); + + // Define the minimal list of networks to include const minimalNetworks = { supportedNetworks: [ "supported-networks/any-evm-with-rpc", "supported-networks/local-anvil", "supported-networks/local-hardhat", - "supported-networks/ethereum", "supported-networks/arbitrum", "supported-networks/polygon", "supported-networks/optimism", @@ -21,8 +22,25 @@ if (process.env.DOCS_FOR_LLM !== "true") { ], }; + // Make sure we're only including network files that actually exist + const docsDir = path.join(__dirname, "..", "docs", "HyperIndex"); + const networksDir = path.join(docsDir, "supported-networks"); + + // Check if each network file exists; if not, remove from the list + minimalNetworks.supportedNetworks = minimalNetworks.supportedNetworks.filter( + (network) => { + const filename = path.basename(network) + ".md"; + return fs.existsSync(path.join(networksDir, filename)); + } + ); + + // Save the filtered minimal networks list fs.writeFileSync( - "./supported-networks.json", + path.join(__dirname, "..", "supported-networks.json"), JSON.stringify(minimalNetworks, null, 2) ); + + console.log( + `Created minimal network list with ${minimalNetworks.supportedNetworks.length} networks` + ); } From 1615ecc9a0ed6b96f717054cbc0fbc32b374e4eb Mon Sep 17 00:00:00 2001 From: moose-code Date: Thu, 13 Mar 2025 12:20:54 +0000 Subject: [PATCH 5/7] Improve logic --- scripts/conditionally-update-endpoints.js | 69 ++++++++++++----------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/scripts/conditionally-update-endpoints.js b/scripts/conditionally-update-endpoints.js index f012506b..dc5d6ed2 100644 --- a/scripts/conditionally-update-endpoints.js +++ b/scripts/conditionally-update-endpoints.js @@ -1,46 +1,49 @@ -// Check if we're in LLM mode -if (process.env.DOCS_FOR_LLM !== "true") { - // Run the regular endpoint update if not in LLM mode - require("./update-endpoints.js"); - console.log("Generated full documentation with network pages"); -} else { - console.log("Creating LLM-friendly build with minimal network pages"); +// Always run the regular endpoint update to generate all network files +require("./update-endpoints.js"); +console.log("Generated full documentation with network pages"); + +// Then conditionally filter the networks list for LLM mode +if (process.env.DOCS_FOR_LLM === "true") { + console.log("Creating LLM-friendly filtered network list"); const fs = require("fs"); const path = require("path"); - // Define the minimal list of networks to include - const minimalNetworks = { - supportedNetworks: [ - "supported-networks/any-evm-with-rpc", - "supported-networks/local-anvil", - "supported-networks/local-hardhat", - "supported-networks/arbitrum", - "supported-networks/polygon", - "supported-networks/optimism", - // Just a few key networks for LLM context efficiency - ], - }; - - // Make sure we're only including network files that actually exist - const docsDir = path.join(__dirname, "..", "docs", "HyperIndex"); - const networksDir = path.join(docsDir, "supported-networks"); + // Define the minimal list of important networks to include + const keysNetworks = [ + "supported-networks/any-evm-with-rpc", + "supported-networks/local-anvil", + "supported-networks/local-hardhat", + "supported-networks/arbitrum", + "supported-networks/polygon", + "supported-networks/optimism", + "supported-networks/ethereum", + ]; - // Check if each network file exists; if not, remove from the list - minimalNetworks.supportedNetworks = minimalNetworks.supportedNetworks.filter( - (network) => { - const filename = path.basename(network) + ".md"; - return fs.existsSync(path.join(networksDir, filename)); - } + // Read the current networks list that was generated + const supportedNetworksPath = path.join( + __dirname, + "..", + "supported-networks.json" ); + const currentNetworks = JSON.parse( + fs.readFileSync(supportedNetworksPath, "utf8") + ); + + // Filter to only include the key networks (ensuring they exist in the original list) + const filteredNetworks = { + supportedNetworks: keysNetworks.filter((network) => + currentNetworks.supportedNetworks.includes(network) + ), + }; - // Save the filtered minimal networks list + // Save the filtered list back to the file fs.writeFileSync( - path.join(__dirname, "..", "supported-networks.json"), - JSON.stringify(minimalNetworks, null, 2) + supportedNetworksPath, + JSON.stringify(filteredNetworks, null, 2) ); console.log( - `Created minimal network list with ${minimalNetworks.supportedNetworks.length} networks` + `Filtered network list from ${currentNetworks.supportedNetworks.length} to ${filteredNetworks.supportedNetworks.length} networks for LLM-friendly build` ); } From 3f5eec0d7130211e0c83fbf3e36071ad98b1717e Mon Sep 17 00:00:00 2001 From: moose-code Date: Thu, 13 Mar 2025 12:31:07 +0000 Subject: [PATCH 6/7] Filtered approach for llms --- package.json | 8 ++-- scripts/conditionally-update-endpoints.js | 49 -------------------- sidebarsHyperIndex.js | 56 ++++++++++++++--------- 3 files changed, 38 insertions(+), 75 deletions(-) delete mode 100644 scripts/conditionally-update-endpoints.js diff --git a/package.json b/package.json index 41a46d86..265814c4 100644 --- a/package.json +++ b/package.json @@ -4,17 +4,17 @@ "private": true, "scripts": { "docusaurus": "docusaurus", - "prestart": "node ./scripts/conditionally-update-endpoints.js", + "prestart": "node ./scripts/update-endpoints.js", "start": "docusaurus start", - "predev": "node ./scripts/conditionally-update-endpoints.js", + "predev": "node ./scripts/update-endpoints.js", "dev": "yarn start", - "prebuild": "node ./scripts/conditionally-update-endpoints.js", + "prebuild": "node ./scripts/update-endpoints.js", "build": "docusaurus build", "swizzle": "docusaurus swizzle", "update-endpoints": "node ./scripts/update-endpoints.js", "update-cli-help": "node ./scripts/update-cli-help.js", "update-evm-config-schema": "curl https://raw.githubusercontent.com/enviodev/hyperindex/main/codegenerator/cli/npm/envio/evm.schema.json > ./static/schemas/config.evm.json", - "predeploy": "yarn prebuild && yarn update-cli-help && yarn update-evm-config-schema", + "predeploy": "yarn update-endpoints && yarn update-cli-help && yarn update-evm-config-schema", "deploy": "docusaurus deploy", "clear": "docusaurus clear", "serve": "docusaurus serve", diff --git a/scripts/conditionally-update-endpoints.js b/scripts/conditionally-update-endpoints.js deleted file mode 100644 index dc5d6ed2..00000000 --- a/scripts/conditionally-update-endpoints.js +++ /dev/null @@ -1,49 +0,0 @@ -// Always run the regular endpoint update to generate all network files -require("./update-endpoints.js"); -console.log("Generated full documentation with network pages"); - -// Then conditionally filter the networks list for LLM mode -if (process.env.DOCS_FOR_LLM === "true") { - console.log("Creating LLM-friendly filtered network list"); - - const fs = require("fs"); - const path = require("path"); - - // Define the minimal list of important networks to include - const keysNetworks = [ - "supported-networks/any-evm-with-rpc", - "supported-networks/local-anvil", - "supported-networks/local-hardhat", - "supported-networks/arbitrum", - "supported-networks/polygon", - "supported-networks/optimism", - "supported-networks/ethereum", - ]; - - // Read the current networks list that was generated - const supportedNetworksPath = path.join( - __dirname, - "..", - "supported-networks.json" - ); - const currentNetworks = JSON.parse( - fs.readFileSync(supportedNetworksPath, "utf8") - ); - - // Filter to only include the key networks (ensuring they exist in the original list) - const filteredNetworks = { - supportedNetworks: keysNetworks.filter((network) => - currentNetworks.supportedNetworks.includes(network) - ), - }; - - // Save the filtered list back to the file - fs.writeFileSync( - supportedNetworksPath, - JSON.stringify(filteredNetworks, null, 2) - ); - - console.log( - `Filtered network list from ${currentNetworks.supportedNetworks.length} to ${filteredNetworks.supportedNetworks.length} networks for LLM-friendly build` - ); -} diff --git a/sidebarsHyperIndex.js b/sidebarsHyperIndex.js index 675e289b..2df86721 100644 --- a/sidebarsHyperIndex.js +++ b/sidebarsHyperIndex.js @@ -1,27 +1,39 @@ var { supportedNetworks } = require("./supported-networks.json"); -// Create a conditional networks section based on environment variable -const networksSection = - process.env.DOCS_FOR_LLM === "true" - ? { - type: "category", - label: "Supported Networks", - link: { - type: "doc", - id: "supported-networks/index", - }, - // In LLM mode, we use the minimal list from supported-networks.json - items: supportedNetworks, - } - : { - type: "category", - label: "Supported Networks", - link: { - type: "doc", - id: "supported-networks/index", - }, - items: supportedNetworks, - }; +// Direct filtering in the sidebar code +let filteredNetworks = supportedNetworks; + +// If in LLM mode, filter the networks directly here +if (process.env.DOCS_FOR_LLM === "true") { + const keyNetworks = [ + "supported-networks/any-evm-with-rpc", + "supported-networks/local-anvil", + "supported-networks/local-hardhat", + "supported-networks/arbitrum", + "supported-networks/polygon", + "supported-networks/optimism", + "supported-networks/ethereum", + ]; + + // Filter to only include key networks that exist in the original list + filteredNetworks = supportedNetworks.filter((network) => + keyNetworks.includes(network) + ); + + console.log( + `Sidebar using filtered networks: ${filteredNetworks.length} (from ${supportedNetworks.length})` + ); +} + +const networksSection = { + type: "category", + label: "Supported Networks", + link: { + type: "doc", + id: "supported-networks/index", + }, + items: filteredNetworks, +}; module.exports = { someSidebar: [ From aec02da1f9552d36f0389ecc4df3878321c1612b Mon Sep 17 00:00:00 2001 From: moose-code Date: Thu, 13 Mar 2025 12:33:27 +0000 Subject: [PATCH 7/7] Filtered approach for llms --- sidebarsHyperIndex.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sidebarsHyperIndex.js b/sidebarsHyperIndex.js index 2df86721..9ca869e4 100644 --- a/sidebarsHyperIndex.js +++ b/sidebarsHyperIndex.js @@ -12,7 +12,7 @@ if (process.env.DOCS_FOR_LLM === "true") { "supported-networks/arbitrum", "supported-networks/polygon", "supported-networks/optimism", - "supported-networks/ethereum", + "supported-networks/eth", ]; // Filter to only include key networks that exist in the original list