From b7b74f03621e17470360c52c400cc2525b2fffce Mon Sep 17 00:00:00 2001 From: Stefan Date: Tue, 10 Feb 2026 19:56:59 +0100 Subject: [PATCH 1/4] fix besu prometheus scrape path to /metrics Besu serves metrics at /metrics, not /debug/metrics/prometheus (which is geth/erigon-specific). This was causing 404s and no Besu EL metrics reaching Victoria Metrics. --- ansible/inventories/devnet-2/group_vars/besu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/inventories/devnet-2/group_vars/besu.yaml b/ansible/inventories/devnet-2/group_vars/besu.yaml index a68551c..22ba7a3 100644 --- a/ansible/inventories/devnet-2/group_vars/besu.yaml +++ b/ansible/inventories/devnet-2/group_vars/besu.yaml @@ -90,7 +90,7 @@ prometheus_config: | labels: instance: "{{ ethereum_network_name }}-{{ inventory_hostname }}" - job_name: "execution" - metrics_path: "/debug/metrics/prometheus" + metrics_path: "/metrics" static_configs: - targets: ["execution:{{ ethereum_node_el_ports_metrics }}"] labels: From 5347747309e9da31887490ec02fa724502f13e81 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 12 Feb 2026 14:13:47 +0100 Subject: [PATCH 2/4] add dedicated benchmark nodes --- terraform/devnet-2/hetzner/main.tf | 7 +++ terraform/devnet-2/hetzner/nodes.tf | 66 +++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/terraform/devnet-2/hetzner/main.tf b/terraform/devnet-2/hetzner/main.tf index 4aae915..928068a 100644 --- a/terraform/devnet-2/hetzner/main.tf +++ b/terraform/devnet-2/hetzner/main.tf @@ -141,5 +141,12 @@ locals { var.grandine_besu_super, var.grandine_reth_super, var.grandine_nimbusel_super, + # Benchmark Nodes + var.lighthouse_besu_sequential_super, + var.lighthouse_besu_optimized_super, + var.lighthouse_besu_noprefetch_super, + var.lighthouse_geth_sequential_super, + var.lighthouse_geth_optimized_super, + var.lighthouse_geth_noprefetch_super ] } diff --git a/terraform/devnet-2/hetzner/nodes.tf b/terraform/devnet-2/hetzner/nodes.tf index 23204f5..1dd0db7 100644 --- a/terraform/devnet-2/hetzner/nodes.tf +++ b/terraform/devnet-2/hetzner/nodes.tf @@ -30,6 +30,39 @@ variable "lighthouse_geth_super" { } } +variable "lighthouse_geth_optimized_super" { + default = { + name = "lighthouse-geth-optimized-super" + count = 1 + validator_start = 0 + validator_end = 0 + size = "ccX33" + supernode = true + } +} + +variable "lighthouse_geth_sequential_super" { + default = { + name = "lighthouse-geth-sequential-super" + count = 1 + validator_start = 0 + validator_end = 0 + size = "ccx33" + supernode = true + } +} + +variable "lighthouse_geth_noprefetch_super" { + default = { + name = "lighthouse-geth-noprefetch-super" + count = 1 + validator_start = 0 + validator_end = 0 + size = "ccx33" + supernode = true + } +} + variable "lighthouse_besu_super" { default = { name = "lighthouse-besu-super" @@ -39,6 +72,39 @@ variable "lighthouse_besu_super" { } } +variable "lighthouse_besu_optimized_super" { + default = { + name = "lighthouse-besu-optimized-super" + count = 1 + validator_start = 0 + validator_end = 0 + size = "ccX33" + supernode = true + } +} + +variable "lighthouse_besu_sequential_super" { + default = { + name = "lighthouse-besu-sequential-super" + count = 1 + validator_start = 0 + validator_end = 0 + size = "ccx33" + supernode = true + } +} + +variable "lighthouse_besu_noprefetch_super" { + default = { + name = "lighthouse-besu-noprefetch-super" + count = 1 + validator_start = 0 + validator_end = 0 + size = "ccx33" + supernode = true + } +} + variable "lighthouse_nethermind_super" { default = { name = "lighthouse-nethermind-super" From 5be5f97f4406a1b58297eac35198571299ac7c34 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 12 Feb 2026 14:27:09 +0100 Subject: [PATCH 3/4] prepare bench ansible config --- .../devnet-2/group_vars/ethereum_node.yaml | 2 +- .../inventories/devnet-2/group_vars/geth.yaml | 2 +- .../hc-lighthouse-besu-noprefetch-super-1.yaml | 16 ++++++++++++++++ .../hc-lighthouse-besu-optimized-super-1.yaml | 15 +++++++++++++++ .../hc-lighthouse-besu-sequential-super-1.yaml | 13 +++++++++++++ .../hc-lighthouse-geth-noprefetch-super-1.yaml | 16 ++++++++++++++++ .../hc-lighthouse-geth-optimized-super-1.yaml | 16 ++++++++++++++++ .../hc-lighthouse-geth-sequential-super-1.yaml | 16 ++++++++++++++++ 8 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-noprefetch-super-1.yaml create mode 100644 ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-optimized-super-1.yaml create mode 100644 ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml create mode 100644 ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-noprefetch-super-1.yaml create mode 100644 ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-optimized-super-1.yaml create mode 100644 ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-sequential-super-1.yaml diff --git a/ansible/inventories/devnet-2/group_vars/ethereum_node.yaml b/ansible/inventories/devnet-2/group_vars/ethereum_node.yaml index c6c0e29..a016b78 100644 --- a/ansible/inventories/devnet-2/group_vars/ethereum_node.yaml +++ b/ansible/inventories/devnet-2/group_vars/ethereum_node.yaml @@ -14,7 +14,7 @@ prometheus_remote_write_username: "{{ secret_prometheus_remote_write.username }} prometheus_remote_write_password: "{{ secret_prometheus_remote_write.password }}" prometheus_config: | global: - scrape_interval: 30s + scrape_interval: 12s evaluation_interval: 30s scrape_timeout: 10s external_labels: diff --git a/ansible/inventories/devnet-2/group_vars/geth.yaml b/ansible/inventories/devnet-2/group_vars/geth.yaml index 036330b..1775627 100644 --- a/ansible/inventories/devnet-2/group_vars/geth.yaml +++ b/ansible/inventories/devnet-2/group_vars/geth.yaml @@ -35,7 +35,7 @@ geth_container_pull: true prometheus_config: | global: - scrape_interval: 30s + scrape_interval: 12s evaluation_interval: 30s scrape_timeout: 10s external_labels: diff --git a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-noprefetch-super-1.yaml b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-noprefetch-super-1.yaml new file mode 100644 index 0000000..bf0ceb6 --- /dev/null +++ b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-noprefetch-super-1.yaml @@ -0,0 +1,16 @@ +# Benchmark: noprefetch (prefetch disabled, bonsai cache enabled) +besu_container_image: ethpandaops/besu:bal-devnet-2-with-prefetch + +besu_container_command_extra_args: + - --genesis-file=/network-config/besu.json + - --rpc-http-api=ADMIN,DEBUG,ETH,MINER,NET,TRACE,TXPOOL,WEB3 + - --sync-mode=FULL + - --data-storage-format=BONSAI + - --bonsai-limit-trie-logs-enabled=false + - --bonsai-historical-block-limit=100000000 + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --target-gas-limit=80000000 + - --Xbal-prefetch-reading-enabled=false + - --bonsai-cache-enabled=true + - --Xbal-trust-state-root=true + - --Xbal-perfect-parallelization-enabled=true diff --git a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-optimized-super-1.yaml b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-optimized-super-1.yaml new file mode 100644 index 0000000..525ad2a --- /dev/null +++ b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-optimized-super-1.yaml @@ -0,0 +1,15 @@ +# Benchmark: optimized (prefetch enabled) +besu_container_image: ethpandaops/besu:bal-devnet-2-with-prefetch + +besu_container_command_extra_args: + - --genesis-file=/network-config/besu.json + - --rpc-http-api=ADMIN,DEBUG,ETH,MINER,NET,TRACE,TXPOOL,WEB3 + - --sync-mode=FULL + - --data-storage-format=BONSAI + - --bonsai-limit-trie-logs-enabled=false + - --bonsai-historical-block-limit=100000000 + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --target-gas-limit=80000000 + - --Xbal-prefetch-reading-enabled=true + - --Xbal-trust-state-root=true + - --Xbal-perfect-parallelization-enabled=true diff --git a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml new file mode 100644 index 0000000..298674e --- /dev/null +++ b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml @@ -0,0 +1,13 @@ +# Benchmark: sequential (BAL optimization disabled) +besu_container_image: ethpandaops/besu:bal-devnet-2-with-prefetch + +besu_container_command_extra_args: + - --genesis-file=/network-config/besu.json + - --rpc-http-api=ADMIN,DEBUG,ETH,MINER,NET,TRACE,TXPOOL,WEB3 + - --sync-mode=FULL + - --data-storage-format=BONSAI + - --bonsai-limit-trie-logs-enabled=false + - --bonsai-historical-block-limit=100000000 + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --target-gas-limit=80000000 + - --Xbal-optimization-enabled=false diff --git a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-noprefetch-super-1.yaml b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-noprefetch-super-1.yaml new file mode 100644 index 0000000..5b31338 --- /dev/null +++ b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-noprefetch-super-1.yaml @@ -0,0 +1,16 @@ +# Benchmark: noprefetch (parallel execution without async warming) +default_ethereum_client_images: + lighthouse: ethpandaops/lighthouse:bal-devnet-2-65bb283 + geth: ethpandaops/geth:bal-devnet-2 + +geth_container_command_extra_args: + - --http.api=eth,net,web3,debug,admin,txpool,trace + - --http.vhosts=* + - --networkid={{ ethereum_network_id }} + - --syncmode=full + - --history.state=0 + - --gcmode=archive + - --syncmode=full + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --miner.gaslimit=80000000 + - --bal.executionmode=nobatchio diff --git a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-optimized-super-1.yaml b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-optimized-super-1.yaml new file mode 100644 index 0000000..a82d800 --- /dev/null +++ b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-optimized-super-1.yaml @@ -0,0 +1,16 @@ +# Benchmark: optimized (full BAL acceleration) +default_ethereum_client_images: + lighthouse: ethpandaops/lighthouse:bal-devnet-2 + geth: ethpandaops/geth:bal-devnet-2 + +geth_container_command_extra_args: + - --http.api=eth,net,web3,debug,admin,txpool,trace + - --http.vhosts=* + - --networkid={{ ethereum_network_id }} + - --syncmode=full + - --history.state=0 + - --gcmode=archive + - --syncmode=full + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --miner.gaslimit=80000000 + - --bal.executionmode=full diff --git a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-sequential-super-1.yaml b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-sequential-super-1.yaml new file mode 100644 index 0000000..7024730 --- /dev/null +++ b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-geth-sequential-super-1.yaml @@ -0,0 +1,16 @@ +# Benchmark: sequential (no BAL acceleration) +default_ethereum_client_images: + lighthouse: ethpandaops/lighthouse:bal-devnet-2 + geth: ethpandaops/geth:bal-devnet-2 + +geth_container_command_extra_args: + - --http.api=eth,net,web3,debug,admin,txpool,trace + - --http.vhosts=* + - --networkid={{ ethereum_network_id }} + - --syncmode=full + - --history.state=0 + - --gcmode=archive + - --syncmode=full + - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --miner.gaslimit=80000000 + - --bal.executionmode=sequential From cfd291df365f9d7af0377639541d047b66d9151c Mon Sep 17 00:00:00 2001 From: qu0b Date: Sun, 15 Feb 2026 02:23:35 +0000 Subject: [PATCH 4/4] fix: disable parallelization in besu sequential benchmark Add --Xbal-perfect-parallelization-enabled=false and --Xbal-trust-state-root=false to ensure a truly sequential baseline without optimistic parallelization. Co-Authored-By: Claude Opus 4.6 --- .../host_vars/hc-lighthouse-besu-sequential-super-1.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml index 298674e..29fd006 100644 --- a/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml +++ b/ansible/inventories/devnet-2/host_vars/hc-lighthouse-besu-sequential-super-1.yaml @@ -11,3 +11,5 @@ besu_container_command_extra_args: - --bootnodes={{ ethereum_el_bootnodes | join(',') }} - --target-gas-limit=80000000 - --Xbal-optimization-enabled=false + - --Xbal-perfect-parallelization-enabled=false + - --Xbal-trust-state-root=false