From faa1319ee6c1fc0e60eaafb87ab8066634c3923e Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Wed, 24 Sep 2025 20:55:05 +0530 Subject: [PATCH 01/18] feat(indexer): add builder observer data indexer --- tools/indexer/cmd/indexer/main.go | 434 ++++++++++++++++++++++ tools/indexer/go.mod | 28 ++ tools/indexer/go.sum | 48 +++ tools/indexer/pkg/backfill/backfill.go | 250 +++++++++++++ tools/indexer/pkg/beacon/client.go | 211 +++++++++++ tools/indexer/pkg/config/config.go | 33 ++ tools/indexer/pkg/database/starrock.go | 193 ++++++++++ tools/indexer/pkg/ethereum/cleint.go | 191 ++++++++++ tools/indexer/pkg/ethereum/conversions.go | 15 + tools/indexer/pkg/http/client.go | 82 ++++ tools/indexer/pkg/relay/client.go | 172 +++++++++ 11 files changed, 1657 insertions(+) create mode 100644 tools/indexer/cmd/indexer/main.go create mode 100644 tools/indexer/go.mod create mode 100644 tools/indexer/go.sum create mode 100644 tools/indexer/pkg/backfill/backfill.go create mode 100644 tools/indexer/pkg/beacon/client.go create mode 100644 tools/indexer/pkg/config/config.go create mode 100644 tools/indexer/pkg/database/starrock.go create mode 100644 tools/indexer/pkg/ethereum/cleint.go create mode 100644 tools/indexer/pkg/ethereum/conversions.go create mode 100644 tools/indexer/pkg/http/client.go create mode 100644 tools/indexer/pkg/relay/client.go diff --git a/tools/indexer/cmd/indexer/main.go b/tools/indexer/cmd/indexer/main.go new file mode 100644 index 000000000..c8527a202 --- /dev/null +++ b/tools/indexer/cmd/indexer/main.go @@ -0,0 +1,434 @@ +package main + +import ( + "context" + + "fmt" + _ "github.com/go-sql-driver/mysql" + "github.com/primev/mev-commit/indexer/pkg/backfill" + "github.com/primev/mev-commit/indexer/pkg/beacon" + "github.com/primev/mev-commit/indexer/pkg/config" + "github.com/primev/mev-commit/indexer/pkg/database" + "github.com/primev/mev-commit/indexer/pkg/ethereum" + httputil "github.com/primev/mev-commit/indexer/pkg/http" + "github.com/primev/mev-commit/indexer/pkg/relay" + "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2/altsrc" + "log" + "math/rand" + "os" + "os/signal" + "syscall" + "time" +) + +type Options struct { + BlockTick time.Duration + ValidatorWait time.Duration + BackfillEvery time.Duration + BackfillLookback int64 + BackfillBatch int + HTTPTimeout time.Duration + OptInContract string + EtherscanKey string + InfuraRPC string + BeaconBase string +} + +var ( + optionConfig = &cli.StringFlag{ + Name: "config", + Usage: "Path to config file", + EnvVars: []string{"INDEXER_CONFIG"}, + } + optionDatabaseURL = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "database-url", + Usage: "Database connection URL", + EnvVars: []string{"INDEXER_DATABASE_URL"}, + Required: true, + }) + optionOptInContract = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "opt-in-contract", + Usage: "Opt-in contract address", + EnvVars: []string{"INDEXER_OPT_IN_CONTRACT"}, + Value: "0x821798d7b9d57dF7Ed7616ef9111A616aB19ed64", + }) + optionEtherscanKey = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "etherscan-key", + Usage: "Etherscan API key", + EnvVars: []string{"INDEXER_ETHERSCAN_KEY"}, + }) + optionInfuraRPC = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "infura-rpc", + Usage: "Infura RPC URL", + EnvVars: []string{"INDEXER_INFURA_RPC"}, + Required: true, + }) + optionBeaconBase = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "beacon-base", + Usage: "Beacon API base URL", + EnvVars: []string{"INDEXER_BEACON_BASE"}, + Value: "https://beaconcha.in/api/v1", + }) + optionBlockInterval = altsrc.NewDurationFlag(&cli.DurationFlag{ + Name: "block-interval", + Usage: "interval between block processing", + EnvVars: []string{"INDEXER_BLOCK_INTERVAL"}, + Value: 12 * time.Second, + }) + + optionValidatorDelay = altsrc.NewDurationFlag(&cli.DurationFlag{ + Name: "validator-delay", + Usage: "delay before fetching validator data", + EnvVars: []string{"INDEXER_VALIDATOR_DELAY"}, + Value: 1500 * time.Millisecond, + }) + + optionBackfillEvery = altsrc.NewDurationFlag(&cli.DurationFlag{ + Name: "backfill-every", + Usage: "interval for backfill operations", + EnvVars: []string{"INDEXER_BACKFILL_EVERY"}, + Value: 5 * time.Minute, + }) + + optionBackfillLookback = altsrc.NewIntFlag(&cli.IntFlag{ + Name: "backfill-lookback", + Usage: "number of slots to look back for backfill", + EnvVars: []string{"INDEXER_BACKFILL_LOOKBACK"}, + Value: 512, + }) + + optionBackfillBatch = altsrc.NewIntFlag(&cli.IntFlag{ + Name: "backfill-batch", + Usage: "batch size for backfill operations", + EnvVars: []string{"INDEXER_BACKFILL_BATCH"}, + Value: 50, + }) + + optionHTTPTimeout = altsrc.NewDurationFlag(&cli.DurationFlag{ + Name: "http-timeout", + Usage: "HTTP client timeout", + EnvVars: []string{"INDEXER_HTTP_TIMEOUT"}, + Value: 15 * time.Second, + }) +) + +func createOptionsFromCLI(c *cli.Context) *config.Config { + return &config.Config{ + BlockTick: c.Duration("block-interval"), + ValidatorWait: c.Duration("validator-delay"), + BackfillEvery: c.Duration("backfill-every"), + BackfillLookback: int64(c.Int("backfill-lookback-slots")), + BackfillBatch: c.Int("backfill-batch"), + HTTPTimeout: c.Duration("http-timeout"), + OptInContract: c.String("opt-in-contract"), + EtherscanKey: c.String("etherscan-api-key"), + InfuraRPC: c.String("infura-rpc"), + BeaconBase: c.String("beacon-base"), + } +} + +func startIndexer(c *cli.Context) error { + + dbURL := c.String(optionDatabaseURL.Name) + infuraRPC := c.String(optionInfuraRPC.Name) + beaconBase := c.String(optionBeaconBase.Name) + // Initialize random seed + rand.Seed(time.Now().UnixNano()) + + // Load configuration + + // Validate required configuration + + log.Printf("[INIT] Starting blockchain indexer with StarRocks database") + log.Printf("[CONFIG] Block interval: %s, Validator delay: %s, Backfill every: %s", + c.Duration("block-interval"), c.Duration("validator-delay"), c.Duration("backfill-every")) + + // Setup graceful shutdown + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + // Connect to StarRocks database + db, err := database.MustConnect(ctx, dbURL, 20, 5) + if err != nil { + log.Fatalf("[DB] Connection failed: %v", err) + } + defer db.Close() + log.Printf("[DB] Connected to StarRocks database") + + // Ensure required tables exist + if err := db.EnsureStateTable(ctx); err != nil { + log.Fatalf("[DB] Failed to ensure state table: %v", err) + } + log.Printf(" [DB] State table ready") + + // Initialize HTTP client + httpc := httputil.NewHTTPClient(c.Duration("http-timeout")) + log.Printf("[HTTP] Client initialized with %s timeout", c.Duration("http-timeout")) + + // Load relay configurations + relays, err := relay.UpsertRelaysAndLoad(ctx, db) + if err != nil { + log.Fatalf("[RELAYS] Failed to load: %v", err) + } + log.Printf("[RELAYS] Loaded %d active relays:", len(relays)) + for _, r := range relays { + log.Printf(" - Relay ID %d: %s", r.ID, r.URL) + } + + // Initialize starting block number + lastBN, found := db.LoadLastBlockNumber(ctx) + if !found || lastBN == 0 { + log.Printf(" [INIT] No previous state found, checking database for latest block...") + err := db.Conn.QueryRowContext(ctx, `SELECT COALESCE(MAX(block_number),0) FROM blocks`).Scan(&lastBN) + if err != nil { + log.Printf("[INIT] Database query failed: %v", err) + } + } + + // Replace the hardcoded block search with: + if lastBN == 0 { + log.Printf("[INIT] Getting latest block from Ethereum RPC...") + + latestBlock, err := ethereum.GetLatestBlockNumber(httpc, infuraRPC) + if err != nil { + log.Fatalf("[INIT] Failed to get latest block from RPC: %v", err) + } + + lastBN = latestBlock - 10 // Start 10 blocks behind to ensure data availability + log.Printf("[INIT] Starting from block %d (latest: %d)", lastBN, latestBlock) + } + + log.Printf(" [INIT] Starting from block number: %d", lastBN) + log.Printf("[INIT] Indexer configuration - Lookback: %d slots, Batch size: %d", + c.Int("backfill-lookback-slots"), c.Int("backfill-batch")) + + // Setup tickers + backfillTicker := time.NewTicker(c.Duration("backfill-batch")) + defer backfillTicker.Stop() + + mainTicker := time.NewTicker(c.Duration("block-interval")) + defer mainTicker.Stop() + + log.Printf("🎉 [INIT] Blockchain indexer started successfully") + + // Main processing loop + for { + select { + case <-ctx.Done(): + log.Printf(" [SHUTDOWN] Graceful shutdown initiated: %v", ctx.Err()) + if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { + log.Printf("[SHUTDOWN] Failed to save last block number: %v", err) + } + log.Printf("[SHUTDOWN] Indexer stopped at block %d", lastBN) + return nil + + case <-backfillTicker.C: + log.Printf("[BACKFILL] Starting backfill operations...") + backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) + + case <-mainTicker.C: + nextBN := lastBN + 1 + + // Fetch execution block data + ei, err := beacon.FetchCombinedBlockData(httpc, infuraRPC, beaconBase, nextBN) + if err != nil || ei == nil { + log.Printf("⏳ [BLOCK] Block %d not available yet: %v", nextBN, err) + continue + } + + // Log block details + log.Printf("[BLOCK] Processing block %d → slot %d", nextBN, ei.Slot) + if ei.Timestamp != nil { + log.Printf("[BLOCK] Timestamp: %v", ei.Timestamp.Format(time.RFC3339)) + } + if ei.ProposerIdx != nil { + log.Printf("[VALIDATOR] Proposer index: %d", *ei.ProposerIdx) + } + if ei.RelayTag != nil { + log.Printf("[RELAY] Winning relay: %s", *ei.RelayTag) + } + if ei.BuilderHex != nil && len(*ei.BuilderHex) > 20 { + log.Printf("🔨 [BUILDER] Builder pubkey: %s...", (*ei.BuilderHex)[:20]) + } + if ei.RewardEth != nil { + log.Printf("[REWARD] Producer reward: %.6f ETH", *ei.RewardEth) + } + + // Save block to database + if err := db.UpsertBlockFromExec(ctx, ei); err != nil { + log.Printf("[DB] Failed to save block %d: %v", nextBN, err) + continue + } + log.Printf("[DB] Block %d saved successfully", nextBN) + + // Fetch and store bid data from all relays + totalBids := 0 + successfulRelays := 0 + mainContextCanceled := false + + for _, rr := range relays { + // Check if main context is canceled before processing each relay + if ctx.Err() != nil { + log.Printf("[BIDS] Main context canceled, stopping relay processing") + mainContextCanceled = true + break + } + + bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, ei.Slot) + if err != nil { + log.Printf(" [BIDS] Relay %d (%s) failed: %v", rr.ID, rr.URL, err) + continue + } + + relayBids := 0 + // Create a separate context with timeout for bid insertions + bidCtx, bidCancel := context.WithTimeout(context.Background(), 30*time.Second) + + for _, bid := range bids { + // Check if main context is still valid + if ctx.Err() != nil { + log.Printf("[BIDS] Main context canceled, stopping bid insertion") + mainContextCanceled = true + break + } + + if err := relay.InsertBid(bidCtx, db, ei.Slot, rr.ID, bid); err != nil { + log.Printf(" [BIDS] Failed to insert bid for slot %d, relay %d: %v", ei.Slot, rr.ID, err) + } else { + relayBids++ + } + } + bidCancel() + + if mainContextCanceled { + break + } + + if relayBids > 0 { + log.Printf(" [BIDS] Relay %d: %d bids collected", rr.ID, relayBids) + totalBids += relayBids + successfulRelays++ + } + } + + log.Printf(" [SUMMARY] Block %d: %d bids from %d relays", nextBN, totalBids, successfulRelays) + // Async validator pubkey fetch + if ei.ProposerIdx != nil { + go func(slot int64, proposerIdx int64) { + time.Sleep(c.Duration("validator-delay")) + + vpub, err := beacon.FetchValidatorPubkey(httpc, beaconBase, proposerIdx) + if err != nil { + log.Printf("[VALIDATOR] Failed to fetch pubkey for proposer %d: %v", proposerIdx, err) + return + } + + if len(vpub) > 0 { + if err := db.UpdateValidatorPubkey(context.Background(), slot, vpub); err != nil { + log.Printf(" [VALIDATOR] Failed to save pubkey for slot %d: %v", slot, err) + } else { + log.Printf("[VALIDATOR] Pubkey saved for proposer %d (slot %d)", proposerIdx, slot) + } + } + }(ei.Slot, *ei.ProposerIdx) + } + + // Async opt-in status check + if ei.ProposerIdx != nil { + go func(slot int64, blockNumber int64) { + time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) + + // Wait for validator pubkey to be available + var vpk []byte + retries := 3 + for i := 0; i < retries; i++ { + err := db.Conn.QueryRowContext(context.Background(), + `SELECT validator_pubkey FROM blocks WHERE slot=?`, slot).Scan(&vpk) + if err == nil && len(vpk) > 0 { + break + } + if i < retries-1 { + time.Sleep(time.Second) + } + } + + if len(vpk) == 0 { + log.Printf("[OPT-IN] Validator pubkey not available for slot %d", slot) + return + } + + opted, err := ethereum.CallAreOptedInAtBlock(httpc, createOptionsFromCLI(c), blockNumber, vpk) + if err != nil { + log.Printf("[OPT-IN] Failed to check opt-in status for slot %d: %v", slot, err) + return + } + + _, err = db.Conn.ExecContext(context.Background(), + `UPDATE blocks SET validator_opted_in=? WHERE slot=?`, opted, slot) + if err != nil { + log.Printf("[OPT-IN] Failed to save opt-in status for slot %d: %v", slot, err) + } else { + log.Printf("[OPT-IN] Slot %d validator opted-in: %t", slot, opted) + } + }(ei.Slot, ei.BlockNumber) + } + + lastBN = nextBN + if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { + log.Printf("[PROGRESS] Failed to save block number %d: %v", lastBN, err) + } else { + log.Printf("[PROGRESS] Advanced to block %d", lastBN) + } + } + } +} + +func main() { + flags := []cli.Flag{ + optionConfig, + optionDatabaseURL, + optionInfuraRPC, + optionBeaconBase, + optionBlockInterval, + optionValidatorDelay, + optionBackfillEvery, + optionBackfillLookback, + optionBackfillBatch, + optionHTTPTimeout, + optionOptInContract, + optionEtherscanKey, + } + + app := &cli.App{ + Name: "mev-indexer", + Usage: "Builder/observer indexer", + Commands: []*cli.Command{{ + Name: "start", + Usage: "Start the indexer", + Flags: flags, + Before: altsrc.InitInputSourceWithContext( + flags, altsrc.NewYamlSourceFromFlagFunc("config"), + ), + Action: func(c *cli.Context) error { + return startIndexer(c) + }, + }}, + } + ctx, cancel := context.WithCancel(context.Background()) + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigc + _, _ = fmt.Fprintln(app.Writer, "received interrupt signal, exiting... Force exit with Ctrl+C") + cancel() + <-sigc + _, _ = fmt.Fprintln(app.Writer, "force exiting...") + os.Exit(1) + }() + + if err := app.RunContext(ctx, os.Args); err != nil { + _, _ = fmt.Fprintf(app.Writer, "exited with error: %v\n", err) + } + +} diff --git a/tools/indexer/go.mod b/tools/indexer/go.mod new file mode 100644 index 000000000..99cd44ee1 --- /dev/null +++ b/tools/indexer/go.mod @@ -0,0 +1,28 @@ +module github.com/primev/mev-commit/indexer + +go 1.23.0 + +require ( + github.com/ethereum/go-ethereum v1.16.3 + github.com/go-sql-driver/mysql v1.9.3 + github.com/urfave/cli/v2 v2.27.5 +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect +) + +require ( + filippo.io/edwards25519 v1.1.0 // indirect + github.com/BurntSushi/toml v1.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/holiman/uint256 v1.3.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sys v0.32.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/tools/indexer/go.sum b/tools/indexer/go.sum new file mode 100644 index 000000000..1c90dec19 --- /dev/null +++ b/tools/indexer/go.sum @@ -0,0 +1,48 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/ethereum/go-ethereum v1.16.3 h1:nDoBSrmsrPbrDIVLTkDQCy1U9KdHN+F2PzvMbDoS42Q= +github.com/ethereum/go-ethereum v1.16.3/go.mod h1:Lrsc6bt9Gm9RyvhfFK53vboCia8kpF9nv+2Ukntnl+8= +github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= +github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= +github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go new file mode 100644 index 000000000..01c5ccba8 --- /dev/null +++ b/tools/indexer/pkg/backfill/backfill.go @@ -0,0 +1,250 @@ +// pkg/backfill/backfill.go +package backfill + +import ( + "context" + "log" + "net/http" + "time" + + "github.com/primev/mev-commit/indexer/pkg/beacon" + "github.com/primev/mev-commit/indexer/pkg/config" + "github.com/primev/mev-commit/indexer/pkg/database" + "github.com/primev/mev-commit/indexer/pkg/ethereum" + "github.com/primev/mev-commit/indexer/pkg/relay" +) + +// RecentMissing backfills recent blocks that are missing data +func RecentMissing(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, lookback int64, batch int) error { + + rows, err := db.Conn.QueryContext(ctx, ` +WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) +SELECT slot, block_number +FROM blocks, recent +WHERE slot > recent.s - ? + AND block_number IS NOT NULL + AND (winning_relay IS NULL + OR winning_builder_pubkey IS NULL + OR fee_recipient IS NULL + OR producer_reward_eth IS NULL + OR timestamp IS NULL + OR proposer_index IS NULL) +ORDER BY slot DESC +LIMIT ?`, lookback, batch) + if err != nil { + log.Printf("[BACKFILL] RecentMissing query failed: %v", err) + return err + } + defer rows.Close() + + processed := 0 + for rows.Next() { + var slot int64 + var bn int64 + if err := rows.Scan(&slot, &bn); err != nil { + log.Printf("[BACKFILL] RecentMissing scan failed: %v", err) + continue + } + + // Fetch beacon execution block data + if ei, err := beacon.FetchBeaconExecutionBlock(httpc, cfg.BeaconBase, bn); err == nil && ei != nil { + if err := db.UpsertBlockFromExec(ctx, ei); err != nil { + log.Printf("[BACKFILL] RecentMissing upsert failed for slot %d: %v", slot, err) + continue + } + + // Schedule async validator pubkey fetch + if ei.ProposerIdx != nil { + go func(slot int64, idx int64) { + time.Sleep(cfg.ValidatorWait) + if vpub, err := beacon.FetchValidatorPubkey(httpc, cfg.BeaconBase, idx); err == nil && len(vpub) > 0 { + if err := db.UpdateValidatorPubkey(context.Background(), slot, vpub); err != nil { + log.Printf("[BACKFILL] RecentMissing validator pubkey update failed for slot %d: %v", slot, err) + } + } + }(ei.Slot, *ei.ProposerIdx) + } + processed++ + } else { + log.Printf("[BACKFILL] RecentMissing beacon fetch failed for block %d: %v", bn, err) + } + } + + if err := rows.Err(); err != nil { + log.Printf("[BACKFILL] RecentMissing rows iteration failed: %v", err) + return err + } + + log.Printf("[BACKFILL] RecentMissing processed %d blocks", processed) + return nil +} + +// RecentBids backfills bid data for ALL recent slots (not just opted-in blocks) +func RecentBids(ctx context.Context, db *database.DB, httpc *http.Client, relays []relay.Row, lookback int64, batch int) error { + + rows, err := db.Conn.QueryContext(ctx, ` +WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) +SELECT DISTINCT slot +FROM blocks, recent +WHERE slot > recent.s - ? + AND block_number IS NOT NULL +ORDER BY slot DESC +LIMIT ?`, lookback, batch) + if err != nil { + log.Printf("[BACKFILL] RecentBids query failed: %v", err) + return err + } + defer rows.Close() + + processed := 0 + totalBids := 0 + for rows.Next() { + var slot int64 + if err := rows.Scan(&slot); err != nil { + log.Printf("[BACKFILL] RecentBids scan failed: %v", err) + continue + } + + // Fetch bids from ALL relays for this slot + slotBids := 0 + for _, rr := range relays { + if bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, slot); err == nil { + for _, b := range bids { + if err := relay.InsertBid(ctx, db, slot, rr.ID, b); err != nil { + log.Printf("[BACKFILL] RecentBids insert failed for slot %d, relay %d: %v", slot, rr.ID, err) + } else { + slotBids++ + } + } + } else { + log.Printf("[BACKFILL] RecentBids fetch failed for slot %d, relay %d (%s): %v", slot, rr.ID, rr.URL, err) + } + } + + if slotBids > 0 { + totalBids += slotBids + processed++ + } + } + + if err := rows.Err(); err != nil { + log.Printf("[BACKFILL] RecentBids rows iteration failed: %v", err) + return err + } + + log.Printf("[BACKFILL] RecentBids processed %d slots with %d total bids from ALL blocks", processed, totalBids) + return nil +} + +// ValidatorOptIn backfills validator opt-in status (this is opt-in specific data) +func ValidatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, lookback int64, batch int) error { + + rows, err := db.Conn.QueryContext(ctx, ` +WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) +SELECT slot, block_number, validator_pubkey +FROM blocks, recent +WHERE slot > recent.s - ? + AND block_number IS NOT NULL + AND validator_pubkey IS NOT NULL + AND validator_opted_in IS NULL +ORDER BY slot DESC +LIMIT ?`, lookback, batch) + if err != nil { + log.Printf("[BACKFILL] ValidatorOptIn query failed: %v", err) + return err + } + defer rows.Close() + + processed := 0 + for rows.Next() { + var slot, bn int64 + var vpk []byte + if err := rows.Scan(&slot, &bn, &vpk); err != nil { + log.Printf("[BACKFILL] ValidatorOptIn scan failed: %v", err) + continue + } + + // Check opt-in status + opted, err := ethereum.CallAreOptedInAtBlock(httpc, cfg, bn, vpk) + if err == nil { + updateCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + // Fixed query parameter style for StarRocks + if _, err := db.Conn.ExecContext(updateCtx, `UPDATE blocks SET validator_opted_in=? WHERE slot=? AND validator_opted_in IS NULL`, opted, slot); err != nil { + log.Printf("[BACKFILL] ValidatorOptIn update failed for slot %d: %v", slot, err) + } else { + processed++ + } + cancel() + } else { + log.Printf("[BACKFILL] ValidatorOptIn check failed for slot %d: %v", slot, err) + } + } + + if err := rows.Err(); err != nil { + log.Printf("[BACKFILL] ValidatorOptIn rows iteration failed: %v", err) + return err + } + + log.Printf("[BACKFILL] ValidatorOptIn processed %d validators", processed) + return nil +} + +// AllBlocksBids ensures bid data is collected for ALL blocks, regardless of opt-in status +func AllBlocksBids(ctx context.Context, db *database.DB, httpc *http.Client, relays []relay.Row, startSlot, endSlot int64) error { + log.Printf("[BACKFILL] AllBlocksBids ensuring bid coverage from slot %d to %d", startSlot, endSlot) + + totalProcessed := 0 + totalBids := 0 + + for slot := startSlot; slot <= endSlot; slot++ { + slotBids := 0 + + // Fetch bids from ALL relays for every single slot + for _, rr := range relays { + if bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, slot); err == nil { + for _, b := range bids { + if err := relay.InsertBid(ctx, db, slot, rr.ID, b); err == nil { + slotBids++ + } + } + } + } + + if slotBids > 0 { + totalBids += slotBids + totalProcessed++ + } + + // Respect context cancellation + select { + case <-ctx.Done(): + log.Printf("[BACKFILL] AllBlocksBids cancelled at slot %d", slot) + return ctx.Err() + default: + } + } + + log.Printf("[BACKFILL] AllBlocksBids completed: %d slots processed, %d total bids collected", totalProcessed, totalBids) + return nil +} + +// RunAll executes all backfill operations ensuring complete coverage +func RunAll(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, relays []relay.Row) { + log.Printf("[BACKFILL] Starting comprehensive backfill for ALL blocks (not just opted-in)") + + // Run backfill operations + if err := RecentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + log.Printf("[BACKFILL] RecentMissing failed: %v", err) + } + + if err := ValidatorOptIn(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + log.Printf("[BACKFILL] ValidatorOptIn failed: %v", err) + } + + // This ensures bid data for ALL blocks, not just mev-commit opted-in blocks + if err := RecentBids(ctx, db, httpc, relays, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + log.Printf("[BACKFILL] RecentBids failed: %v", err) + } + + log.Printf("[BACKFILL] All operations completed - relay data covers ALL blocks") +} diff --git a/tools/indexer/pkg/beacon/client.go b/tools/indexer/pkg/beacon/client.go new file mode 100644 index 000000000..d4e90e74f --- /dev/null +++ b/tools/indexer/pkg/beacon/client.go @@ -0,0 +1,211 @@ +package beacon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math/big" + "net/http" + "strconv" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/primev/mev-commit/indexer/pkg/ethereum" + httputil "github.com/primev/mev-commit/indexer/pkg/http" +) + +type ExecInfo struct { + BlockNumber int64 + Slot int64 + ProposerIdx *int64 + Timestamp *time.Time + RelayTag *string + BuilderHex *string + FeeRecHex *string + RewardEth *float64 +} + +func FetchBeaconExecutionBlock(httpc *http.Client, beaconBase string, blockNum int64) (*ExecInfo, error) { + url := fmt.Sprintf("%s/execution/block/%d", beaconBase, blockNum) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var wrap struct { + Data []map[string]any `json:"data"` + } + if err := httputil.FetchJSONWithRetry(ctx, httpc, url, &wrap, 3, 300*time.Millisecond); err != nil || len(wrap.Data) == 0 { + return nil, fmt.Errorf("no exec block %d", blockNum) + } + j := wrap.Data[0] + out := &ExecInfo{BlockNumber: blockNum} + + // posConsensus.slot & proposerIndex + if pc, ok := j["posConsensus"].(map[string]any); ok { + if v, ok := pc["slot"].(float64); ok { + out.Slot = int64(v) + } else if s, ok := pc["slot"].(string); ok { + if n, err := strconv.ParseInt(s, 10, 64); err == nil { + out.Slot = n + } + } + if v, ok := pc["proposerIndex"].(float64); ok { + x := int64(v) + out.ProposerIdx = &x + } else if s, ok := pc["proposerIndex"].(string); ok { + if n, err := strconv.ParseInt(s, 10, 64); err == nil { + out.ProposerIdx = &n + } + } + } + + // timestamp + if v, ok := j["timestamp"]; ok { + switch t := v.(type) { + case float64: + u := time.Unix(int64(t), 0).UTC() + out.Timestamp = &u + case string: + if n, err := strconv.ParseInt(t, 10, 64); err == nil { + u := time.Unix(n, 0).UTC() + out.Timestamp = &u + } + } + } + + // relay + if rel, ok := j["relay"].(map[string]any); ok { + if s, ok := rel["tag"].(string); ok { + out.RelayTag = &s + } + if s, ok := rel["builderPubkey"].(string); ok { + out.BuilderHex = &s + } + if s, ok := rel["producerFeeRecipient"].(string); ok { + out.FeeRecHex = &s + } + } + + // reward eth from blockMevReward or producerReward + if v, ok := j["blockMevReward"]; ok { + switch t := v.(type) { + case float64: + f := t + if f > 1e10 { + f = f / 1e18 // wei -> ETH + } + out.RewardEth = &f + case string: + if strings.HasPrefix(t, "0x") { + if bi, ok := new(big.Int).SetString(t[2:], 16); ok { + f, _ := new(big.Rat).SetFrac(bi, big.NewInt(1e18)).Float64() + out.RewardEth = &f + } + } else if f, err := strconv.ParseFloat(t, 64); err == nil { + out.RewardEth = &f + } + } + } else if v, ok := j["producerReward"]; ok { + if f, ok := v.(float64); ok { + out.RewardEth = &f + } + } + + // sanity + if out.Slot == 0 { + return nil, fmt.Errorf("exec block missing posConsensus.slot for %d", blockNum) + } + return out, nil +} + +// validator pubkey from proposer index +func FetchValidatorPubkey(httpc *http.Client, beaconBase string, proposerIndex int64) ([]byte, error) { + url := fmt.Sprintf("%s/validator/%d", beaconBase, proposerIndex) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var resp struct { + Data struct { + Pubkey string `json:"pubkey"` + } `json:"data"` + } + if err := httputil.FetchJSONWithRetry(ctx, httpc, url, &resp, 3, 300*time.Millisecond); err != nil { + return nil, err + } + if strings.TrimSpace(resp.Data.Pubkey) == "" { + return nil, fmt.Errorf("validator %d pubkey empty", proposerIndex) + } + return common.FromHex(resp.Data.Pubkey), nil +} + +// Add this new function to fetch blocks from Alchemy RPC +func FetchBlockFromRPC(httpc *http.Client, rpcURL string, blockNumber int64) (*ExecInfo, error) { + // Get block data from Alchemy + payload := map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_getBlockByNumber", + "params": []any{fmt.Sprintf("0x%x", blockNumber), true}, // true for full transaction objects + } + + buf, _ := json.Marshal(payload) + req, _ := http.NewRequest("POST", rpcURL, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + + resp, err := httpc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result struct { + Result struct { + Number string `json:"number"` + Timestamp string `json:"timestamp"` + Miner string `json:"miner"` + } `json:"result"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + if result.Result.Number == "" { + return nil, fmt.Errorf("block not found") + } + + // Convert hex timestamp to time + timestampHex := result.Result.Timestamp[2:] // Remove 0x + timestamp, _ := strconv.ParseInt(timestampHex, 16, 64) + blockTime := time.Unix(timestamp, 0) + + return &ExecInfo{ + BlockNumber: blockNumber, + Timestamp: &blockTime, + }, nil +} +func FetchCombinedBlockData(httpc *http.Client, rpcURL string, beaconBase string, blockNumber int64) (*ExecInfo, error) { + // Get execution block from Alchemy (always available) + execBlock, err := FetchBlockFromRPC(httpc, rpcURL, blockNumber) + if err != nil { + return nil, err + } + + // Convert block number to slot for beacon chain query + slotNumber := ethereum.BlockNumberToSlot(blockNumber) + + // Try to get beacon chain data using slot number (may not exist for recent blocks) + beaconData, _ := FetchBeaconExecutionBlock(httpc, beaconBase, slotNumber) + + // Merge data - use Alchemy as primary, beacon as supplement + if beaconData != nil { + execBlock.Slot = beaconData.Slot + execBlock.ProposerIdx = beaconData.ProposerIdx + execBlock.RelayTag = beaconData.RelayTag + execBlock.RewardEth = beaconData.RewardEth + } else { + // Set the calculated slot if beacon data not available + execBlock.Slot = slotNumber + } + + return execBlock, nil +} diff --git a/tools/indexer/pkg/config/config.go b/tools/indexer/pkg/config/config.go new file mode 100644 index 000000000..91d3cf543 --- /dev/null +++ b/tools/indexer/pkg/config/config.go @@ -0,0 +1,33 @@ +package config + +import ( + "time" +) + +type Relay struct { + Name string + Tag string + URL string +} + +var RelaysDefault = []Relay{ + {Name: "Titan", Tag: "titan-relay", URL: "https://regional.titanrelay.xyz"}, + {Name: "Aestus", Tag: "aestus-relay", URL: "https://aestus.live"}, + {Name: "Bloxroute Max Profit", Tag: "bloxroute-max-profit-relay", URL: "https://bloxroute.max-profit.blxrbdn.com"}, + {Name: "Bloxroute Regulated", Tag: "bloxroute-regulated-relay", URL: "https://bloxroute.regulated.blxrbdn.com"}, +} + +type Config struct { + BlockTick time.Duration + ValidatorWait time.Duration + BackfillEvery time.Duration + BackfillLookback int64 + BackfillBatch int + MaxRetries int + BaseRetryDelay time.Duration + HTTPTimeout time.Duration + OptInContract string + EtherscanKey string + InfuraRPC string + BeaconBase string +} diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go new file mode 100644 index 000000000..bdef445b0 --- /dev/null +++ b/tools/indexer/pkg/database/starrock.go @@ -0,0 +1,193 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + _ "github.com/go-sql-driver/mysql" + "github.com/primev/mev-commit/indexer/pkg/beacon" + "github.com/primev/mev-commit/indexer/pkg/config" + "strings" + "time" +) + +type DB struct { + Conn *sql.DB +} + +func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, error) { + conn, err := sql.Open("mysql", dsn) + if err != nil { + return nil, fmt.Errorf("failed to open StarRocks connection: %w", err) + } + + // Configure connection pool + conn.SetMaxOpenConns(maxConns) + conn.SetMaxIdleConns(minConns) + conn.SetConnMaxLifetime(time.Hour) + conn.SetConnMaxIdleTime(30 * time.Minute) + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := conn.PingContext(pingCtx); err != nil { + conn.Close() + return nil, fmt.Errorf("StarRocks ping failed: %v", err) + } + + return &DB{Conn: conn}, nil + +} +func (db *DB) Close() { + db.Conn.Close() +} + +func (db *DB) EnsureStateTable(ctx context.Context) error { + ctx2, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + ddl := ` + CREATE TABLE IF NOT EXISTS ingestor_state ( + id TINYINT, + last_block_number BIGINT + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + )` + + if _, err := db.Conn.ExecContext(ctx2, ddl); err != nil { + return fmt.Errorf("failed to create state table: %w", err) + } + + var count int + err := db.Conn.QueryRowContext(ctx2, `SELECT COUNT(*) FROM ingestor_state WHERE id = 1`).Scan(&count) + if err != nil || count == 0 { + _, err = db.Conn.ExecContext(ctx2, + `INSERT INTO ingestor_state (id, last_block_number) VALUES (1, 0)`) + if err != nil { + return fmt.Errorf("failed to insert initial state: %w", err) + } + } + + return nil +} + +func (db *DB) LoadLastBlockNumber(ctx context.Context) (int64, bool) { + ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var bn int64 + err := db.Conn.QueryRowContext(ctx2, + `SELECT last_block_number FROM ingestor_state WHERE id = 1 LIMIT 1`).Scan(&bn) + if err != nil { + return 0, false + } + return bn, true +} + +func (db *DB) SaveLastBlockNumber(ctx context.Context, bn int64) error { + ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + _, err := db.Conn.ExecContext(ctx2, `DELETE FROM ingestor_state WHERE id = 1`) + if err != nil { + return fmt.Errorf("failed to delete old state: %w", err) + } + + query := fmt.Sprintf(`INSERT INTO ingestor_state (id, last_block_number) VALUES (1, %d)`, bn) + _, err = db.Conn.ExecContext(ctx2, query) + if err != nil { + return fmt.Errorf("save last_block_number failed: %w", err) + } + return nil +} + +func (db *DB) UpsertRelays(ctx context.Context, relays []config.Relay) error { + if len(relays) == 0 { + return nil + } + + ctx2, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + // StarRocks batch insert approach + var values []string + for _, r := range relays { + value := fmt.Sprintf("('%s', '%s', '%s', 1)", r.Name, r.Tag, r.URL) + values = append(values, value) + } + + query := fmt.Sprintf(`INSERT INTO relays (name, tag, base_url, is_active) VALUES %s`, + strings.Join(values, ",")) + + _, err := db.Conn.ExecContext(ctx2, query) + return err +} + +func (db *DB) UpsertBlockFromExec(ctx context.Context, ei *beacon.ExecInfo) error { + if ei == nil || ei.BlockNumber == 0 { + return fmt.Errorf("upsert block: nil exec info or block_number=0") + } + + ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + var timestamp, proposerIndex, relayTag, rewardEth string + + if ei.Timestamp != nil { + timestamp = fmt.Sprintf("'%s'", ei.Timestamp.Format("2006-01-02 15:04:05")) + } else { + timestamp = "NULL" + } + + if ei.ProposerIdx != nil { + proposerIndex = fmt.Sprintf("%d", *ei.ProposerIdx) + } else { + proposerIndex = "NULL" + } + + if ei.RelayTag != nil { + relayTag = fmt.Sprintf("'%s'", *ei.RelayTag) + } else { + relayTag = "NULL" + } + + if ei.RewardEth != nil { + rewardEth = fmt.Sprintf("%.6f", *ei.RewardEth) + } else { + rewardEth = "NULL" + } + + query := fmt.Sprintf(` +INSERT INTO blocks( + slot, block_number, timestamp, proposer_index, + winning_relay, producer_reward_eth +) VALUES (%d, %d, %s, %s, %s, %s)`, + ei.Slot, ei.BlockNumber, timestamp, proposerIndex, relayTag, rewardEth) + + _, err := db.Conn.ExecContext(ctx2, query) + if err != nil { + return fmt.Errorf("upsert block slot=%d: %w", ei.Slot, err) + } + return nil +} + +func (db *DB) UpdateValidatorPubkey(ctx context.Context, slot int64, vpub []byte) error { + if slot == 0 { + return fmt.Errorf("update validator: slot=0") + } + if len(vpub) == 0 { + return nil + } + + ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + _, err := db.Conn.ExecContext(ctx2, ` + UPDATE blocks SET validator_pubkey = ? WHERE slot = ?`, + vpub, slot) + if err != nil { + return fmt.Errorf("update validator slot=%d: %w", slot, err) + } + return nil +} diff --git a/tools/indexer/pkg/ethereum/cleint.go b/tools/indexer/pkg/ethereum/cleint.go new file mode 100644 index 000000000..f25b90d65 --- /dev/null +++ b/tools/indexer/pkg/ethereum/cleint.go @@ -0,0 +1,191 @@ +package ethereum + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/primev/mev-commit/indexer/pkg/config" + "io" + "net/http" + "strconv" + "strings" + "time" + + httputil "github.com/primev/mev-commit/indexer/pkg/http" +) + +const optInABIJSON = `[ + { + "inputs":[{"internalType":"bytes[]","name":"valBLSPubKeys","type":"bytes[]"}], + "name":"areValidatorsOptedIn", + "outputs":[{"components":[ + {"internalType":"bool","name":"isVanillaOptedIn","type":"bool"}, + {"internalType":"bool","name":"isAvsOptedIn","type":"bool"}, + {"internalType":"bool","name":"isMiddlewareOptedIn","type":"bool"} + ],"internalType":"struct OptInStatus[]","name":"","type":"tuple[]"}], + "stateMutability":"view","type":"function" + } +]` + +func BuildAreOptedInCallData(pubkey []byte) ([]byte, error) { + ab, err := abi.JSON(strings.NewReader(optInABIJSON)) + if err != nil { + return nil, err + } + // Solidity expects bytes[]; pass []byte{pubkey} length 1 + return ab.Pack("areValidatorsOptedIn", [][]byte{pubkey}) +} + +// JSON-RPC helper (Infura / Alchemy / any node) +func EthCallJSONRPC(httpc *http.Client, rpcURL string, to string, data []byte, blockNum int64) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + tag := "0x" + strconv.FormatInt(blockNum, 16) + payload := map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_call", + "params": []any{ + map[string]any{ + "to": to, + "data": "0x" + hex.EncodeToString(data), + }, + tag, + }, + } + buf, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("eth_call marshal: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, rpcURL, bytes.NewReader(buf)) + if err != nil { + return nil, fmt.Errorf("build eth_call request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := httpc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("eth_call http %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var out struct { + Result string `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + if out.Error != nil { + return nil, fmt.Errorf("eth_call rpc error %d: %s", out.Error.Code, out.Error.Message) + } + if out.Result == "" || out.Result == "0x" { + return nil, fmt.Errorf("eth_call empty result (err=%v)", out.Error) + } + return hex.DecodeString(strings.TrimPrefix(out.Result, "0x")) +} + +func CallAreOptedInAtBlock(httpc *http.Client, cfg *config.Config, blockNum int64, pubkey []byte) (bool, error) { + if len(pubkey) == 0 { + return false, fmt.Errorf("empty pubkey") + } + data, err := BuildAreOptedInCallData(pubkey) + if err != nil { + return false, err + } + + var ret []byte + if cfg.InfuraRPC != "" { + // Preferred: direct JSON-RPC via Infura + ret, err = EthCallJSONRPC(httpc, cfg.InfuraRPC, cfg.OptInContract, data, blockNum) + if err != nil { + return false, err + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // Fallback: Etherscan proxy + tag := "0x" + strconv.FormatInt(blockNum, 16) + url := fmt.Sprintf("https://api.etherscan.io/api?module=proxy&action=eth_call&to=%s&data=0x%s&tag=%s", + cfg.OptInContract, hex.EncodeToString(data), tag) + if cfg.EtherscanKey != "" { + url += "&apikey=" + cfg.EtherscanKey + } + var resp struct { + Result string `json:"result"` + } + if err := httputil.FetchJSONWithRetry(ctx, httpc, url, &resp, cfg.MaxRetries, cfg.BaseRetryDelay); err != nil { + return false, err + } + if resp.Result == "" || resp.Result == "0x" { + return false, fmt.Errorf("empty result") + } + ret, err = hex.DecodeString(strings.TrimPrefix(resp.Result, "0x")) + if err != nil { + return false, err + } + } + + // Decode OptInStatus[] where OptInStatus=(bool,bool,bool) + ab, err := abi.JSON(strings.NewReader(optInABIJSON)) + if err != nil { + return false, err + } + var out []struct { + IsVanillaOptedIn bool + IsAvsOptedIn bool + IsMiddlewareOptedIn bool + } + if err := ab.UnpackIntoInterface(&out, "areValidatorsOptedIn", ret); err != nil { + return false, err + } + if len(out) == 0 { + return false, nil + } + o := out[0] + return o.IsVanillaOptedIn || o.IsAvsOptedIn || o.IsMiddlewareOptedIn, nil +} + +// GetLatestBlockNumber gets the latest block number from Ethereum RPC +func GetLatestBlockNumber(httpc *http.Client, rpcURL string) (int64, error) { + payload := map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_blockNumber", + "params": []any{}, + } + + buf, _ := json.Marshal(payload) + req, _ := http.NewRequest("POST", rpcURL, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + + resp, err := httpc.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + var result struct { + Result string `json:"result"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return 0, err + } + + // Convert hex to int64 + blockNum, err := strconv.ParseInt(result.Result[2:], 16, 64) + return blockNum, err +} diff --git a/tools/indexer/pkg/ethereum/conversions.go b/tools/indexer/pkg/ethereum/conversions.go new file mode 100644 index 000000000..8f4fffff6 --- /dev/null +++ b/tools/indexer/pkg/ethereum/conversions.go @@ -0,0 +1,15 @@ +// pkg/utils/conversions.go +package ethereum + +func BlockNumberToSlot(blockNumber int64) int64 { + // Ethereum mainnet merge happened at slot 4700013 (block 15537394) + const MERGE_BLOCK = 15537394 + const MERGE_SLOT = 4700013 + + if blockNumber < MERGE_BLOCK { + return 0 // Pre-merge blocks don't have valid slots + } + + // Post-merge: roughly 1 slot per block + return MERGE_SLOT + (blockNumber - MERGE_BLOCK) +} diff --git a/tools/indexer/pkg/http/client.go b/tools/indexer/pkg/http/client.go new file mode 100644 index 000000000..04ebe3524 --- /dev/null +++ b/tools/indexer/pkg/http/client.go @@ -0,0 +1,82 @@ +package http + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "net" + "net/http" + "strconv" + "time" +) + +func NewHTTPClient(timeout time.Duration) *http.Client { + return &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, + }, + } +} +func FetchJSONWithRetry(ctx context.Context, httpc *http.Client, url string, out any, attempts int, baseDelay time.Duration) error { + var lastErr error + + for i := 0; i < attempts; i++ { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + lastErr = err + continue + } + resp, err := httpc.Do(req) + if err == nil && resp != nil && resp.StatusCode == 200 { + defer resp.Body.Close() + return json.NewDecoder(resp.Body).Decode(out) + } + if resp != nil { + // 429 courtesy backoff if provided + if resp.StatusCode == 429 { + if ra := resp.Header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(ra); err == nil { + select { + case <-ctx.Done(): + resp.Body.Close() + return ctx.Err() + + case <-time.After(time.Duration(secs) * time.Second): + } + + } + } + } + if resp.StatusCode != http.StatusOK { + lastErr = fmt.Errorf("GET %s: status %d", url, resp.StatusCode) + } + resp.Body.Close() + } else if err != nil { + lastErr = err + } + if i < attempts-1 { + sleep := baseDelay * time.Duration(1< Date: Mon, 29 Sep 2025 20:30:08 +0530 Subject: [PATCH 02/18] Fix StarRocks compatibility and migrate to slog --- go.work.sum | 7 +- tools/go.mod | 3 + tools/go.sum | 6 + tools/indexer/go.mod | 28 --- tools/indexer/go.sum | 48 ---- tools/indexer/{cmd/indexer => }/main.go | 197 ++++++++------- tools/indexer/pkg/backfill/backfill.go | 170 +++++-------- tools/indexer/pkg/beacon/client.go | 25 +- tools/indexer/pkg/config/config.go | 15 +- tools/indexer/pkg/database/starrock.go | 284 ++++++++++++++++++++-- tools/indexer/pkg/ethereum/cleint.go | 191 --------------- tools/indexer/pkg/ethereum/client.go | 76 ++++++ tools/indexer/pkg/ethereum/conversions.go | 1 - tools/indexer/pkg/http/client.go | 87 ++----- tools/indexer/pkg/relay/client.go | 50 ++-- 15 files changed, 557 insertions(+), 631 deletions(-) delete mode 100644 tools/indexer/go.mod delete mode 100644 tools/indexer/go.sum rename tools/indexer/{cmd/indexer => }/main.go (59%) delete mode 100644 tools/indexer/pkg/ethereum/cleint.go create mode 100644 tools/indexer/pkg/ethereum/client.go diff --git a/go.work.sum b/go.work.sum index 86ab8f23c..7e9c788a8 100644 --- a/go.work.sum +++ b/go.work.sum @@ -818,6 +818,7 @@ github.com/chromedp/chromedp v0.9.2 h1:dKtNz4kApb06KuSXoTQIyUC2TrA0fhGDwNZf3bcgf github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= @@ -826,6 +827,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= @@ -1337,8 +1339,6 @@ github.com/iris-contrib/pongo2 v0.0.1 h1:zGP7pW51oi5eQZMIlGA3I+FHY9/HOQWDB+572yi github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY= @@ -2131,7 +2131,6 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2204,7 +2203,6 @@ golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 h1:zf5N6UOrA487eEFacMePxjXAJctxKmyjKUsjA11Uzuk= @@ -2222,6 +2220,7 @@ golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/tools/go.mod b/tools/go.mod index 131ff8d50..c210fb0b8 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -23,6 +23,7 @@ require ( require ( github.com/DATA-DOG/go-sqlmock v1.5.2 + github.com/go-sql-driver/mysql v1.9.3 github.com/google/go-cmp v0.6.0 github.com/testcontainers/testcontainers-go v0.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 @@ -31,7 +32,9 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/containerd/containerd v1.7.11 // indirect diff --git a/tools/go.sum b/tools/go.sum index dcdab0538..8f8bd4347 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -2,10 +2,14 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-2024022118033 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.32.0-20240221180331-f05a6f4403ce.1/go.mod h1:tiTMKD8j6Pd/D2WzREoweufjzaJKHZg35f/VGcZ2v3I= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= @@ -109,6 +113,8 @@ github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= +github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/tools/indexer/go.mod b/tools/indexer/go.mod deleted file mode 100644 index 99cd44ee1..000000000 --- a/tools/indexer/go.mod +++ /dev/null @@ -1,28 +0,0 @@ -module github.com/primev/mev-commit/indexer - -go 1.23.0 - -require ( - github.com/ethereum/go-ethereum v1.16.3 - github.com/go-sql-driver/mysql v1.9.3 - github.com/urfave/cli/v2 v2.27.5 -) - -require ( - github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect -) - -require ( - filippo.io/edwards25519 v1.1.0 // indirect - github.com/BurntSushi/toml v1.5.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect - github.com/holiman/uint256 v1.3.2 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/sys v0.32.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/tools/indexer/go.sum b/tools/indexer/go.sum deleted file mode 100644 index 1c90dec19..000000000 --- a/tools/indexer/go.sum +++ /dev/null @@ -1,48 +0,0 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= -github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= -github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/ethereum/go-ethereum v1.16.3 h1:nDoBSrmsrPbrDIVLTkDQCy1U9KdHN+F2PzvMbDoS42Q= -github.com/ethereum/go-ethereum v1.16.3/go.mod h1:Lrsc6bt9Gm9RyvhfFK53vboCia8kpF9nv+2Ukntnl+8= -github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= -github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= -github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= -github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/indexer/cmd/indexer/main.go b/tools/indexer/main.go similarity index 59% rename from tools/indexer/cmd/indexer/main.go rename to tools/indexer/main.go index c8527a202..90fa145b5 100644 --- a/tools/indexer/cmd/indexer/main.go +++ b/tools/indexer/main.go @@ -4,17 +4,19 @@ import ( "context" "fmt" + _ "github.com/go-sql-driver/mysql" - "github.com/primev/mev-commit/indexer/pkg/backfill" - "github.com/primev/mev-commit/indexer/pkg/beacon" - "github.com/primev/mev-commit/indexer/pkg/config" - "github.com/primev/mev-commit/indexer/pkg/database" - "github.com/primev/mev-commit/indexer/pkg/ethereum" - httputil "github.com/primev/mev-commit/indexer/pkg/http" - "github.com/primev/mev-commit/indexer/pkg/relay" + "github.com/primev/mev-commit/tools/indexer/pkg/backfill" + "github.com/primev/mev-commit/tools/indexer/pkg/beacon" + "github.com/primev/mev-commit/tools/indexer/pkg/config" + "github.com/primev/mev-commit/tools/indexer/pkg/database" + "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" + httputil "github.com/primev/mev-commit/tools/indexer/pkg/http" + "github.com/primev/mev-commit/tools/indexer/pkg/relay" "github.com/urfave/cli/v2" "github.com/urfave/cli/v2/altsrc" - "log" + + "log/slog" "math/rand" "os" "os/signal" @@ -23,9 +25,9 @@ import ( ) type Options struct { - BlockTick time.Duration - ValidatorWait time.Duration - BackfillEvery time.Duration + BlockTick time.Duration + ValidatorWait time.Duration + BackfillLookback int64 BackfillBatch int HTTPTimeout time.Duration @@ -84,13 +86,6 @@ var ( Value: 1500 * time.Millisecond, }) - optionBackfillEvery = altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "backfill-every", - Usage: "interval for backfill operations", - EnvVars: []string{"INDEXER_BACKFILL_EVERY"}, - Value: 5 * time.Minute, - }) - optionBackfillLookback = altsrc.NewIntFlag(&cli.IntFlag{ Name: "backfill-lookback", Usage: "number of slots to look back for backfill", @@ -117,12 +112,11 @@ func createOptionsFromCLI(c *cli.Context) *config.Config { return &config.Config{ BlockTick: c.Duration("block-interval"), ValidatorWait: c.Duration("validator-delay"), - BackfillEvery: c.Duration("backfill-every"), - BackfillLookback: int64(c.Int("backfill-lookback-slots")), + BackfillLookback: int64(c.Int("backfill-lookback")), BackfillBatch: c.Int("backfill-batch"), HTTPTimeout: c.Duration("http-timeout"), OptInContract: c.String("opt-in-contract"), - EtherscanKey: c.String("etherscan-api-key"), + EtherscanKey: c.String("etherscan-key"), InfuraRPC: c.String("infura-rpc"), BeaconBase: c.String("beacon-base"), } @@ -130,19 +124,28 @@ func createOptionsFromCLI(c *cli.Context) *config.Config { func startIndexer(c *cli.Context) error { + initLogger := slog.With("component", "init") + dbLogger := slog.With("component", "db") + httpLogger := slog.With("component", "http") + relayLogger := slog.With("component", "relay") + backfillLogger := slog.With("component", "backfill") + blockLogger := slog.With("component", "block") + bidsLogger := slog.With("component", "bids") + validatorLogger := slog.With("component", "validator") + optInLogger := slog.With("component", "opt-in") + progressLogger := slog.With("component", "progress") + shutdownLogger := slog.With("component", "shutdown") + dbURL := c.String(optionDatabaseURL.Name) infuraRPC := c.String(optionInfuraRPC.Name) beaconBase := c.String(optionBeaconBase.Name) // Initialize random seed rand.Seed(time.Now().UnixNano()) - // Load configuration - - // Validate required configuration - - log.Printf("[INIT] Starting blockchain indexer with StarRocks database") - log.Printf("[CONFIG] Block interval: %s, Validator delay: %s, Backfill every: %s", - c.Duration("block-interval"), c.Duration("validator-delay"), c.Duration("backfill-every")) + initLogger.Info("starting blockchain indexer with StarRocks database") + initLogger.Info("configuration loaded", + "block_interval", c.Duration("block-interval"), + "validator_delay", c.Duration("validator-delay")) // Setup graceful shutdown ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) @@ -151,116 +154,119 @@ func startIndexer(c *cli.Context) error { // Connect to StarRocks database db, err := database.MustConnect(ctx, dbURL, 20, 5) if err != nil { - log.Fatalf("[DB] Connection failed: %v", err) + dbLogger.Error("connection failed", "error", err) } defer db.Close() - log.Printf("[DB] Connected to StarRocks database") + dbLogger.Info("connected to StarRocks database") // Ensure required tables exist if err := db.EnsureStateTable(ctx); err != nil { - log.Fatalf("[DB] Failed to ensure state table: %v", err) + dbLogger.Error("failed to ensure state table", "error", err) + return err } - log.Printf(" [DB] State table ready") + dbLogger.Info("state table ready") // Initialize HTTP client httpc := httputil.NewHTTPClient(c.Duration("http-timeout")) - log.Printf("[HTTP] Client initialized with %s timeout", c.Duration("http-timeout")) + httpLogger.Info("client initialized", "timeout", c.Duration("http-timeout")) // Load relay configurations relays, err := relay.UpsertRelaysAndLoad(ctx, db) if err != nil { - log.Fatalf("[RELAYS] Failed to load: %v", err) + relayLogger.Error("failed to load", "error", err) } - log.Printf("[RELAYS] Loaded %d active relays:", len(relays)) + relayLogger.Info("loaded active relays", "count", len(relays)) for _, r := range relays { - log.Printf(" - Relay ID %d: %s", r.ID, r.URL) + relayLogger.Info("relay found", "id", r.ID, "url", r.URL) } // Initialize starting block number lastBN, found := db.LoadLastBlockNumber(ctx) if !found || lastBN == 0 { - log.Printf(" [INIT] No previous state found, checking database for latest block...") - err := db.Conn.QueryRowContext(ctx, `SELECT COALESCE(MAX(block_number),0) FROM blocks`).Scan(&lastBN) + initLogger.Info("no previous state found, checking database for latest block") + lastBN, err = db.GetMaxBlockNumber(ctx) if err != nil { - log.Printf("[INIT] Database query failed: %v", err) + initLogger.Error("database query failed", "error", err) } } // Replace the hardcoded block search with: if lastBN == 0 { - log.Printf("[INIT] Getting latest block from Ethereum RPC...") + initLogger.Info("getting latest block from Ethereum RPC...") - latestBlock, err := ethereum.GetLatestBlockNumber(httpc, infuraRPC) + latestBlock, err := ethereum.GetLatestBlockNumber(httpc.HTTPClient, infuraRPC) if err != nil { - log.Fatalf("[INIT] Failed to get latest block from RPC: %v", err) + initLogger.Error("failed to get latest block from RPC", "error", err) } lastBN = latestBlock - 10 // Start 10 blocks behind to ensure data availability - log.Printf("[INIT] Starting from block %d (latest: %d)", lastBN, latestBlock) + initLogger.Info("starting from block", "block", lastBN, "latest", latestBlock) } - log.Printf(" [INIT] Starting from block number: %d", lastBN) - log.Printf("[INIT] Indexer configuration - Lookback: %d slots, Batch size: %d", - c.Int("backfill-lookback-slots"), c.Int("backfill-batch")) - - // Setup tickers - backfillTicker := time.NewTicker(c.Duration("backfill-batch")) - defer backfillTicker.Stop() - + initLogger.Info("starting from block number", "block", lastBN) + initLogger.Info("indexer configuration", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) + + if c.Int("backfill-lookback") > 0 { + backfillLogger.Info("running one-time backfill", + "lookback", c.Int("backfill-lookback"), + "batch", c.Int("backfill-batch")) + backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) + backfillLogger.Info("completed startup backfill") + } else { + backfillLogger.Info("skipped", "reason", "backfill-lookback=0") + } mainTicker := time.NewTicker(c.Duration("block-interval")) defer mainTicker.Stop() - - log.Printf("🎉 [INIT] Blockchain indexer started successfully") + initLogger.Info("blockchain indexer started successfully") + go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) // Main processing loop for { select { case <-ctx.Done(): - log.Printf(" [SHUTDOWN] Graceful shutdown initiated: %v", ctx.Err()) + shutdownLogger.Info("graceful shutdown initiated", "reason", ctx.Err()) if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { - log.Printf("[SHUTDOWN] Failed to save last block number: %v", err) + shutdownLogger.Error("failed to save last block number", "error", err) } - log.Printf("[SHUTDOWN] Indexer stopped at block %d", lastBN) + shutdownLogger.Info("indexer stopped", "block", lastBN) return nil - case <-backfillTicker.C: - log.Printf("[BACKFILL] Starting backfill operations...") - backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) - case <-mainTicker.C: nextBN := lastBN + 1 // Fetch execution block data ei, err := beacon.FetchCombinedBlockData(httpc, infuraRPC, beaconBase, nextBN) if err != nil || ei == nil { - log.Printf("⏳ [BLOCK] Block %d not available yet: %v", nextBN, err) + blockLogger.Warn("block not available yet", "block", nextBN, "error", err) + continue } // Log block details - log.Printf("[BLOCK] Processing block %d → slot %d", nextBN, ei.Slot) + blockLogger.Info("processing block", "block", nextBN, "slot", ei.Slot) + if ei.Timestamp != nil { - log.Printf("[BLOCK] Timestamp: %v", ei.Timestamp.Format(time.RFC3339)) + blockLogger.Info("block timestamp", "block", nextBN, "timestamp", ei.Timestamp.Format(time.RFC3339)) } if ei.ProposerIdx != nil { - log.Printf("[VALIDATOR] Proposer index: %d", *ei.ProposerIdx) + validatorLogger.Info("proposer index", "index", *ei.ProposerIdx) } if ei.RelayTag != nil { - log.Printf("[RELAY] Winning relay: %s", *ei.RelayTag) + relayLogger.Info("winning relay", "tag", *ei.RelayTag) } if ei.BuilderHex != nil && len(*ei.BuilderHex) > 20 { - log.Printf("🔨 [BUILDER] Builder pubkey: %s...", (*ei.BuilderHex)[:20]) + blockLogger.Info("builder pubkey", "pubkey_prefix", (*ei.BuilderHex)[:20]) } if ei.RewardEth != nil { - log.Printf("[REWARD] Producer reward: %.6f ETH", *ei.RewardEth) + blockLogger.Info("producer reward", "reward_eth", *ei.RewardEth) } // Save block to database if err := db.UpsertBlockFromExec(ctx, ei); err != nil { - log.Printf("[DB] Failed to save block %d: %v", nextBN, err) + dbLogger.Error("failed to save block", "block", nextBN, "error", err) continue } - log.Printf("[DB] Block %d saved successfully", nextBN) + dbLogger.Info("block saved successfully", "block", nextBN) // Fetch and store bid data from all relays totalBids := 0 @@ -270,14 +276,14 @@ func startIndexer(c *cli.Context) error { for _, rr := range relays { // Check if main context is canceled before processing each relay if ctx.Err() != nil { - log.Printf("[BIDS] Main context canceled, stopping relay processing") + bidsLogger.Warn("main context canceled, stopping relay processing") mainContextCanceled = true break } bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, ei.Slot) if err != nil { - log.Printf(" [BIDS] Relay %d (%s) failed: %v", rr.ID, rr.URL, err) + bidsLogger.Error("relay failed", "relay_id", rr.ID, "url", rr.URL, "error", err) continue } @@ -288,13 +294,13 @@ func startIndexer(c *cli.Context) error { for _, bid := range bids { // Check if main context is still valid if ctx.Err() != nil { - log.Printf("[BIDS] Main context canceled, stopping bid insertion") + bidsLogger.Warn("main context canceled, stopping bid insertion") mainContextCanceled = true break } if err := relay.InsertBid(bidCtx, db, ei.Slot, rr.ID, bid); err != nil { - log.Printf(" [BIDS] Failed to insert bid for slot %d, relay %d: %v", ei.Slot, rr.ID, err) + bidsLogger.Error("failed to insert bid", "slot", ei.Slot, "relay_id", rr.ID, "error", err) } else { relayBids++ } @@ -306,13 +312,13 @@ func startIndexer(c *cli.Context) error { } if relayBids > 0 { - log.Printf(" [BIDS] Relay %d: %d bids collected", rr.ID, relayBids) + bidsLogger.Info("bids collected", "relay_id", rr.ID, "count", relayBids) totalBids += relayBids successfulRelays++ } } - log.Printf(" [SUMMARY] Block %d: %d bids from %d relays", nextBN, totalBids, successfulRelays) + bidsLogger.Info("summary", "block", nextBN, "total_bids", totalBids, "successful_relays", successfulRelays) // Async validator pubkey fetch if ei.ProposerIdx != nil { go func(slot int64, proposerIdx int64) { @@ -320,15 +326,15 @@ func startIndexer(c *cli.Context) error { vpub, err := beacon.FetchValidatorPubkey(httpc, beaconBase, proposerIdx) if err != nil { - log.Printf("[VALIDATOR] Failed to fetch pubkey for proposer %d: %v", proposerIdx, err) + validatorLogger.Error("failed to fetch pubkey", "proposer", proposerIdx, "error", err) return } if len(vpub) > 0 { if err := db.UpdateValidatorPubkey(context.Background(), slot, vpub); err != nil { - log.Printf(" [VALIDATOR] Failed to save pubkey for slot %d: %v", slot, err) + validatorLogger.Error("failed to save pubkey", "slot", slot, "error", err) } else { - log.Printf("[VALIDATOR] Pubkey saved for proposer %d (slot %d)", proposerIdx, slot) + validatorLogger.Info("pubkey saved", "proposer", proposerIdx, "slot", slot) } } }(ei.Slot, *ei.ProposerIdx) @@ -340,45 +346,32 @@ func startIndexer(c *cli.Context) error { time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) // Wait for validator pubkey to be available - var vpk []byte - retries := 3 - for i := 0; i < retries; i++ { - err := db.Conn.QueryRowContext(context.Background(), - `SELECT validator_pubkey FROM blocks WHERE slot=?`, slot).Scan(&vpk) - if err == nil && len(vpk) > 0 { - break - } - if i < retries-1 { - time.Sleep(time.Second) - } - } - - if len(vpk) == 0 { - log.Printf("[OPT-IN] Validator pubkey not available for slot %d", slot) + vpk, err := db.GetValidatorPubkeyWithRetry(context.Background(), slot, 3, time.Second) + if err != nil { + optInLogger.Error("validator pubkey not available", "slot", slot, "error", err) return } - opted, err := ethereum.CallAreOptedInAtBlock(httpc, createOptionsFromCLI(c), blockNumber, vpk) + opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, createOptionsFromCLI(c), blockNumber, vpk) if err != nil { - log.Printf("[OPT-IN] Failed to check opt-in status for slot %d: %v", slot, err) + optInLogger.Error("failed to check opt-in status", "slot", slot, "error", err) return } - _, err = db.Conn.ExecContext(context.Background(), - `UPDATE blocks SET validator_opted_in=? WHERE slot=?`, opted, slot) + err = db.UpdateValidatorOptInStatus(context.Background(), slot, opted) if err != nil { - log.Printf("[OPT-IN] Failed to save opt-in status for slot %d: %v", slot, err) + optInLogger.Error("failed to save opt-in status", "slot", slot, "error", err) } else { - log.Printf("[OPT-IN] Slot %d validator opted-in: %t", slot, opted) + optInLogger.Info("validator opt-in status", "slot", slot, "opted_in", opted) } }(ei.Slot, ei.BlockNumber) } lastBN = nextBN if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { - log.Printf("[PROGRESS] Failed to save block number %d: %v", lastBN, err) + progressLogger.Error("failed to save block number", "block", lastBN, "error", err) } else { - log.Printf("[PROGRESS] Advanced to block %d", lastBN) + progressLogger.Info("advanced to block", "block", lastBN) } } } @@ -392,7 +385,7 @@ func main() { optionBeaconBase, optionBlockInterval, optionValidatorDelay, - optionBackfillEvery, + optionBackfillLookback, optionBackfillBatch, optionHTTPTimeout, diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index 01c5ccba8..4026a15dd 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -1,55 +1,37 @@ -// pkg/backfill/backfill.go package backfill import ( "context" - "log" + "log/slog" "net/http" "time" - "github.com/primev/mev-commit/indexer/pkg/beacon" - "github.com/primev/mev-commit/indexer/pkg/config" - "github.com/primev/mev-commit/indexer/pkg/database" - "github.com/primev/mev-commit/indexer/pkg/ethereum" - "github.com/primev/mev-commit/indexer/pkg/relay" + "github.com/hashicorp/go-retryablehttp" + "github.com/primev/mev-commit/tools/indexer/pkg/beacon" + "github.com/primev/mev-commit/tools/indexer/pkg/config" + "github.com/primev/mev-commit/tools/indexer/pkg/database" + "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" + "github.com/primev/mev-commit/tools/indexer/pkg/relay" ) // RecentMissing backfills recent blocks that are missing data -func RecentMissing(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, lookback int64, batch int) error { - - rows, err := db.Conn.QueryContext(ctx, ` -WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) -SELECT slot, block_number -FROM blocks, recent -WHERE slot > recent.s - ? - AND block_number IS NOT NULL - AND (winning_relay IS NULL - OR winning_builder_pubkey IS NULL - OR fee_recipient IS NULL - OR producer_reward_eth IS NULL - OR timestamp IS NULL - OR proposer_index IS NULL) -ORDER BY slot DESC -LIMIT ?`, lookback, batch) +func RecentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, lookback int64, batch int) error { + logger := slog.With("component", "backfill") + + blocks, err := db.GetRecentMissingBlocks(ctx, lookback, batch) if err != nil { - log.Printf("[BACKFILL] RecentMissing query failed: %v", err) + logger.Error("RecentMissing query failed", "error", err) return err } - defer rows.Close() processed := 0 - for rows.Next() { - var slot int64 - var bn int64 - if err := rows.Scan(&slot, &bn); err != nil { - log.Printf("[BACKFILL] RecentMissing scan failed: %v", err) - continue - } + for _, block := range blocks { // Fetch beacon execution block data - if ei, err := beacon.FetchBeaconExecutionBlock(httpc, cfg.BeaconBase, bn); err == nil && ei != nil { + if ei, err := beacon.FetchBeaconExecutionBlock(httpc, cfg.BeaconBase, block.BlockNumber); err == nil && ei != nil { if err := db.UpsertBlockFromExec(ctx, ei); err != nil { - log.Printf("[BACKFILL] RecentMissing upsert failed for slot %d: %v", slot, err) + logger.Error("RecentMissing upsert failed", "slot", block.Slot, "error", err) + continue } @@ -59,65 +41,55 @@ LIMIT ?`, lookback, batch) time.Sleep(cfg.ValidatorWait) if vpub, err := beacon.FetchValidatorPubkey(httpc, cfg.BeaconBase, idx); err == nil && len(vpub) > 0 { if err := db.UpdateValidatorPubkey(context.Background(), slot, vpub); err != nil { - log.Printf("[BACKFILL] RecentMissing validator pubkey update failed for slot %d: %v", slot, err) + logger.Error("RecentMissing validator pubkey update failed", "slot", slot, "error", err) } } }(ei.Slot, *ei.ProposerIdx) } processed++ } else { - log.Printf("[BACKFILL] RecentMissing beacon fetch failed for block %d: %v", bn, err) - } - } + logger.Error("RecentMissing beacon fetch failed", "block_number", block.BlockNumber, "error", err) - if err := rows.Err(); err != nil { - log.Printf("[BACKFILL] RecentMissing rows iteration failed: %v", err) - return err + } } - log.Printf("[BACKFILL] RecentMissing processed %d blocks", processed) + logger.Info("RecentMissing processed", "blocks", processed) return nil } // RecentBids backfills bid data for ALL recent slots (not just opted-in blocks) -func RecentBids(ctx context.Context, db *database.DB, httpc *http.Client, relays []relay.Row, lookback int64, batch int) error { - - rows, err := db.Conn.QueryContext(ctx, ` -WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) -SELECT DISTINCT slot -FROM blocks, recent -WHERE slot > recent.s - ? - AND block_number IS NOT NULL -ORDER BY slot DESC -LIMIT ?`, lookback, batch) +func RecentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, lookback int64, batch int) error { + logger := slog.With("component", "backfill") + slots, err := db.GetRecentSlotsWithBlocks(ctx, lookback, batch) if err != nil { - log.Printf("[BACKFILL] RecentBids query failed: %v", err) + logger.Error("RecentBids query failed", "error", err) return err } - defer rows.Close() processed := 0 totalBids := 0 - for rows.Next() { - var slot int64 - if err := rows.Scan(&slot); err != nil { - log.Printf("[BACKFILL] RecentBids scan failed: %v", err) - continue - } + for _, slot := range slots { // Fetch bids from ALL relays for this slot slotBids := 0 for _, rr := range relays { + if ctx.Err() != nil { // graceful exit on cancel + break + } if bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, slot); err == nil { for _, b := range bids { + if ctx.Err() != nil { + break + } if err := relay.InsertBid(ctx, db, slot, rr.ID, b); err != nil { - log.Printf("[BACKFILL] RecentBids insert failed for slot %d, relay %d: %v", slot, rr.ID, err) + logger.Error("RecentBids insert failed", "slot", slot, "relay_id", rr.ID, "error", err) + } else { slotBids++ } } } else { - log.Printf("[BACKFILL] RecentBids fetch failed for slot %d, relay %d (%s): %v", slot, rr.ID, rr.URL, err) + logger.Error("RecentBids fetch failed", "slot", slot, "relay_id", rr.ID, "relay_url", rr.URL, "error", err) } } @@ -127,71 +99,42 @@ LIMIT ?`, lookback, batch) } } - if err := rows.Err(); err != nil { - log.Printf("[BACKFILL] RecentBids rows iteration failed: %v", err) - return err - } - - log.Printf("[BACKFILL] RecentBids processed %d slots with %d total bids from ALL blocks", processed, totalBids) + logger.Info("RecentBids processed", "slots", processed, "total_bids", totalBids) return nil } // ValidatorOptIn backfills validator opt-in status (this is opt-in specific data) func ValidatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, lookback int64, batch int) error { - - rows, err := db.Conn.QueryContext(ctx, ` -WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) -SELECT slot, block_number, validator_pubkey -FROM blocks, recent -WHERE slot > recent.s - ? - AND block_number IS NOT NULL - AND validator_pubkey IS NOT NULL - AND validator_opted_in IS NULL -ORDER BY slot DESC -LIMIT ?`, lookback, batch) + logger := slog.With("component", "backfill") + validators, err := db.GetValidatorsNeedingOptInCheck(ctx, lookback, batch) if err != nil { - log.Printf("[BACKFILL] ValidatorOptIn query failed: %v", err) + logger.Error("ValidatorOptIn query failed", "error", err) return err } - defer rows.Close() processed := 0 - for rows.Next() { - var slot, bn int64 - var vpk []byte - if err := rows.Scan(&slot, &bn, &vpk); err != nil { - log.Printf("[BACKFILL] ValidatorOptIn scan failed: %v", err) - continue - } - // Check opt-in status - opted, err := ethereum.CallAreOptedInAtBlock(httpc, cfg, bn, vpk) + for _, v := range validators { + opted, err := ethereum.CallAreOptedInAtBlock(httpc, cfg, v.BlockNumber, v.ValidatorPubkey) if err == nil { - updateCtx, cancel := context.WithTimeout(ctx, 3*time.Second) - // Fixed query parameter style for StarRocks - if _, err := db.Conn.ExecContext(updateCtx, `UPDATE blocks SET validator_opted_in=? WHERE slot=? AND validator_opted_in IS NULL`, opted, slot); err != nil { - log.Printf("[BACKFILL] ValidatorOptIn update failed for slot %d: %v", slot, err) + if err := db.UpdateValidatorOptInStatus(ctx, v.Slot, opted); err != nil { + logger.Error("ValidatorOptIn update failed", "slot", v.Slot, "error", err) } else { processed++ } - cancel() } else { - log.Printf("[BACKFILL] ValidatorOptIn check failed for slot %d: %v", slot, err) + logger.Error("ValidatorOptIn check failed", "slot", v.Slot, "error", err) } } - if err := rows.Err(); err != nil { - log.Printf("[BACKFILL] ValidatorOptIn rows iteration failed: %v", err) - return err - } - - log.Printf("[BACKFILL] ValidatorOptIn processed %d validators", processed) + logger.Info("ValidatorOptIn processed", "validators", processed) return nil } // AllBlocksBids ensures bid data is collected for ALL blocks, regardless of opt-in status -func AllBlocksBids(ctx context.Context, db *database.DB, httpc *http.Client, relays []relay.Row, startSlot, endSlot int64) error { - log.Printf("[BACKFILL] AllBlocksBids ensuring bid coverage from slot %d to %d", startSlot, endSlot) +func AllBlocksBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, startSlot, endSlot int64) error { + logger := slog.With("component", "backfill") + logger.Info("AllBlocksBids starting", "start_slot", startSlot, "end_slot", endSlot) totalProcessed := 0 totalBids := 0 @@ -218,33 +161,34 @@ func AllBlocksBids(ctx context.Context, db *database.DB, httpc *http.Client, rel // Respect context cancellation select { case <-ctx.Done(): - log.Printf("[BACKFILL] AllBlocksBids cancelled at slot %d", slot) + logger.Warn("AllBlocksBids cancelled", "current_slot", slot) return ctx.Err() default: } } - log.Printf("[BACKFILL] AllBlocksBids completed: %d slots processed, %d total bids collected", totalProcessed, totalBids) + logger.Info("AllBlocksBids completed", "slots", totalProcessed, "total_bids", totalBids) return nil } // RunAll executes all backfill operations ensuring complete coverage -func RunAll(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, relays []relay.Row) { - log.Printf("[BACKFILL] Starting comprehensive backfill for ALL blocks (not just opted-in)") +func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, relays []relay.Row) { + logger := slog.With("component", "backfill") + logger.Info("Starting comprehensive backfill for ALL blocks (not just opted-in)") // Run backfill operations if err := RecentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { - log.Printf("[BACKFILL] RecentMissing failed: %v", err) + logger.Error("RecentMissing failed", "error", err) } - if err := ValidatorOptIn(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { - log.Printf("[BACKFILL] ValidatorOptIn failed: %v", err) + if err := ValidatorOptIn(ctx, db, httpc.HTTPClient, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + logger.Error("ValidatorOptIn failed", "error", err) } // This ensures bid data for ALL blocks, not just mev-commit opted-in blocks if err := RecentBids(ctx, db, httpc, relays, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { - log.Printf("[BACKFILL] RecentBids failed: %v", err) + logger.Error("RecentBids failed", "error", err) } - log.Printf("[BACKFILL] All operations completed - relay data covers ALL blocks") + logger.Info("All operations completed - relay data covers ALL blocks") } diff --git a/tools/indexer/pkg/beacon/client.go b/tools/indexer/pkg/beacon/client.go index d4e90e74f..9c05c6ed0 100644 --- a/tools/indexer/pkg/beacon/client.go +++ b/tools/indexer/pkg/beacon/client.go @@ -12,8 +12,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/primev/mev-commit/indexer/pkg/ethereum" - httputil "github.com/primev/mev-commit/indexer/pkg/http" + "github.com/hashicorp/go-retryablehttp" + "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" + httputil "github.com/primev/mev-commit/tools/indexer/pkg/http" ) type ExecInfo struct { @@ -27,14 +28,14 @@ type ExecInfo struct { RewardEth *float64 } -func FetchBeaconExecutionBlock(httpc *http.Client, beaconBase string, blockNum int64) (*ExecInfo, error) { +func FetchBeaconExecutionBlock(httpc *retryablehttp.Client, beaconBase string, blockNum int64) (*ExecInfo, error) { url := fmt.Sprintf("%s/execution/block/%d", beaconBase, blockNum) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var wrap struct { Data []map[string]any `json:"data"` } - if err := httputil.FetchJSONWithRetry(ctx, httpc, url, &wrap, 3, 300*time.Millisecond); err != nil || len(wrap.Data) == 0 { + if err := httputil.FetchJSON(ctx, httpc, url, &wrap); err != nil || len(wrap.Data) == 0 { return nil, fmt.Errorf("no exec block %d", blockNum) } j := wrap.Data[0] @@ -119,7 +120,7 @@ func FetchBeaconExecutionBlock(httpc *http.Client, beaconBase string, blockNum i } // validator pubkey from proposer index -func FetchValidatorPubkey(httpc *http.Client, beaconBase string, proposerIndex int64) ([]byte, error) { +func FetchValidatorPubkey(httpc *retryablehttp.Client, beaconBase string, proposerIndex int64) ([]byte, error) { url := fmt.Sprintf("%s/validator/%d", beaconBase, proposerIndex) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -128,7 +129,7 @@ func FetchValidatorPubkey(httpc *http.Client, beaconBase string, proposerIndex i Pubkey string `json:"pubkey"` } `json:"data"` } - if err := httputil.FetchJSONWithRetry(ctx, httpc, url, &resp, 3, 300*time.Millisecond); err != nil { + if err := httputil.FetchJSON(ctx, httpc, url, &resp); err != nil { return nil, err } if strings.TrimSpace(resp.Data.Pubkey) == "" { @@ -137,8 +138,9 @@ func FetchValidatorPubkey(httpc *http.Client, beaconBase string, proposerIndex i return common.FromHex(resp.Data.Pubkey), nil } -// Add this new function to fetch blocks from Alchemy RPC -func FetchBlockFromRPC(httpc *http.Client, rpcURL string, blockNumber int64) (*ExecInfo, error) { +// to fetch blocks from Alchemy RPC +func FetchBlockFromRPC(httpc *retryablehttp.Client, rpcURL string, blockNumber int64) (*ExecInfo, error) { + underlyingClient := httpc.HTTPClient // Get block data from Alchemy payload := map[string]any{ "jsonrpc": "2.0", @@ -151,7 +153,7 @@ func FetchBlockFromRPC(httpc *http.Client, rpcURL string, blockNumber int64) (*E req, _ := http.NewRequest("POST", rpcURL, bytes.NewReader(buf)) req.Header.Set("Content-Type", "application/json") - resp, err := httpc.Do(req) + resp, err := underlyingClient.Do(req) if err != nil { return nil, err } @@ -183,7 +185,7 @@ func FetchBlockFromRPC(httpc *http.Client, rpcURL string, blockNumber int64) (*E Timestamp: &blockTime, }, nil } -func FetchCombinedBlockData(httpc *http.Client, rpcURL string, beaconBase string, blockNumber int64) (*ExecInfo, error) { +func FetchCombinedBlockData(httpc *retryablehttp.Client, rpcURL string, beaconBase string, blockNumber int64) (*ExecInfo, error) { // Get execution block from Alchemy (always available) execBlock, err := FetchBlockFromRPC(httpc, rpcURL, blockNumber) if err != nil { @@ -193,7 +195,6 @@ func FetchCombinedBlockData(httpc *http.Client, rpcURL string, beaconBase string // Convert block number to slot for beacon chain query slotNumber := ethereum.BlockNumberToSlot(blockNumber) - // Try to get beacon chain data using slot number (may not exist for recent blocks) beaconData, _ := FetchBeaconExecutionBlock(httpc, beaconBase, slotNumber) // Merge data - use Alchemy as primary, beacon as supplement @@ -203,7 +204,7 @@ func FetchCombinedBlockData(httpc *http.Client, rpcURL string, beaconBase string execBlock.RelayTag = beaconData.RelayTag execBlock.RewardEth = beaconData.RewardEth } else { - // Set the calculated slot if beacon data not available + execBlock.Slot = slotNumber } diff --git a/tools/indexer/pkg/config/config.go b/tools/indexer/pkg/config/config.go index 91d3cf543..227e30610 100644 --- a/tools/indexer/pkg/config/config.go +++ b/tools/indexer/pkg/config/config.go @@ -5,16 +5,17 @@ import ( ) type Relay struct { - Name string - Tag string - URL string + Relay_id int64 + Name string + Tag string + URL string } var RelaysDefault = []Relay{ - {Name: "Titan", Tag: "titan-relay", URL: "https://regional.titanrelay.xyz"}, - {Name: "Aestus", Tag: "aestus-relay", URL: "https://aestus.live"}, - {Name: "Bloxroute Max Profit", Tag: "bloxroute-max-profit-relay", URL: "https://bloxroute.max-profit.blxrbdn.com"}, - {Name: "Bloxroute Regulated", Tag: "bloxroute-regulated-relay", URL: "https://bloxroute.regulated.blxrbdn.com"}, + {Relay_id: 1, Name: "Titan", Tag: "titan-relay", URL: "https://regional.titanrelay.xyz"}, + {Relay_id: 2, Name: "Aestus", Tag: "aestus-relay", URL: "https://aestus.live"}, + {Relay_id: 3, Name: "Bloxroute Max Profit", Tag: "bloxroute-max-profit-relay", URL: "https://bloxroute.max-profit.blxrbdn.com"}, + {Relay_id: 4, Name: "Bloxroute Regulated", Tag: "bloxroute-regulated-relay", URL: "https://bloxroute.regulated.blxrbdn.com"}, } type Config struct { diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go index bdef445b0..5bd38f231 100644 --- a/tools/indexer/pkg/database/starrock.go +++ b/tools/indexer/pkg/database/starrock.go @@ -4,15 +4,17 @@ import ( "context" "database/sql" "fmt" - _ "github.com/go-sql-driver/mysql" - "github.com/primev/mev-commit/indexer/pkg/beacon" - "github.com/primev/mev-commit/indexer/pkg/config" "strings" "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + _ "github.com/go-sql-driver/mysql" + "github.com/primev/mev-commit/tools/indexer/pkg/beacon" + "github.com/primev/mev-commit/tools/indexer/pkg/config" ) type DB struct { - Conn *sql.DB + conn *sql.DB } func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, error) { @@ -33,11 +35,11 @@ func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, return nil, fmt.Errorf("StarRocks ping failed: %v", err) } - return &DB{Conn: conn}, nil + return &DB{conn: conn}, nil } func (db *DB) Close() { - db.Conn.Close() + db.conn.Close() } func (db *DB) EnsureStateTable(ctx context.Context) error { @@ -55,14 +57,14 @@ func (db *DB) EnsureStateTable(ctx context.Context) error { "replication_num" = "1" )` - if _, err := db.Conn.ExecContext(ctx2, ddl); err != nil { + if _, err := db.conn.ExecContext(ctx2, ddl); err != nil { return fmt.Errorf("failed to create state table: %w", err) } var count int - err := db.Conn.QueryRowContext(ctx2, `SELECT COUNT(*) FROM ingestor_state WHERE id = 1`).Scan(&count) + err := db.conn.QueryRowContext(ctx2, `SELECT COUNT(*) FROM ingestor_state WHERE id = 1`).Scan(&count) if err != nil || count == 0 { - _, err = db.Conn.ExecContext(ctx2, + _, err = db.conn.ExecContext(ctx2, `INSERT INTO ingestor_state (id, last_block_number) VALUES (1, 0)`) if err != nil { return fmt.Errorf("failed to insert initial state: %w", err) @@ -71,13 +73,28 @@ func (db *DB) EnsureStateTable(ctx context.Context) error { return nil } +func (db *DB) GetMaxBlockNumber(ctx context.Context) (int64, error) { + ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + var bn int64 + err := db.conn.QueryRowContext(ctx2, `SELECT COALESCE(MAX(block_number),0) FROM blocks`).Scan(&bn) + return bn, err +} +func (db *DB) GetValidatorPubkey(ctx context.Context, slot int64) ([]byte, error) { + ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + var vpk []byte + err := db.conn.QueryRowContext(ctx2, `SELECT validator_pubkey FROM blocks WHERE slot=?`, slot).Scan(&vpk) + return vpk, err +} func (db *DB) LoadLastBlockNumber(ctx context.Context) (int64, bool) { ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() var bn int64 - err := db.Conn.QueryRowContext(ctx2, + err := db.conn.QueryRowContext(ctx2, `SELECT last_block_number FROM ingestor_state WHERE id = 1 LIMIT 1`).Scan(&bn) if err != nil { return 0, false @@ -89,13 +106,8 @@ func (db *DB) SaveLastBlockNumber(ctx context.Context, bn int64) error { ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() - _, err := db.Conn.ExecContext(ctx2, `DELETE FROM ingestor_state WHERE id = 1`) - if err != nil { - return fmt.Errorf("failed to delete old state: %w", err) - } - - query := fmt.Sprintf(`INSERT INTO ingestor_state (id, last_block_number) VALUES (1, %d)`, bn) - _, err = db.Conn.ExecContext(ctx2, query) + query := fmt.Sprintf(`REPLACE INTO ingestor_state (id, last_block_number) VALUES (1, %d)`, bn) + _, err := db.conn.ExecContext(ctx2, query) if err != nil { return fmt.Errorf("save last_block_number failed: %w", err) } @@ -113,14 +125,14 @@ func (db *DB) UpsertRelays(ctx context.Context, relays []config.Relay) error { // StarRocks batch insert approach var values []string for _, r := range relays { - value := fmt.Sprintf("('%s', '%s', '%s', 1)", r.Name, r.Tag, r.URL) + value := fmt.Sprintf("(%d, '%s', '%s', '%s', 1)", r.Relay_id, r.Name, r.Tag, r.URL) values = append(values, value) } - query := fmt.Sprintf(`INSERT INTO relays (name, tag, base_url, is_active) VALUES %s`, + query := fmt.Sprintf(`INSERT INTO relays (relay_id, name, tag, base_url, is_active) VALUES %s`, strings.Join(values, ",")) - _, err := db.Conn.ExecContext(ctx2, query) + _, err := db.conn.ExecContext(ctx2, query) return err } @@ -165,7 +177,7 @@ INSERT INTO blocks( ) VALUES (%d, %d, %s, %s, %s, %s)`, ei.Slot, ei.BlockNumber, timestamp, proposerIndex, relayTag, rewardEth) - _, err := db.Conn.ExecContext(ctx2, query) + _, err := db.conn.ExecContext(ctx2, query) if err != nil { return fmt.Errorf("upsert block slot=%d: %w", ei.Slot, err) } @@ -182,12 +194,234 @@ func (db *DB) UpdateValidatorPubkey(ctx context.Context, slot int64, vpub []byte ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() + vhex := hexutil.Encode(vpub) - _, err := db.Conn.ExecContext(ctx2, ` - UPDATE blocks SET validator_pubkey = ? WHERE slot = ?`, - vpub, slot) - if err != nil { + q := fmt.Sprintf("INSERT INTO blocks (slot, validator_pubkey) VALUES (%d, '%s')", slot, vhex) + + if _, err := db.conn.ExecContext(ctx2, q); err != nil { return fmt.Errorf("update validator slot=%d: %w", slot, err) } + return nil } + +// Add these new methods to your DB struct +func (db *DB) InsertBid(ctx context.Context, slot int64, relayID int64, builder, proposer, feeRec []byte, valStr string, blockNum *int64, tsMS *int64) error { + ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + builderHex := hexutil.Encode(builder) + proposerHex := hexutil.Encode(proposer) + feeRecHex := hexutil.Encode(feeRec) + blockNumSQL := "NULL" + if blockNum != nil { + blockNumSQL = fmt.Sprintf("%d", *blockNum) + } + + tsMSSQL := "NULL" + if tsMS != nil { + tsMSSQL = fmt.Sprintf("%d", *tsMS) + } + query := fmt.Sprintf(` + INSERT INTO bids( + slot, relay_id, builder_pubkey, proposer_pubkey, + proposer_fee_recipient, value_wei, block_number, timestamp_ms + ) + VALUES (%d, %d, '%s', '%s', '%s', '%s', %s, %s)`, + slot, relayID, builderHex, proposerHex, feeRecHex, valStr, blockNumSQL, tsMSSQL, + ) + + _, err := db.conn.ExecContext(ctx2, query) + return err +} + +func (db *DB) GetActiveRelays(ctx context.Context) ([]struct { + ID int64 + URL string +}, error) { + ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + rows, err := db.conn.QueryContext(ctx2, `SELECT relay_id, base_url FROM relays WHERE is_active = 1`) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []struct { + ID int64 + URL string + } + for rows.Next() { + var id int64 + var url string + if err := rows.Scan(&id, &url); err != nil { + continue // Skip bad rows + } + results = append(results, struct { + ID int64 + URL string + }{ID: id, URL: url}) + } + return results, rows.Err() +} + +func (db *DB) GetRecentMissingBlocks(ctx context.Context, lookback int64, batch int) ([]struct { + Slot int64 + BlockNumber int64 +}, error) { + ctx2, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + if lookback < 0 || batch < 0 || batch > 10000 { + return nil, fmt.Errorf("invalid parameters: lookback=%d, batch=%d", lookback, batch) + } + + // Build query with literal values + query := fmt.Sprintf(` + WITH recent AS ( + SELECT COALESCE(MAX(slot), 0) AS s FROM blocks + ) + SELECT slot, block_number + FROM blocks, recent + WHERE slot > recent.s - %d + AND block_number IS NOT NULL + AND (winning_relay IS NULL + OR winning_builder_pubkey IS NULL + OR fee_recipient IS NULL + OR producer_reward_eth IS NULL + OR timestamp IS NULL + OR proposer_index IS NULL) + ORDER BY slot DESC + LIMIT %d`, lookback, batch) + + rows, err := db.conn.QueryContext(ctx2, query) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []struct { + Slot int64 + BlockNumber int64 + } + for rows.Next() { + var slot, bn int64 + if err := rows.Scan(&slot, &bn); err != nil { + continue + } + results = append(results, struct { + Slot int64 + BlockNumber int64 + }{Slot: slot, BlockNumber: bn}) + } + return results, rows.Err() +} + +func (db *DB) GetRecentSlotsWithBlocks(ctx context.Context, lookback int64, batch int) ([]int64, error) { + ctx2, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + q := fmt.Sprintf(` +WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) +SELECT DISTINCT slot +FROM blocks, recent +WHERE slot > recent.s - ? + AND block_number IS NOT NULL +ORDER BY slot DESC +LIMIT %d`, batch) + rows, err := db.conn.QueryContext(ctx2, q, lookback) + if err != nil { + return nil, err + } + defer rows.Close() + + var slots []int64 + for rows.Next() { + var slot int64 + if err := rows.Scan(&slot); err != nil { + continue + } + slots = append(slots, slot) + } + return slots, rows.Err() +} + +func (db *DB) GetValidatorsNeedingOptInCheck(ctx context.Context, lookback int64, batch int) ([]struct { + Slot int64 + BlockNumber int64 + ValidatorPubkey []byte +}, error) { + ctx2, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + q := fmt.Sprintf(` +WITH recent AS (SELECT COALESCE(MAX(slot),0) AS s FROM blocks) +SELECT slot, block_number, validator_pubkey +FROM blocks, recent +WHERE slot > recent.s - ? + AND block_number IS NOT NULL + AND validator_pubkey IS NOT NULL + AND validator_opted_in IS NULL +ORDER BY slot DESC +LIMIT %d`, batch) + rows, err := db.conn.QueryContext(ctx2, q, lookback) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []struct { + Slot int64 + BlockNumber int64 + ValidatorPubkey []byte + } + for rows.Next() { + var slot, bn int64 + var vpk []byte + if err := rows.Scan(&slot, &bn, &vpk); err != nil { + continue + } + results = append(results, struct { + Slot int64 + BlockNumber int64 + ValidatorPubkey []byte + }{ + Slot: slot, BlockNumber: bn, ValidatorPubkey: vpk, + }) + } + return results, rows.Err() +} + +func (db *DB) UpdateValidatorOptInStatus(ctx context.Context, slot int64, opted bool) error { + ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + v := 0 + if opted { + v = 1 + } // TINYINT(1) in StarRocks + q := fmt.Sprintf( + "UPDATE blocks SET validator_opted_in=%d WHERE slot=%d AND validator_opted_in IS NULL", + v, slot, + ) + _, err := db.conn.ExecContext(ctx2, q) + return err +} + +func (db *DB) GetValidatorPubkeyWithRetry(ctx context.Context, slot int64, retries int, retryDelay time.Duration) ([]byte, error) { + var vpk []byte + for i := 0; i < retries; i++ { + ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) + err := db.conn.QueryRowContext(ctx2, `SELECT validator_pubkey FROM blocks WHERE slot=?`, slot).Scan(&vpk) + cancel() + + if err == nil && len(vpk) > 0 { + return vpk, nil + } + + if i < retries-1 { + time.Sleep(retryDelay) + } + } + return nil, fmt.Errorf("validator pubkey not available after %d retries", retries) +} diff --git a/tools/indexer/pkg/ethereum/cleint.go b/tools/indexer/pkg/ethereum/cleint.go deleted file mode 100644 index f25b90d65..000000000 --- a/tools/indexer/pkg/ethereum/cleint.go +++ /dev/null @@ -1,191 +0,0 @@ -package ethereum - -import ( - "bytes" - "context" - "encoding/hex" - "encoding/json" - "fmt" - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/primev/mev-commit/indexer/pkg/config" - "io" - "net/http" - "strconv" - "strings" - "time" - - httputil "github.com/primev/mev-commit/indexer/pkg/http" -) - -const optInABIJSON = `[ - { - "inputs":[{"internalType":"bytes[]","name":"valBLSPubKeys","type":"bytes[]"}], - "name":"areValidatorsOptedIn", - "outputs":[{"components":[ - {"internalType":"bool","name":"isVanillaOptedIn","type":"bool"}, - {"internalType":"bool","name":"isAvsOptedIn","type":"bool"}, - {"internalType":"bool","name":"isMiddlewareOptedIn","type":"bool"} - ],"internalType":"struct OptInStatus[]","name":"","type":"tuple[]"}], - "stateMutability":"view","type":"function" - } -]` - -func BuildAreOptedInCallData(pubkey []byte) ([]byte, error) { - ab, err := abi.JSON(strings.NewReader(optInABIJSON)) - if err != nil { - return nil, err - } - // Solidity expects bytes[]; pass []byte{pubkey} length 1 - return ab.Pack("areValidatorsOptedIn", [][]byte{pubkey}) -} - -// JSON-RPC helper (Infura / Alchemy / any node) -func EthCallJSONRPC(httpc *http.Client, rpcURL string, to string, data []byte, blockNum int64) ([]byte, error) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - tag := "0x" + strconv.FormatInt(blockNum, 16) - payload := map[string]any{ - "jsonrpc": "2.0", - "id": 1, - "method": "eth_call", - "params": []any{ - map[string]any{ - "to": to, - "data": "0x" + hex.EncodeToString(data), - }, - tag, - }, - } - buf, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("eth_call marshal: %w", err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, rpcURL, bytes.NewReader(buf)) - if err != nil { - return nil, fmt.Errorf("build eth_call request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - resp, err := httpc.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - - body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - return nil, fmt.Errorf("eth_call http %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) - } - - var out struct { - Result string `json:"result"` - Error *struct { - Code int `json:"code"` - Message string `json:"message"` - } `json:"error"` - } - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return nil, err - } - if out.Error != nil { - return nil, fmt.Errorf("eth_call rpc error %d: %s", out.Error.Code, out.Error.Message) - } - if out.Result == "" || out.Result == "0x" { - return nil, fmt.Errorf("eth_call empty result (err=%v)", out.Error) - } - return hex.DecodeString(strings.TrimPrefix(out.Result, "0x")) -} - -func CallAreOptedInAtBlock(httpc *http.Client, cfg *config.Config, blockNum int64, pubkey []byte) (bool, error) { - if len(pubkey) == 0 { - return false, fmt.Errorf("empty pubkey") - } - data, err := BuildAreOptedInCallData(pubkey) - if err != nil { - return false, err - } - - var ret []byte - if cfg.InfuraRPC != "" { - // Preferred: direct JSON-RPC via Infura - ret, err = EthCallJSONRPC(httpc, cfg.InfuraRPC, cfg.OptInContract, data, blockNum) - if err != nil { - return false, err - } - } else { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - // Fallback: Etherscan proxy - tag := "0x" + strconv.FormatInt(blockNum, 16) - url := fmt.Sprintf("https://api.etherscan.io/api?module=proxy&action=eth_call&to=%s&data=0x%s&tag=%s", - cfg.OptInContract, hex.EncodeToString(data), tag) - if cfg.EtherscanKey != "" { - url += "&apikey=" + cfg.EtherscanKey - } - var resp struct { - Result string `json:"result"` - } - if err := httputil.FetchJSONWithRetry(ctx, httpc, url, &resp, cfg.MaxRetries, cfg.BaseRetryDelay); err != nil { - return false, err - } - if resp.Result == "" || resp.Result == "0x" { - return false, fmt.Errorf("empty result") - } - ret, err = hex.DecodeString(strings.TrimPrefix(resp.Result, "0x")) - if err != nil { - return false, err - } - } - - // Decode OptInStatus[] where OptInStatus=(bool,bool,bool) - ab, err := abi.JSON(strings.NewReader(optInABIJSON)) - if err != nil { - return false, err - } - var out []struct { - IsVanillaOptedIn bool - IsAvsOptedIn bool - IsMiddlewareOptedIn bool - } - if err := ab.UnpackIntoInterface(&out, "areValidatorsOptedIn", ret); err != nil { - return false, err - } - if len(out) == 0 { - return false, nil - } - o := out[0] - return o.IsVanillaOptedIn || o.IsAvsOptedIn || o.IsMiddlewareOptedIn, nil -} - -// GetLatestBlockNumber gets the latest block number from Ethereum RPC -func GetLatestBlockNumber(httpc *http.Client, rpcURL string) (int64, error) { - payload := map[string]any{ - "jsonrpc": "2.0", - "id": 1, - "method": "eth_blockNumber", - "params": []any{}, - } - - buf, _ := json.Marshal(payload) - req, _ := http.NewRequest("POST", rpcURL, bytes.NewReader(buf)) - req.Header.Set("Content-Type", "application/json") - - resp, err := httpc.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - - var result struct { - Result string `json:"result"` - } - - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return 0, err - } - - // Convert hex to int64 - blockNum, err := strconv.ParseInt(result.Result[2:], 16, 64) - return blockNum, err -} diff --git a/tools/indexer/pkg/ethereum/client.go b/tools/indexer/pkg/ethereum/client.go new file mode 100644 index 000000000..ea4b601c7 --- /dev/null +++ b/tools/indexer/pkg/ethereum/client.go @@ -0,0 +1,76 @@ +package ethereum + +import ( + "bytes" + + "encoding/json" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/primev/mev-commit/tools/indexer/pkg/config" + + "net/http" + "strconv" + + "github.com/primev/mev-commit/contracts-abi/clients/ValidatorOptInRouter" +) + +func CallAreOptedInAtBlock(httpc *http.Client, cfg *config.Config, blockNum int64, pubkey []byte) (bool, error) { + if len(pubkey) == 0 { + return false, fmt.Errorf("empty pubkey") + } + client, err := ethclient.Dial(cfg.InfuraRPC) + if err != nil { + return false, err + } + contract, err := validatoroptinrouter.NewValidatoroptinrouter(common.HexToAddress(cfg.OptInContract), client) + if err != nil { + return false, err + } + + result, err := contract.AreValidatorsOptedIn(&bind.CallOpts{BlockNumber: big.NewInt(blockNum)}, [][]byte{pubkey}) + if err != nil { + return false, err + } + + if len(result) == 0 { + return false, nil + } + o := result[0] + return o.IsVanillaOptedIn || o.IsAvsOptedIn || o.IsMiddlewareOptedIn, nil +} + +// GetLatestBlockNumber gets the latest block number from Ethereum RPC +func GetLatestBlockNumber(httpc *http.Client, rpcURL string) (int64, error) { + payload := map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_blockNumber", + "params": []any{}, + } + + buf, _ := json.Marshal(payload) + req, _ := http.NewRequest("POST", rpcURL, bytes.NewReader(buf)) + req.Header.Set("Content-Type", "application/json") + + resp, err := httpc.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + var result struct { + Result string `json:"result"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return 0, err + } + + // Convert hex to int64 + blockNum, err := strconv.ParseInt(result.Result[2:], 16, 64) + return blockNum, err +} diff --git a/tools/indexer/pkg/ethereum/conversions.go b/tools/indexer/pkg/ethereum/conversions.go index 8f4fffff6..962bacf5a 100644 --- a/tools/indexer/pkg/ethereum/conversions.go +++ b/tools/indexer/pkg/ethereum/conversions.go @@ -1,4 +1,3 @@ -// pkg/utils/conversions.go package ethereum func BlockNumberToSlot(blockNumber int64) int64 { diff --git a/tools/indexer/pkg/http/client.go b/tools/indexer/pkg/http/client.go index 04ebe3524..46b1b114d 100644 --- a/tools/indexer/pkg/http/client.go +++ b/tools/indexer/pkg/http/client.go @@ -4,79 +4,34 @@ import ( "context" "encoding/json" "fmt" - "math/rand" - "net" - "net/http" - "strconv" + "github.com/hashicorp/go-retryablehttp" "time" ) -func NewHTTPClient(timeout time.Duration) *http.Client { - return &http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 5 * time.Second, - }, - } +func NewHTTPClient(timeout time.Duration) *retryablehttp.Client { + client := retryablehttp.NewClient() + client.HTTPClient.Timeout = timeout + client.RetryMax = 3 + client.RetryWaitMin = 200 * time.Millisecond + client.RetryWaitMax = 2 * time.Second + return client } -func FetchJSONWithRetry(ctx context.Context, httpc *http.Client, url string, out any, attempts int, baseDelay time.Duration) error { - var lastErr error - for i := 0; i < attempts; i++ { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - lastErr = err - continue - } - resp, err := httpc.Do(req) - if err == nil && resp != nil && resp.StatusCode == 200 { - defer resp.Body.Close() - return json.NewDecoder(resp.Body).Decode(out) - } - if resp != nil { - // 429 courtesy backoff if provided - if resp.StatusCode == 429 { - if ra := resp.Header.Get("Retry-After"); ra != "" { - if secs, err := strconv.Atoi(ra); err == nil { - select { - case <-ctx.Done(): - resp.Body.Close() - return ctx.Err() +func FetchJSON(ctx context.Context, client *retryablehttp.Client, url string, out any) error { + req, err := retryablehttp.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return err + } - case <-time.After(time.Duration(secs) * time.Second): - } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() - } - } - } - if resp.StatusCode != http.StatusOK { - lastErr = fmt.Errorf("GET %s: status %d", url, resp.StatusCode) - } - resp.Body.Close() - } else if err != nil { - lastErr = err - } - if i < attempts-1 { - sleep := baseDelay * time.Duration(1< Date: Mon, 29 Sep 2025 20:36:37 +0530 Subject: [PATCH 03/18] fixed db close error --- tools/indexer/pkg/database/starrock.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go index 5bd38f231..ccd84dd66 100644 --- a/tools/indexer/pkg/database/starrock.go +++ b/tools/indexer/pkg/database/starrock.go @@ -38,8 +38,8 @@ func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, return &DB{conn: conn}, nil } -func (db *DB) Close() { - db.conn.Close() +func (db *DB) Close() error { + return db.conn.Close() } func (db *DB) EnsureStateTable(ctx context.Context) error { From 1b503b2eba49f33fb7b70c04c530d27af3021413 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Fri, 3 Oct 2025 23:19:54 +0530 Subject: [PATCH 04/18] fixes --- tools/indexer/main.go | 56 ++++++++++----- tools/indexer/pkg/backfill/backfill.go | 97 +++++++++++++++++++------ tools/indexer/pkg/database/starrock.go | 99 +++++++++++++++++++------- tools/indexer/pkg/http/client.go | 4 +- tools/indexer/pkg/relay/client.go | 30 +++++--- 5 files changed, 210 insertions(+), 76 deletions(-) diff --git a/tools/indexer/main.go b/tools/indexer/main.go index 90fa145b5..943776f67 100644 --- a/tools/indexer/main.go +++ b/tools/indexer/main.go @@ -97,7 +97,7 @@ var ( Name: "backfill-batch", Usage: "batch size for backfill operations", EnvVars: []string{"INDEXER_BACKFILL_BATCH"}, - Value: 50, + Value: 5, }) optionHTTPTimeout = altsrc.NewDurationFlag(&cli.DurationFlag{ @@ -210,15 +210,15 @@ func startIndexer(c *cli.Context) error { backfillLogger.Info("running one-time backfill", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) - backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) + go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) backfillLogger.Info("completed startup backfill") } else { backfillLogger.Info("skipped", "reason", "backfill-lookback=0") } mainTicker := time.NewTicker(c.Duration("block-interval")) defer mainTicker.Stop() - initLogger.Info("blockchain indexer started successfully") - go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) + // initLogger.Info("blockchain indexer started successfully") + // go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) // Main processing loop for { @@ -272,7 +272,7 @@ func startIndexer(c *cli.Context) error { totalBids := 0 successfulRelays := 0 mainContextCanceled := false - +const batchSize = 500 for _, rr := range relays { // Check if main context is canceled before processing each relay if ctx.Err() != nil { @@ -281,15 +281,16 @@ func startIndexer(c *cli.Context) error { break } - bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, ei.Slot) + bids, err := relay.FetchBuilderBlocksReceived(ctx, httpc, rr.URL, ei.Slot) if err != nil { bidsLogger.Error("relay failed", "relay_id", rr.ID, "url", rr.URL, "error", err) continue } relayBids := 0 - // Create a separate context with timeout for bid insertions - bidCtx, bidCancel := context.WithTimeout(context.Background(), 30*time.Second) + batch := make([]database.BidRow, 0, batchSize) + // a separate context with timeout for bid insertions + // bidCtx, bidCancel := context.WithTimeout(ctx, 30*time.Second) for _, bid := range bids { // Check if main context is still valid @@ -299,13 +300,32 @@ func startIndexer(c *cli.Context) error { break } - if err := relay.InsertBid(bidCtx, db, ei.Slot, rr.ID, bid); err != nil { - bidsLogger.Error("failed to insert bid", "slot", ei.Slot, "relay_id", rr.ID, "error", err) - } else { - relayBids++ - } - } - bidCancel() + if row, ok := relay.BuildBidInsert(ei.Slot, rr.ID, bid); ok { + batch = append(batch, row) + // for _, bid := range bids { + if len(batch) >= batchSize { + insCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + if err := db.InsertBidsBatch(insCtx, batch); err != nil { + + bidsLogger.Error("batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) + } else { + relayBids += len(batch) + } + cancel() + batch = batch[:0] + } + } + + + // final flush + if len(batch) > 0 { + if err := db.InsertBidsBatch(ctx, batch); err != nil { + bidsLogger.Error("batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) + } else { + relayBids += len(batch) + } + } +} if mainContextCanceled { break @@ -331,7 +351,7 @@ func startIndexer(c *cli.Context) error { } if len(vpub) > 0 { - if err := db.UpdateValidatorPubkey(context.Background(), slot, vpub); err != nil { + if err := db.UpdateValidatorPubkey(ctx, slot, vpub); err != nil { validatorLogger.Error("failed to save pubkey", "slot", slot, "error", err) } else { validatorLogger.Info("pubkey saved", "proposer", proposerIdx, "slot", slot) @@ -346,7 +366,7 @@ func startIndexer(c *cli.Context) error { time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) // Wait for validator pubkey to be available - vpk, err := db.GetValidatorPubkeyWithRetry(context.Background(), slot, 3, time.Second) + vpk, err := db.GetValidatorPubkeyWithRetry(ctx, slot, 3, time.Second) if err != nil { optInLogger.Error("validator pubkey not available", "slot", slot, "error", err) return @@ -358,7 +378,7 @@ func startIndexer(c *cli.Context) error { return } - err = db.UpdateValidatorOptInStatus(context.Background(), slot, opted) + err = db.UpdateValidatorOptInStatus(ctx, slot, opted) if err != nil { optInLogger.Error("failed to save opt-in status", "slot", slot, "error", err) } else { diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index 4026a15dd..f2cbd86ec 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -60,36 +60,61 @@ func RecentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Cl // RecentBids backfills bid data for ALL recent slots (not just opted-in blocks) func RecentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, lookback int64, batch int) error { logger := slog.With("component", "backfill") - slots, err := db.GetRecentSlotsWithBlocks(ctx, lookback, batch) + opCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + slots, err := db.GetRecentSlotsWithBlocks(opCtx, lookback, batch) if err != nil { logger.Error("RecentBids query failed", "error", err) return err } + logger.Info("RecentBids fetched slots", "count", len(slots)) processed := 0 totalBids := 0 + const batchSize = 500 + for _, slot := range slots { + if ctx.Err() != nil { + break + } - // Fetch bids from ALL relays for this slot slotBids := 0 + for _, rr := range relays { - if ctx.Err() != nil { // graceful exit on cancel + if ctx.Err() != nil { break } - if bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, slot); err == nil { - for _, b := range bids { - if ctx.Err() != nil { - break - } - if err := relay.InsertBid(ctx, db, slot, rr.ID, b); err != nil { - logger.Error("RecentBids insert failed", "slot", slot, "relay_id", rr.ID, "error", err) - } else { - slotBids++ - } - } - } else { + fetchCtx, fcancel := context.WithTimeout(ctx, 5*time.Second) + bids, err := relay.FetchBuilderBlocksReceived(fetchCtx, httpc, rr.URL, slot) + fcancel() + if err != nil { logger.Error("RecentBids fetch failed", "slot", slot, "relay_id", rr.ID, "relay_url", rr.URL, "error", err) + continue + } + + rows := make([]database.BidRow, 0, len(bids)) + for _, b := range bids { + if row, ok := relay.BuildBidInsert(slot, rr.ID, b); ok { + rows = append(rows, row) + } + } + + for i := 0; i < len(rows); i += batchSize { + end := i + batchSize + if end > len(rows) { + end = len(rows) + } + insCtx, icancel := context.WithTimeout(ctx, 5*time.Second) + if err := db.InsertBidsBatch(insCtx, rows[i:end]); err != nil { + logger.Error("RecentBids batch insert failed", "slot", slot, "relay_id", rr.ID, "count", end-i, "error", err) + } else { + slotBids += end - i + } + icancel() + if ctx.Err() != nil { + break + } } } @@ -135,20 +160,48 @@ func ValidatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cf func AllBlocksBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, startSlot, endSlot int64) error { logger := slog.With("component", "backfill") logger.Info("AllBlocksBids starting", "start_slot", startSlot, "end_slot", endSlot) - + const batchSize = 500 totalProcessed := 0 totalBids := 0 for slot := startSlot; slot <= endSlot; slot++ { + if ctx.Err() != nil { + break + } slotBids := 0 - // Fetch bids from ALL relays for every single slot for _, rr := range relays { - if bids, err := relay.FetchBuilderBlocksReceived(httpc, rr.URL, slot); err == nil { - for _, b := range bids { - if err := relay.InsertBid(ctx, db, slot, rr.ID, b); err == nil { - slotBids++ - } + fetchCtx, fcancel := context.WithTimeout(ctx, 5*time.Second) + bids, err := relay.FetchBuilderBlocksReceived(fetchCtx, httpc, rr.URL, slot) + fcancel() + if err != nil { + logger.Error("AllBlocksBids fetch failed", "slot", slot, "relay_id", rr.ID, "url", rr.URL, "error", err) + continue + } + + rows := make([]database.BidRow, 0, len(bids)) + for _, b := range bids { + if row, ok := relay.BuildBidInsert(slot, rr.ID, b); ok { + rows = append(rows, row) + } + } + + for i := 0; i < len(rows); i += batchSize { + end := i + batchSize + if end > len(rows) { + end = len(rows) + } + + insCtx, icancel := context.WithTimeout(ctx, 5*time.Second) + if err := db.InsertBidsBatch(insCtx, rows[i:end]); err != nil { + logger.Error("AllBlocksBids batch insert failed", "slot", slot, "relay_id", rr.ID, "count", end-i, "error", err) + } else { + slotBids += end - i + } + icancel() + + if ctx.Err() != nil { + break } } } diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go index ccd84dd66..1bed4f7d1 100644 --- a/tools/indexer/pkg/database/starrock.go +++ b/tools/indexer/pkg/database/starrock.go @@ -16,7 +16,16 @@ import ( type DB struct { conn *sql.DB } - +type BidInsert struct { + Slot int64 + RelayID int64 + BuilderHex string + ProposerHex string + FeeRecHex string + ValStr string + BlockNum *int64 + TsMS *int64 +} func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, error) { conn, err := sql.Open("mysql", dsn) if err != nil { @@ -39,7 +48,7 @@ func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, } func (db *DB) Close() error { - return db.conn.Close() + return db.conn.Close() } func (db *DB) EnsureStateTable(ctx context.Context) error { @@ -205,33 +214,75 @@ func (db *DB) UpdateValidatorPubkey(ctx context.Context, slot int64, vpub []byte return nil } -// Add these new methods to your DB struct -func (db *DB) InsertBid(ctx context.Context, slot int64, relayID int64, builder, proposer, feeRec []byte, valStr string, blockNum *int64, tsMS *int64) error { - ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - builderHex := hexutil.Encode(builder) - proposerHex := hexutil.Encode(proposer) - feeRecHex := hexutil.Encode(feeRec) - blockNumSQL := "NULL" - if blockNum != nil { - blockNumSQL = fmt.Sprintf("%d", *blockNum) - } +// // Add these new methods to your DB struct +// func (db *DB) InsertBid(ctx context.Context, slot int64, relayID int64, builder, proposer, feeRec []byte, valStr string, blockNum *int64, tsMS *int64) error { +// ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) +// defer cancel() +// builderHex := hexutil.Encode(builder) +// proposerHex := hexutil.Encode(proposer) +// feeRecHex := hexutil.Encode(feeRec) +// blockNumSQL := "NULL" +// if blockNum != nil { +// blockNumSQL = fmt.Sprintf("%d", *blockNum) +// } + +// tsMSSQL := "NULL" +// if tsMS != nil { +// tsMSSQL = fmt.Sprintf("%d", *tsMS) +// } +// query := fmt.Sprintf(` +// INSERT INTO bids( +// slot, relay_id, builder_pubkey, proposer_pubkey, +// proposer_fee_recipient, value_wei, block_number, timestamp_ms +// ) +// VALUES (%d, %d, '%s', '%s', '%s', '%s', %s, %s)`, +// slot, relayID, builderHex, proposerHex, feeRecHex, valStr, blockNumSQL, tsMSSQL, +// ) + +// _, err := db.conn.ExecContext(ctx2, query) +// return err +// } +// Minimal batching: builds one multi-VALUES INSERT. +// NOTE: relies on same string building pattern you already use. +type BidRow struct { + Slot, RelayID int64 + Builder, Proposer, FeeRec string + ValStr string + BlockNum, TsMS *int64 +} - tsMSSQL := "NULL" - if tsMS != nil { - tsMSSQL = fmt.Sprintf("%d", *tsMS) - } - query := fmt.Sprintf(` +func (db *DB) InsertBidsBatch(ctx context.Context, rows []BidRow) error { + if len(rows) == 0 { return nil } + + ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var sb strings.Builder + sb.WriteString(` INSERT INTO bids( slot, relay_id, builder_pubkey, proposer_pubkey, proposer_fee_recipient, value_wei, block_number, timestamp_ms - ) - VALUES (%d, %d, '%s', '%s', '%s', '%s', %s, %s)`, - slot, relayID, builderHex, proposerHex, feeRecHex, valStr, blockNumSQL, tsMSSQL, - ) + ) VALUES `) - _, err := db.conn.ExecContext(ctx2, query) - return err + for i, r := range rows { + if i > 0 { sb.WriteString(",") } + + // builderHex := hexutil.Encode(r.Builder) + // proposerHex := hexutil.Encode(r.Proposer) + // feeRecHex := hexutil.Encode(r.FeeRec) + + blockNumSQL := "NULL" + if r.BlockNum != nil { blockNumSQL = fmt.Sprintf("%d", *r.BlockNum) } + + tsMSSQL := "NULL" + if r.TsMS != nil { tsMSSQL = fmt.Sprintf("%d", *r.TsMS) } + + fmt.Fprintf(&sb, "(%d,%d,'%s','%s','%s','%s',%s,%s)", + r.Slot, r.RelayID, r.Builder, r.Proposer, r.FeeRec, r.ValStr, blockNumSQL, tsMSSQL) + } + + _, err := db.conn.ExecContext(ctx2, sb.String()) + return err } func (db *DB) GetActiveRelays(ctx context.Context) ([]struct { diff --git a/tools/indexer/pkg/http/client.go b/tools/indexer/pkg/http/client.go index 46b1b114d..77515b8ec 100644 --- a/tools/indexer/pkg/http/client.go +++ b/tools/indexer/pkg/http/client.go @@ -4,8 +4,9 @@ import ( "context" "encoding/json" "fmt" - "github.com/hashicorp/go-retryablehttp" "time" + + "github.com/hashicorp/go-retryablehttp" ) func NewHTTPClient(timeout time.Duration) *retryablehttp.Client { @@ -14,6 +15,7 @@ func NewHTTPClient(timeout time.Duration) *retryablehttp.Client { client.RetryMax = 3 client.RetryWaitMin = 200 * time.Millisecond client.RetryWaitMax = 2 * time.Second + client.Logger = nil return client } diff --git a/tools/indexer/pkg/relay/client.go b/tools/indexer/pkg/relay/client.go index 702e4e555..6fcf2f693 100644 --- a/tools/indexer/pkg/relay/client.go +++ b/tools/indexer/pkg/relay/client.go @@ -57,10 +57,10 @@ func parseBigString(v any) (string, bool) { } } -func InsertBid(ctx context.Context, db *database.DB, slot int64, relayID int64, bid map[string]any) error { +func BuildBidInsert(slot int64, relayID int64, bid map[string]any) (database.BidRow, bool){ if slot <= 0 || relayID <= 0 { - return fmt.Errorf("invalid slot or relayID") + return database.BidRow{}, false } // helper to read alternative keys from different relay schemas @@ -80,7 +80,7 @@ func InsertBid(ctx context.Context, db *database.DB, slot int64, relayID int64, valStr, ok := parseBigString(get("value", "value_wei", "valueWei")) if !ok || valStr == "" { - return nil // skip if no value + return database.BidRow{}, false // skip if no value } var blockNum *int64 @@ -114,11 +114,19 @@ func InsertBid(ctx context.Context, db *database.DB, slot int64, relayID int64, } } - ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - return db.InsertBid(ctx2, slot, relayID, builder, proposer, feeRec, valStr, blockNum, tsMS) - + // ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) + // defer cancel() + + return database.BidRow{ + Slot: slot, + RelayID: relayID, + Builder: hexutil.Encode(builder), + Proposer: hexutil.Encode(proposer), + FeeRec: hexutil.Encode(feeRec), + ValStr: valStr, + BlockNum: blockNum, + TsMS: tsMS, + }, true } func UpsertRelaysAndLoad(ctx context.Context, db *database.DB) ([]Row, error) { @@ -140,13 +148,13 @@ func UpsertRelaysAndLoad(ctx context.Context, db *database.DB) ([]Row, error) { return rws, nil } -func FetchBuilderBlocksReceived(httpc *retryablehttp.Client, relayBase string, slot int64) ([]map[string]any, error) { +func FetchBuilderBlocksReceived(ctx context.Context, httpc *retryablehttp.Client, relayBase string, slot int64) ([]map[string]any, error) { url := fmt.Sprintf("%s/relay/v1/data/bidtraces/builder_blocks_received?slot=%d", strings.TrimRight(relayBase, "/"), slot) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() var arr []map[string]any - if err := httputil.FetchJSON(ctx, httpc, url, &arr); err != nil { + if err := httputil.FetchJSON(ctx2, httpc, url, &arr); err != nil { return nil, err } From 98e511eafeb0da3baeef79fe5f1543ab68284989 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Mon, 6 Oct 2025 16:42:05 +0530 Subject: [PATCH 05/18] fixed backfill operations and refactoring --- tools/indexer/cmd/main.go | 151 +++++++++ tools/indexer/cmd/start.go | 279 +++++++++++++++ tools/indexer/main.go | 447 ------------------------- tools/indexer/pkg/backfill/backfill.go | 170 +++------- tools/indexer/pkg/beacon/client.go | 29 +- tools/indexer/pkg/database/starrock.go | 107 +++--- tools/indexer/pkg/relay/client.go | 25 +- 7 files changed, 544 insertions(+), 664 deletions(-) create mode 100644 tools/indexer/cmd/main.go create mode 100644 tools/indexer/cmd/start.go delete mode 100644 tools/indexer/main.go diff --git a/tools/indexer/cmd/main.go b/tools/indexer/cmd/main.go new file mode 100644 index 000000000..9f28deb9c --- /dev/null +++ b/tools/indexer/cmd/main.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + + "fmt" + + _ "github.com/go-sql-driver/mysql" + "github.com/primev/mev-commit/tools/indexer/pkg/config" + "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2/altsrc" + + "os" + "os/signal" + "syscall" + "time" +) + +var ( + optionConfig = &cli.StringFlag{ + Name: "config", + Usage: "Path to config file", + EnvVars: []string{"INDEXER_CONFIG"}, + } + optionDatabaseURL = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "database-url", + Usage: "Database connection URL", + EnvVars: []string{"INDEXER_DATABASE_URL"}, + Required: true, + }) + optionOptInContract = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "opt-in-contract", + Usage: "Opt-in contract address", + EnvVars: []string{"INDEXER_OPT_IN_CONTRACT"}, + Value: "0x821798d7b9d57dF7Ed7616ef9111A616aB19ed64", + }) + optionEtherscanKey = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "etherscan-key", + Usage: "Etherscan API key", + EnvVars: []string{"INDEXER_ETHERSCAN_KEY"}, + }) + optionInfuraRPC = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "infura-rpc", + Usage: "Infura RPC URL", + EnvVars: []string{"INDEXER_INFURA_RPC"}, + Required: true, + }) + optionBeaconBase = altsrc.NewStringFlag(&cli.StringFlag{ + Name: "beacon-base", + Usage: "Beacon API base URL", + EnvVars: []string{"INDEXER_BEACON_BASE"}, + Value: "https://beaconcha.in/api/v1", + }) + optionBlockInterval = altsrc.NewDurationFlag(&cli.DurationFlag{ + Name: "block-interval", + Usage: "interval between block processing", + EnvVars: []string{"INDEXER_BLOCK_INTERVAL"}, + Value: 12 * time.Second, + }) + + optionValidatorDelay = altsrc.NewDurationFlag(&cli.DurationFlag{ + Name: "validator-delay", + Usage: "delay before fetching validator data", + EnvVars: []string{"INDEXER_VALIDATOR_DELAY"}, + Value: 1500 * time.Millisecond, + }) + + optionBackfillLookback = altsrc.NewIntFlag(&cli.IntFlag{ + Name: "backfill-lookback", + Usage: "number of slots to look back for backfill", + EnvVars: []string{"INDEXER_BACKFILL_LOOKBACK"}, + Value: 512, + }) + + optionBackfillBatch = altsrc.NewIntFlag(&cli.IntFlag{ + Name: "backfill-batch", + Usage: "batch size for backfill operations", + EnvVars: []string{"INDEXER_BACKFILL_BATCH"}, + Value: 5, + }) + + optionHTTPTimeout = altsrc.NewDurationFlag(&cli.DurationFlag{ + Name: "http-timeout", + Usage: "HTTP client timeout", + EnvVars: []string{"INDEXER_HTTP_TIMEOUT"}, + Value: 15 * time.Second, + }) +) + +func createOptionsFromCLI(c *cli.Context) *config.Config { + return &config.Config{ + BlockTick: c.Duration("block-interval"), + ValidatorWait: c.Duration("validator-delay"), + BackfillLookback: int64(c.Int("backfill-lookback")), + BackfillBatch: c.Int("backfill-batch"), + HTTPTimeout: c.Duration("http-timeout"), + OptInContract: c.String("opt-in-contract"), + EtherscanKey: c.String("etherscan-key"), + InfuraRPC: c.String("infura-rpc"), + BeaconBase: c.String("beacon-base"), + } +} + +func main() { + flags := []cli.Flag{ + optionConfig, + optionDatabaseURL, + optionInfuraRPC, + optionBeaconBase, + optionBlockInterval, + optionValidatorDelay, + + optionBackfillLookback, + optionBackfillBatch, + optionHTTPTimeout, + optionOptInContract, + optionEtherscanKey, + } + + app := &cli.App{ + Name: "mev-indexer", + Usage: "Builder/observer indexer", + Commands: []*cli.Command{{ + Name: "start", + Usage: "Start the indexer", + Flags: flags, + Before: altsrc.InitInputSourceWithContext( + flags, altsrc.NewYamlSourceFromFlagFunc("config"), + ), + Action: func(c *cli.Context) error { + return startIndexer(c) + }, + }}, + } + ctx, cancel := context.WithCancel(context.Background()) + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigc + _, _ = fmt.Fprintln(app.Writer, "received interrupt signal, exiting... Force exit with Ctrl+C") + cancel() + <-sigc + _, _ = fmt.Fprintln(app.Writer, "force exiting...") + os.Exit(1) + }() + + if err := app.RunContext(ctx, os.Args); err != nil { + _, _ = fmt.Fprintf(app.Writer, "exited with error: %v\n", err) + } + +} diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go new file mode 100644 index 000000000..88e59cd7d --- /dev/null +++ b/tools/indexer/cmd/start.go @@ -0,0 +1,279 @@ +package main + +import ( + "context" + "log/slog" + "math/rand" + "os/signal" + "syscall" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/primev/mev-commit/tools/indexer/pkg/backfill" + "github.com/primev/mev-commit/tools/indexer/pkg/beacon" + "github.com/primev/mev-commit/tools/indexer/pkg/database" + "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" + httputil "github.com/primev/mev-commit/tools/indexer/pkg/http" + "github.com/primev/mev-commit/tools/indexer/pkg/relay" + "github.com/urfave/cli/v2" +) + +func startIndexer(c *cli.Context) error { + + initLogger := slog.With("component", "init") + + dbURL := c.String(optionDatabaseURL.Name) + infuraRPC := c.String(optionInfuraRPC.Name) + beaconBase := c.String(optionBeaconBase.Name) + // Initialize random seed + rand.Seed(time.Now().UnixNano()) + + initLogger.Info("starting blockchain indexer with StarRocks database") + initLogger.Info("configuration loaded", + "block_interval", c.Duration("block-interval"), + "validator_delay", c.Duration("validator-delay")) + + // Setup graceful shutdown + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + // Connect to StarRocks database + db, err := database.MustConnect(ctx, dbURL, 20, 5) + if err != nil { + initLogger.Error("[DB] connection failed", "error", err) + } + defer db.Close() + initLogger.Info("[DB] connected to StarRocks database") + + // Ensure required tables exist + if err := db.EnsureStateTable(ctx); err != nil { + initLogger.Error("[DB] failed to ensure state table", "error", err) + return err + } + initLogger.Info("[DB] state table ready") + + // Initialize HTTP client + httpc := httputil.NewHTTPClient(c.Duration("http-timeout")) + initLogger.Info("[HTTP] client initialized", "timeout", c.Duration("http-timeout")) + + // Load relay configurations + relays, err := relay.UpsertRelaysAndLoad(ctx, db) + if err != nil { + initLogger.Error("[RELAY] failed to load", "error", err) + } + initLogger.Info("[RELAY] loaded active relays", "count", len(relays)) + for _, r := range relays { + initLogger.Info("[RELAY] relay found", "id", r.ID, "url", r.URL) + } + + // Initialize starting block number + lastBN, found := db.LoadLastBlockNumber(ctx) + if !found || lastBN == 0 { + initLogger.Info("no previous state found, checking database for latest block") + lastBN, err = db.GetMaxBlockNumber(ctx) + if err != nil { + initLogger.Error("database query failed", "error", err) + } + } + + // Replace the hardcoded block search with: + if lastBN == 0 { + initLogger.Info("getting latest block from Ethereum RPC...") + + latestBlock, err := ethereum.GetLatestBlockNumber(httpc.HTTPClient, infuraRPC) + if err != nil { + initLogger.Error("failed to get latest block from RPC", "error", err) + } + + lastBN = latestBlock - 10 // Start 10 blocks behind to ensure data availability + initLogger.Info("starting from block", "block", lastBN, "latest", latestBlock) + } + + initLogger.Info("starting from block number", "block", lastBN) + initLogger.Info("indexer configuration", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) + + if c.Int("backfill-lookback") > 0 { + initLogger.Info("[BACKFILL] running one-time backfill", + "lookback", c.Int("backfill-lookback"), + "batch", c.Int("backfill-batch")) + go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) + initLogger.Info("[BACKFILL] completed startup backfill") + } else { + initLogger.Info("[BACKFILL] skipped", "reason", "backfill-lookback=0") + } + mainTicker := time.NewTicker(c.Duration("block-interval")) + defer mainTicker.Stop() + + // Main processing loop + for { + select { + case <-ctx.Done(): + initLogger.Info("[SHUTDOWN] graceful shutdown initiated", "reason", ctx.Err()) + if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { + initLogger.Error("[SHUTDOWN] failed to save last block number", "error", err) + } + initLogger.Info("[SHUTDOWN] indexer stopped", "block", lastBN) + return nil + + case <-mainTicker.C: + nextBN := lastBN + 1 + + // Fetch execution block data + ei, err := beacon.FetchCombinedBlockData(ctx, httpc, infuraRPC, beaconBase, nextBN) + if err != nil || ei == nil { + initLogger.Warn("[BLOCK] not available yet", "block", nextBN, "error", err) + + continue + } + + // Log block details + initLogger.Info("[BLOCK] processing block", "block", nextBN, "slot", ei.Slot) + + if ei.Timestamp != nil { + initLogger.Info("[BLOCK] block timestamp", "block", nextBN, "timestamp", ei.Timestamp.Format(time.RFC3339)) + } + if ei.ProposerIdx != nil { + initLogger.Info("[VALIDATOR] proposer index", "index", *ei.ProposerIdx) + } + if ei.RelayTag != nil { + initLogger.Info("[RELAY] winning relay", "tag", *ei.RelayTag) + } + if ei.BuilderHex != nil && len(*ei.BuilderHex) > 20 { + initLogger.Info("[BLOCK] builder pubkey", "pubkey_prefix", (*ei.BuilderHex)[:20]) + } + if ei.RewardEth != nil { + initLogger.Info("[BLOCK] producer reward", "reward_eth", *ei.RewardEth) + } + + // Save block to database + if err := db.UpsertBlockFromExec(ctx, ei); err != nil { + initLogger.Error("[DB] failed to save block", "block", nextBN, "error", err) + continue + } + initLogger.Info("[DB] block saved successfully", "block", nextBN) + + // Fetch and store bid data from all relays + totalBids := 0 + successfulRelays := 0 + mainContextCanceled := false + const batchSize = 500 + for _, rr := range relays { + // Check if main context is canceled before processing each relay + if ctx.Err() != nil { + initLogger.Warn("main context canceled, stopping relay processing") + mainContextCanceled = true + break + } + + bids, err := relay.FetchBuilderBlocksReceived(ctx, httpc, rr.URL, ei.Slot) + if err != nil { + initLogger.Error("[RELAY] failed to fetch bids", "relay_id", rr.ID, "url", rr.URL, "error", err) + continue + } + + relayBids := 0 + batch := make([]database.BidRow, 0, batchSize) + + for _, bid := range bids { + // Check if main context is still valid + if ctx.Err() != nil { + initLogger.Warn("[BIDS] main context canceled, stopping bid insertion") + mainContextCanceled = true + break + } + + if row, ok := relay.BuildBidInsert(ei.Slot, rr.ID, bid); ok { + batch = append(batch, row) + + if len(batch) >= batchSize { + insCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + if err := db.InsertBidsBatch(insCtx, batch); err != nil { + + initLogger.Error("[DB]batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) + } else { + relayBids += len(batch) + } + cancel() + batch = batch[:0] + } + } + } + + // final flush + if len(batch) > 0 { + if err := db.InsertBidsBatch(ctx, batch); err != nil { + initLogger.Error("[DB] batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) + } else { + relayBids += len(batch) + } + } + + if mainContextCanceled { + break + } + + if relayBids > 0 { + initLogger.Info("[BIDS] bids collected", "relay_id", rr.ID, "count", relayBids) + totalBids += relayBids + successfulRelays++ + } + } + + initLogger.Info("[BIDS] summary", "block", nextBN, "total_bids", totalBids, "successful_relays", successfulRelays) + // Async validator pubkey fetch + if ei.ProposerIdx != nil { + go func(slot int64, proposerIdx int64) { + time.Sleep(c.Duration("validator-delay")) + + vpub, err := beacon.FetchValidatorPubkey(ctx, httpc, beaconBase, proposerIdx) + if err != nil { + initLogger.Error("[VALIDATOR] failed to fetch pubkey", "proposer", proposerIdx, "error", err) + return + } + + if len(vpub) > 0 { + if err := db.UpdateValidatorPubkey(ctx, slot, vpub); err != nil { + initLogger.Error("[VALIDATOR] failed to save pubkey", "slot", slot, "error", err) + } else { + initLogger.Info("[VALIDATOR] pubkey saved", "proposer", proposerIdx, "slot", slot) + } + } + }(ei.Slot, *ei.ProposerIdx) + } + + // Async opt-in status check + if ei.ProposerIdx != nil { + go func(slot int64, blockNumber int64) { + time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) + + // Wait for validator pubkey to be available + vpk, err := db.GetValidatorPubkeyWithRetry(ctx, slot, 3, time.Second) + if err != nil { + initLogger.Error("[VALIDATOR] pubkey not available", "slot", slot, "error", err) + return + } + + opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, createOptionsFromCLI(c), blockNumber, vpk) + if err != nil { + initLogger.Error("[OPT-IN] failed to check opt-in status", "slot", slot, "error", err) + return + } + + err = db.UpdateValidatorOptInStatus(ctx, slot, opted) + if err != nil { + initLogger.Error("[OPT-IN] failed to save opt-in status", "slot", slot, "error", err) + } else { + initLogger.Info("[OPT-IN] validator opt-in status", "slot", slot, "opted_in", opted) + } + }(ei.Slot, ei.BlockNumber) + } + + lastBN = nextBN + if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { + initLogger.Error("[PROGRESS] failed to save block number", "block", lastBN, "error", err) + } else { + initLogger.Info("[PROGRESS] advanced to block", "block", lastBN) + } + } + } +} diff --git a/tools/indexer/main.go b/tools/indexer/main.go deleted file mode 100644 index 943776f67..000000000 --- a/tools/indexer/main.go +++ /dev/null @@ -1,447 +0,0 @@ -package main - -import ( - "context" - - "fmt" - - _ "github.com/go-sql-driver/mysql" - "github.com/primev/mev-commit/tools/indexer/pkg/backfill" - "github.com/primev/mev-commit/tools/indexer/pkg/beacon" - "github.com/primev/mev-commit/tools/indexer/pkg/config" - "github.com/primev/mev-commit/tools/indexer/pkg/database" - "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" - httputil "github.com/primev/mev-commit/tools/indexer/pkg/http" - "github.com/primev/mev-commit/tools/indexer/pkg/relay" - "github.com/urfave/cli/v2" - "github.com/urfave/cli/v2/altsrc" - - "log/slog" - "math/rand" - "os" - "os/signal" - "syscall" - "time" -) - -type Options struct { - BlockTick time.Duration - ValidatorWait time.Duration - - BackfillLookback int64 - BackfillBatch int - HTTPTimeout time.Duration - OptInContract string - EtherscanKey string - InfuraRPC string - BeaconBase string -} - -var ( - optionConfig = &cli.StringFlag{ - Name: "config", - Usage: "Path to config file", - EnvVars: []string{"INDEXER_CONFIG"}, - } - optionDatabaseURL = altsrc.NewStringFlag(&cli.StringFlag{ - Name: "database-url", - Usage: "Database connection URL", - EnvVars: []string{"INDEXER_DATABASE_URL"}, - Required: true, - }) - optionOptInContract = altsrc.NewStringFlag(&cli.StringFlag{ - Name: "opt-in-contract", - Usage: "Opt-in contract address", - EnvVars: []string{"INDEXER_OPT_IN_CONTRACT"}, - Value: "0x821798d7b9d57dF7Ed7616ef9111A616aB19ed64", - }) - optionEtherscanKey = altsrc.NewStringFlag(&cli.StringFlag{ - Name: "etherscan-key", - Usage: "Etherscan API key", - EnvVars: []string{"INDEXER_ETHERSCAN_KEY"}, - }) - optionInfuraRPC = altsrc.NewStringFlag(&cli.StringFlag{ - Name: "infura-rpc", - Usage: "Infura RPC URL", - EnvVars: []string{"INDEXER_INFURA_RPC"}, - Required: true, - }) - optionBeaconBase = altsrc.NewStringFlag(&cli.StringFlag{ - Name: "beacon-base", - Usage: "Beacon API base URL", - EnvVars: []string{"INDEXER_BEACON_BASE"}, - Value: "https://beaconcha.in/api/v1", - }) - optionBlockInterval = altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "block-interval", - Usage: "interval between block processing", - EnvVars: []string{"INDEXER_BLOCK_INTERVAL"}, - Value: 12 * time.Second, - }) - - optionValidatorDelay = altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "validator-delay", - Usage: "delay before fetching validator data", - EnvVars: []string{"INDEXER_VALIDATOR_DELAY"}, - Value: 1500 * time.Millisecond, - }) - - optionBackfillLookback = altsrc.NewIntFlag(&cli.IntFlag{ - Name: "backfill-lookback", - Usage: "number of slots to look back for backfill", - EnvVars: []string{"INDEXER_BACKFILL_LOOKBACK"}, - Value: 512, - }) - - optionBackfillBatch = altsrc.NewIntFlag(&cli.IntFlag{ - Name: "backfill-batch", - Usage: "batch size for backfill operations", - EnvVars: []string{"INDEXER_BACKFILL_BATCH"}, - Value: 5, - }) - - optionHTTPTimeout = altsrc.NewDurationFlag(&cli.DurationFlag{ - Name: "http-timeout", - Usage: "HTTP client timeout", - EnvVars: []string{"INDEXER_HTTP_TIMEOUT"}, - Value: 15 * time.Second, - }) -) - -func createOptionsFromCLI(c *cli.Context) *config.Config { - return &config.Config{ - BlockTick: c.Duration("block-interval"), - ValidatorWait: c.Duration("validator-delay"), - BackfillLookback: int64(c.Int("backfill-lookback")), - BackfillBatch: c.Int("backfill-batch"), - HTTPTimeout: c.Duration("http-timeout"), - OptInContract: c.String("opt-in-contract"), - EtherscanKey: c.String("etherscan-key"), - InfuraRPC: c.String("infura-rpc"), - BeaconBase: c.String("beacon-base"), - } -} - -func startIndexer(c *cli.Context) error { - - initLogger := slog.With("component", "init") - dbLogger := slog.With("component", "db") - httpLogger := slog.With("component", "http") - relayLogger := slog.With("component", "relay") - backfillLogger := slog.With("component", "backfill") - blockLogger := slog.With("component", "block") - bidsLogger := slog.With("component", "bids") - validatorLogger := slog.With("component", "validator") - optInLogger := slog.With("component", "opt-in") - progressLogger := slog.With("component", "progress") - shutdownLogger := slog.With("component", "shutdown") - - dbURL := c.String(optionDatabaseURL.Name) - infuraRPC := c.String(optionInfuraRPC.Name) - beaconBase := c.String(optionBeaconBase.Name) - // Initialize random seed - rand.Seed(time.Now().UnixNano()) - - initLogger.Info("starting blockchain indexer with StarRocks database") - initLogger.Info("configuration loaded", - "block_interval", c.Duration("block-interval"), - "validator_delay", c.Duration("validator-delay")) - - // Setup graceful shutdown - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer cancel() - - // Connect to StarRocks database - db, err := database.MustConnect(ctx, dbURL, 20, 5) - if err != nil { - dbLogger.Error("connection failed", "error", err) - } - defer db.Close() - dbLogger.Info("connected to StarRocks database") - - // Ensure required tables exist - if err := db.EnsureStateTable(ctx); err != nil { - dbLogger.Error("failed to ensure state table", "error", err) - return err - } - dbLogger.Info("state table ready") - - // Initialize HTTP client - httpc := httputil.NewHTTPClient(c.Duration("http-timeout")) - httpLogger.Info("client initialized", "timeout", c.Duration("http-timeout")) - - // Load relay configurations - relays, err := relay.UpsertRelaysAndLoad(ctx, db) - if err != nil { - relayLogger.Error("failed to load", "error", err) - } - relayLogger.Info("loaded active relays", "count", len(relays)) - for _, r := range relays { - relayLogger.Info("relay found", "id", r.ID, "url", r.URL) - } - - // Initialize starting block number - lastBN, found := db.LoadLastBlockNumber(ctx) - if !found || lastBN == 0 { - initLogger.Info("no previous state found, checking database for latest block") - lastBN, err = db.GetMaxBlockNumber(ctx) - if err != nil { - initLogger.Error("database query failed", "error", err) - } - } - - // Replace the hardcoded block search with: - if lastBN == 0 { - initLogger.Info("getting latest block from Ethereum RPC...") - - latestBlock, err := ethereum.GetLatestBlockNumber(httpc.HTTPClient, infuraRPC) - if err != nil { - initLogger.Error("failed to get latest block from RPC", "error", err) - } - - lastBN = latestBlock - 10 // Start 10 blocks behind to ensure data availability - initLogger.Info("starting from block", "block", lastBN, "latest", latestBlock) - } - - initLogger.Info("starting from block number", "block", lastBN) - initLogger.Info("indexer configuration", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) - - if c.Int("backfill-lookback") > 0 { - backfillLogger.Info("running one-time backfill", - "lookback", c.Int("backfill-lookback"), - "batch", c.Int("backfill-batch")) - go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) - backfillLogger.Info("completed startup backfill") - } else { - backfillLogger.Info("skipped", "reason", "backfill-lookback=0") - } - mainTicker := time.NewTicker(c.Duration("block-interval")) - defer mainTicker.Stop() - // initLogger.Info("blockchain indexer started successfully") - // go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) - - // Main processing loop - for { - select { - case <-ctx.Done(): - shutdownLogger.Info("graceful shutdown initiated", "reason", ctx.Err()) - if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { - shutdownLogger.Error("failed to save last block number", "error", err) - } - shutdownLogger.Info("indexer stopped", "block", lastBN) - return nil - - case <-mainTicker.C: - nextBN := lastBN + 1 - - // Fetch execution block data - ei, err := beacon.FetchCombinedBlockData(httpc, infuraRPC, beaconBase, nextBN) - if err != nil || ei == nil { - blockLogger.Warn("block not available yet", "block", nextBN, "error", err) - - continue - } - - // Log block details - blockLogger.Info("processing block", "block", nextBN, "slot", ei.Slot) - - if ei.Timestamp != nil { - blockLogger.Info("block timestamp", "block", nextBN, "timestamp", ei.Timestamp.Format(time.RFC3339)) - } - if ei.ProposerIdx != nil { - validatorLogger.Info("proposer index", "index", *ei.ProposerIdx) - } - if ei.RelayTag != nil { - relayLogger.Info("winning relay", "tag", *ei.RelayTag) - } - if ei.BuilderHex != nil && len(*ei.BuilderHex) > 20 { - blockLogger.Info("builder pubkey", "pubkey_prefix", (*ei.BuilderHex)[:20]) - } - if ei.RewardEth != nil { - blockLogger.Info("producer reward", "reward_eth", *ei.RewardEth) - } - - // Save block to database - if err := db.UpsertBlockFromExec(ctx, ei); err != nil { - dbLogger.Error("failed to save block", "block", nextBN, "error", err) - continue - } - dbLogger.Info("block saved successfully", "block", nextBN) - - // Fetch and store bid data from all relays - totalBids := 0 - successfulRelays := 0 - mainContextCanceled := false -const batchSize = 500 - for _, rr := range relays { - // Check if main context is canceled before processing each relay - if ctx.Err() != nil { - bidsLogger.Warn("main context canceled, stopping relay processing") - mainContextCanceled = true - break - } - - bids, err := relay.FetchBuilderBlocksReceived(ctx, httpc, rr.URL, ei.Slot) - if err != nil { - bidsLogger.Error("relay failed", "relay_id", rr.ID, "url", rr.URL, "error", err) - continue - } - - relayBids := 0 - batch := make([]database.BidRow, 0, batchSize) - // a separate context with timeout for bid insertions - // bidCtx, bidCancel := context.WithTimeout(ctx, 30*time.Second) - - for _, bid := range bids { - // Check if main context is still valid - if ctx.Err() != nil { - bidsLogger.Warn("main context canceled, stopping bid insertion") - mainContextCanceled = true - break - } - - if row, ok := relay.BuildBidInsert(ei.Slot, rr.ID, bid); ok { - batch = append(batch, row) - // for _, bid := range bids { - if len(batch) >= batchSize { - insCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - if err := db.InsertBidsBatch(insCtx, batch); err != nil { - - bidsLogger.Error("batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) - } else { - relayBids += len(batch) - } - cancel() - batch = batch[:0] - } - } - - - // final flush - if len(batch) > 0 { - if err := db.InsertBidsBatch(ctx, batch); err != nil { - bidsLogger.Error("batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) - } else { - relayBids += len(batch) - } - } -} - - if mainContextCanceled { - break - } - - if relayBids > 0 { - bidsLogger.Info("bids collected", "relay_id", rr.ID, "count", relayBids) - totalBids += relayBids - successfulRelays++ - } - } - - bidsLogger.Info("summary", "block", nextBN, "total_bids", totalBids, "successful_relays", successfulRelays) - // Async validator pubkey fetch - if ei.ProposerIdx != nil { - go func(slot int64, proposerIdx int64) { - time.Sleep(c.Duration("validator-delay")) - - vpub, err := beacon.FetchValidatorPubkey(httpc, beaconBase, proposerIdx) - if err != nil { - validatorLogger.Error("failed to fetch pubkey", "proposer", proposerIdx, "error", err) - return - } - - if len(vpub) > 0 { - if err := db.UpdateValidatorPubkey(ctx, slot, vpub); err != nil { - validatorLogger.Error("failed to save pubkey", "slot", slot, "error", err) - } else { - validatorLogger.Info("pubkey saved", "proposer", proposerIdx, "slot", slot) - } - } - }(ei.Slot, *ei.ProposerIdx) - } - - // Async opt-in status check - if ei.ProposerIdx != nil { - go func(slot int64, blockNumber int64) { - time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) - - // Wait for validator pubkey to be available - vpk, err := db.GetValidatorPubkeyWithRetry(ctx, slot, 3, time.Second) - if err != nil { - optInLogger.Error("validator pubkey not available", "slot", slot, "error", err) - return - } - - opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, createOptionsFromCLI(c), blockNumber, vpk) - if err != nil { - optInLogger.Error("failed to check opt-in status", "slot", slot, "error", err) - return - } - - err = db.UpdateValidatorOptInStatus(ctx, slot, opted) - if err != nil { - optInLogger.Error("failed to save opt-in status", "slot", slot, "error", err) - } else { - optInLogger.Info("validator opt-in status", "slot", slot, "opted_in", opted) - } - }(ei.Slot, ei.BlockNumber) - } - - lastBN = nextBN - if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { - progressLogger.Error("failed to save block number", "block", lastBN, "error", err) - } else { - progressLogger.Info("advanced to block", "block", lastBN) - } - } - } -} - -func main() { - flags := []cli.Flag{ - optionConfig, - optionDatabaseURL, - optionInfuraRPC, - optionBeaconBase, - optionBlockInterval, - optionValidatorDelay, - - optionBackfillLookback, - optionBackfillBatch, - optionHTTPTimeout, - optionOptInContract, - optionEtherscanKey, - } - - app := &cli.App{ - Name: "mev-indexer", - Usage: "Builder/observer indexer", - Commands: []*cli.Command{{ - Name: "start", - Usage: "Start the indexer", - Flags: flags, - Before: altsrc.InitInputSourceWithContext( - flags, altsrc.NewYamlSourceFromFlagFunc("config"), - ), - Action: func(c *cli.Context) error { - return startIndexer(c) - }, - }}, - } - ctx, cancel := context.WithCancel(context.Background()) - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigc - _, _ = fmt.Fprintln(app.Writer, "received interrupt signal, exiting... Force exit with Ctrl+C") - cancel() - <-sigc - _, _ = fmt.Fprintln(app.Writer, "force exiting...") - os.Exit(1) - }() - - if err := app.RunContext(ctx, os.Args); err != nil { - _, _ = fmt.Fprintf(app.Writer, "exited with error: %v\n", err) - } - -} diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index f2cbd86ec..c997946f5 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -2,20 +2,20 @@ package backfill import ( "context" - "log/slog" - "net/http" - "time" - + "fmt" "github.com/hashicorp/go-retryablehttp" "github.com/primev/mev-commit/tools/indexer/pkg/beacon" "github.com/primev/mev-commit/tools/indexer/pkg/config" "github.com/primev/mev-commit/tools/indexer/pkg/database" "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" "github.com/primev/mev-commit/tools/indexer/pkg/relay" + "log/slog" + "net/http" + "time" ) // RecentMissing backfills recent blocks that are missing data -func RecentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, lookback int64, batch int) error { +func recentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, lookback int64, batch int) error { logger := slog.With("component", "backfill") blocks, err := db.GetRecentMissingBlocks(ctx, lookback, batch) @@ -24,41 +24,43 @@ func RecentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Cl return err } - processed := 0 for _, block := range blocks { - + fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second) // Fetch beacon execution block data - if ei, err := beacon.FetchBeaconExecutionBlock(httpc, cfg.BeaconBase, block.BlockNumber); err == nil && ei != nil { - if err := db.UpsertBlockFromExec(ctx, ei); err != nil { - logger.Error("RecentMissing upsert failed", "slot", block.Slot, "error", err) + ei, ferr := beacon.FetchBeaconExecutionBlock(fetchCtx, httpc, cfg.BeaconBase, block.BlockNumber) + cancel() + if ferr != nil || ei == nil { + return fmt.Errorf("beacon fetch failed for block=%d: %w", block.BlockNumber, ferr) + } + if err := db.UpsertBlockFromExec(ctx, ei); err != nil { + logger.Error("RecentMissing upsert failed", "slot", block.Slot, "error", err) - continue - } + continue + } - // Schedule async validator pubkey fetch - if ei.ProposerIdx != nil { - go func(slot int64, idx int64) { - time.Sleep(cfg.ValidatorWait) - if vpub, err := beacon.FetchValidatorPubkey(httpc, cfg.BeaconBase, idx); err == nil && len(vpub) > 0 { - if err := db.UpdateValidatorPubkey(context.Background(), slot, vpub); err != nil { - logger.Error("RecentMissing validator pubkey update failed", "slot", slot, "error", err) - } - } - }(ei.Slot, *ei.ProposerIdx) - } - processed++ - } else { - logger.Error("RecentMissing beacon fetch failed", "block_number", block.BlockNumber, "error", err) + // Schedule async validator pubkey fetch + if ei.ProposerIdx != nil { + vctx, vcancel := context.WithTimeout(ctx, 5*time.Second) + vpub, verr := beacon.FetchValidatorPubkey(vctx, httpc, cfg.BeaconBase, *ei.ProposerIdx) + vcancel() + if verr != nil { + return fmt.Errorf("validator pubkey fetch failed slot=%d: %w", ei.Slot, verr) + } + if len(vpub) > 0 { + if err := db.UpdateValidatorPubkey(ctx, ei.Slot, vpub); err != nil { + return fmt.Errorf("validator pubkey update failed slot=%d: %w", ei.Slot, err) + } + } } } - logger.Info("RecentMissing processed", "blocks", processed) + logger.Info("RecentMissing processed", "blocks", len(blocks)) return nil } // RecentBids backfills bid data for ALL recent slots (not just opted-in blocks) -func RecentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, lookback int64, batch int) error { +func recentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, lookback int64, batch int) error { logger := slog.With("component", "backfill") opCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -69,17 +71,12 @@ func RecentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Clien } logger.Info("RecentBids fetched slots", "count", len(slots)) - processed := 0 - totalBids := 0 - const batchSize = 500 for _, slot := range slots { if ctx.Err() != nil { break } - slotBids := 0 - for _, rr := range relays { if ctx.Err() != nil { break @@ -100,36 +97,24 @@ func RecentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Clien } } - for i := 0; i < len(rows); i += batchSize { - end := i + batchSize - if end > len(rows) { - end = len(rows) - } + if len(rows) > 0 { insCtx, icancel := context.WithTimeout(ctx, 5*time.Second) - if err := db.InsertBidsBatch(insCtx, rows[i:end]); err != nil { - logger.Error("RecentBids batch insert failed", "slot", slot, "relay_id", rr.ID, "count", end-i, "error", err) - } else { - slotBids += end - i + if err := db.InsertBidsBatch(insCtx, rows); err != nil { + icancel() + return fmt.Errorf("bids insert failed slot=%d relay_id=%d: %w", slot, rr.ID, err) } icancel() - if ctx.Err() != nil { - break - } } } - if slotBids > 0 { - totalBids += slotBids - processed++ - } } - logger.Info("RecentBids processed", "slots", processed, "total_bids", totalBids) + logger.Info("RecentBids processed", "slots", len(slots)) return nil } // ValidatorOptIn backfills validator opt-in status (this is opt-in specific data) -func ValidatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, lookback int64, batch int) error { +func validatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, lookback int64, batch int) error { logger := slog.With("component", "backfill") validators, err := db.GetValidatorsNeedingOptInCheck(ctx, lookback, batch) if err != nil { @@ -137,90 +122,19 @@ func ValidatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cf return err } - processed := 0 - for _, v := range validators { opted, err := ethereum.CallAreOptedInAtBlock(httpc, cfg, v.BlockNumber, v.ValidatorPubkey) if err == nil { if err := db.UpdateValidatorOptInStatus(ctx, v.Slot, opted); err != nil { logger.Error("ValidatorOptIn update failed", "slot", v.Slot, "error", err) - } else { - processed++ } + } else { logger.Error("ValidatorOptIn check failed", "slot", v.Slot, "error", err) } } - logger.Info("ValidatorOptIn processed", "validators", processed) - return nil -} - -// AllBlocksBids ensures bid data is collected for ALL blocks, regardless of opt-in status -func AllBlocksBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, startSlot, endSlot int64) error { - logger := slog.With("component", "backfill") - logger.Info("AllBlocksBids starting", "start_slot", startSlot, "end_slot", endSlot) - const batchSize = 500 - totalProcessed := 0 - totalBids := 0 - - for slot := startSlot; slot <= endSlot; slot++ { - if ctx.Err() != nil { - break - } - slotBids := 0 - - for _, rr := range relays { - fetchCtx, fcancel := context.WithTimeout(ctx, 5*time.Second) - bids, err := relay.FetchBuilderBlocksReceived(fetchCtx, httpc, rr.URL, slot) - fcancel() - if err != nil { - logger.Error("AllBlocksBids fetch failed", "slot", slot, "relay_id", rr.ID, "url", rr.URL, "error", err) - continue - } - - rows := make([]database.BidRow, 0, len(bids)) - for _, b := range bids { - if row, ok := relay.BuildBidInsert(slot, rr.ID, b); ok { - rows = append(rows, row) - } - } - - for i := 0; i < len(rows); i += batchSize { - end := i + batchSize - if end > len(rows) { - end = len(rows) - } - - insCtx, icancel := context.WithTimeout(ctx, 5*time.Second) - if err := db.InsertBidsBatch(insCtx, rows[i:end]); err != nil { - logger.Error("AllBlocksBids batch insert failed", "slot", slot, "relay_id", rr.ID, "count", end-i, "error", err) - } else { - slotBids += end - i - } - icancel() - - if ctx.Err() != nil { - break - } - } - } - - if slotBids > 0 { - totalBids += slotBids - totalProcessed++ - } - - // Respect context cancellation - select { - case <-ctx.Done(): - logger.Warn("AllBlocksBids cancelled", "current_slot", slot) - return ctx.Err() - default: - } - } - - logger.Info("AllBlocksBids completed", "slots", totalProcessed, "total_bids", totalBids) + logger.Info("ValidatorOptIn processed", "validators", len(validators)) return nil } @@ -230,18 +144,18 @@ func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, c logger.Info("Starting comprehensive backfill for ALL blocks (not just opted-in)") // Run backfill operations - if err := RecentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + if err := recentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { logger.Error("RecentMissing failed", "error", err) } - if err := ValidatorOptIn(ctx, db, httpc.HTTPClient, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + if err := validatorOptIn(ctx, db, httpc.HTTPClient, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { logger.Error("ValidatorOptIn failed", "error", err) } // This ensures bid data for ALL blocks, not just mev-commit opted-in blocks - if err := RecentBids(ctx, db, httpc, relays, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + if err := recentBids(ctx, db, httpc, relays, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { logger.Error("RecentBids failed", "error", err) } - logger.Info("All operations completed - relay data covers ALL blocks") + logger.Info("Backfill-done") } diff --git a/tools/indexer/pkg/beacon/client.go b/tools/indexer/pkg/beacon/client.go index 9c05c6ed0..15c3e4b6e 100644 --- a/tools/indexer/pkg/beacon/client.go +++ b/tools/indexer/pkg/beacon/client.go @@ -28,10 +28,15 @@ type ExecInfo struct { RewardEth *float64 } -func FetchBeaconExecutionBlock(httpc *retryablehttp.Client, beaconBase string, blockNum int64) (*ExecInfo, error) { +func FetchBeaconExecutionBlock(ctx context.Context, httpc *retryablehttp.Client, beaconBase string, blockNum int64) (*ExecInfo, error) { url := fmt.Sprintf("%s/execution/block/%d", beaconBase, blockNum) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + + if _, has := ctx.Deadline(); !has { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + } + var wrap struct { Data []map[string]any `json:"data"` } @@ -120,10 +125,14 @@ func FetchBeaconExecutionBlock(httpc *retryablehttp.Client, beaconBase string, b } // validator pubkey from proposer index -func FetchValidatorPubkey(httpc *retryablehttp.Client, beaconBase string, proposerIndex int64) ([]byte, error) { +func FetchValidatorPubkey(ctx context.Context, httpc *retryablehttp.Client, beaconBase string, proposerIndex int64) ([]byte, error) { url := fmt.Sprintf("%s/validator/%d", beaconBase, proposerIndex) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + + if _, has := ctx.Deadline(); !has { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + } var resp struct { Data struct { Pubkey string `json:"pubkey"` @@ -139,7 +148,7 @@ func FetchValidatorPubkey(httpc *retryablehttp.Client, beaconBase string, propos } // to fetch blocks from Alchemy RPC -func FetchBlockFromRPC(httpc *retryablehttp.Client, rpcURL string, blockNumber int64) (*ExecInfo, error) { +func fetchBlockFromRPC(httpc *retryablehttp.Client, rpcURL string, blockNumber int64) (*ExecInfo, error) { underlyingClient := httpc.HTTPClient // Get block data from Alchemy payload := map[string]any{ @@ -185,9 +194,9 @@ func FetchBlockFromRPC(httpc *retryablehttp.Client, rpcURL string, blockNumber i Timestamp: &blockTime, }, nil } -func FetchCombinedBlockData(httpc *retryablehttp.Client, rpcURL string, beaconBase string, blockNumber int64) (*ExecInfo, error) { +func FetchCombinedBlockData(ctx context.Context, httpc *retryablehttp.Client, rpcURL string, beaconBase string, blockNumber int64) (*ExecInfo, error) { // Get execution block from Alchemy (always available) - execBlock, err := FetchBlockFromRPC(httpc, rpcURL, blockNumber) + execBlock, err := fetchBlockFromRPC(httpc, rpcURL, blockNumber) if err != nil { return nil, err } @@ -195,7 +204,7 @@ func FetchCombinedBlockData(httpc *retryablehttp.Client, rpcURL string, beaconBa // Convert block number to slot for beacon chain query slotNumber := ethereum.BlockNumberToSlot(blockNumber) - beaconData, _ := FetchBeaconExecutionBlock(httpc, beaconBase, slotNumber) + beaconData, _ := FetchBeaconExecutionBlock(ctx, httpc, beaconBase, slotNumber) // Merge data - use Alchemy as primary, beacon as supplement if beaconData != nil { diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go index 1bed4f7d1..0d72cb71b 100644 --- a/tools/indexer/pkg/database/starrock.go +++ b/tools/indexer/pkg/database/starrock.go @@ -17,15 +17,16 @@ type DB struct { conn *sql.DB } type BidInsert struct { - Slot int64 - RelayID int64 - BuilderHex string - ProposerHex string - FeeRecHex string - ValStr string - BlockNum *int64 - TsMS *int64 + Slot int64 + RelayID int64 + BuilderHex string + ProposerHex string + FeeRecHex string + ValStr string + BlockNum *int64 + TsMS *int64 } + func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, error) { conn, err := sql.Open("mysql", dsn) if err != nil { @@ -115,11 +116,11 @@ func (db *DB) SaveLastBlockNumber(ctx context.Context, bn int64) error { ctx2, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() - query := fmt.Sprintf(`REPLACE INTO ingestor_state (id, last_block_number) VALUES (1, %d)`, bn) - _, err := db.conn.ExecContext(ctx2, query) - if err != nil { - return fmt.Errorf("save last_block_number failed: %w", err) + q2 := fmt.Sprintf(`INSERT INTO ingestor_state (id, last_block_number) VALUES (1, %d)`, bn) + if _, err := db.conn.ExecContext(ctx2, q2); err != nil { + return fmt.Errorf("save last_block_number failed (insert): %w", err) } + return nil } @@ -214,75 +215,51 @@ func (db *DB) UpdateValidatorPubkey(ctx context.Context, slot int64, vpub []byte return nil } -// // Add these new methods to your DB struct -// func (db *DB) InsertBid(ctx context.Context, slot int64, relayID int64, builder, proposer, feeRec []byte, valStr string, blockNum *int64, tsMS *int64) error { -// ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) -// defer cancel() -// builderHex := hexutil.Encode(builder) -// proposerHex := hexutil.Encode(proposer) -// feeRecHex := hexutil.Encode(feeRec) -// blockNumSQL := "NULL" -// if blockNum != nil { -// blockNumSQL = fmt.Sprintf("%d", *blockNum) -// } - -// tsMSSQL := "NULL" -// if tsMS != nil { -// tsMSSQL = fmt.Sprintf("%d", *tsMS) -// } -// query := fmt.Sprintf(` -// INSERT INTO bids( -// slot, relay_id, builder_pubkey, proposer_pubkey, -// proposer_fee_recipient, value_wei, block_number, timestamp_ms -// ) -// VALUES (%d, %d, '%s', '%s', '%s', '%s', %s, %s)`, -// slot, relayID, builderHex, proposerHex, feeRecHex, valStr, blockNumSQL, tsMSSQL, -// ) - -// _, err := db.conn.ExecContext(ctx2, query) -// return err -// } // Minimal batching: builds one multi-VALUES INSERT. -// NOTE: relies on same string building pattern you already use. + type BidRow struct { - Slot, RelayID int64 - Builder, Proposer, FeeRec string - ValStr string - BlockNum, TsMS *int64 + Slot, RelayID int64 + Builder, Proposer, FeeRec string + ValStr string + BlockNum, TsMS *int64 } func (db *DB) InsertBidsBatch(ctx context.Context, rows []BidRow) error { - if len(rows) == 0 { return nil } + if len(rows) == 0 { + return nil + } - ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() + ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() - var sb strings.Builder - sb.WriteString(` + var sb strings.Builder + sb.WriteString(` INSERT INTO bids( slot, relay_id, builder_pubkey, proposer_pubkey, proposer_fee_recipient, value_wei, block_number, timestamp_ms ) VALUES `) - for i, r := range rows { - if i > 0 { sb.WriteString(",") } - - // builderHex := hexutil.Encode(r.Builder) - // proposerHex := hexutil.Encode(r.Proposer) - // feeRecHex := hexutil.Encode(r.FeeRec) + for i, r := range rows { + if i > 0 { + sb.WriteString(",") + } - blockNumSQL := "NULL" - if r.BlockNum != nil { blockNumSQL = fmt.Sprintf("%d", *r.BlockNum) } + blockNumSQL := "NULL" + if r.BlockNum != nil { + blockNumSQL = fmt.Sprintf("%d", *r.BlockNum) + } - tsMSSQL := "NULL" - if r.TsMS != nil { tsMSSQL = fmt.Sprintf("%d", *r.TsMS) } + tsMSSQL := "NULL" + if r.TsMS != nil { + tsMSSQL = fmt.Sprintf("%d", *r.TsMS) + } - fmt.Fprintf(&sb, "(%d,%d,'%s','%s','%s','%s',%s,%s)", - r.Slot, r.RelayID, r.Builder, r.Proposer, r.FeeRec, r.ValStr, blockNumSQL, tsMSSQL) - } + fmt.Fprintf(&sb, "(%d,%d,'%s','%s','%s','%s',%s,%s)", + r.Slot, r.RelayID, r.Builder, r.Proposer, r.FeeRec, r.ValStr, blockNumSQL, tsMSSQL) + } - _, err := db.conn.ExecContext(ctx2, sb.String()) - return err + _, err := db.conn.ExecContext(ctx2, sb.String()) + return err } func (db *DB) GetActiveRelays(ctx context.Context) ([]struct { diff --git a/tools/indexer/pkg/relay/client.go b/tools/indexer/pkg/relay/client.go index 6fcf2f693..1eafb7d9f 100644 --- a/tools/indexer/pkg/relay/client.go +++ b/tools/indexer/pkg/relay/client.go @@ -57,7 +57,7 @@ func parseBigString(v any) (string, bool) { } } -func BuildBidInsert(slot int64, relayID int64, bid map[string]any) (database.BidRow, bool){ +func BuildBidInsert(slot int64, relayID int64, bid map[string]any) (database.BidRow, bool) { if slot <= 0 || relayID <= 0 { return database.BidRow{}, false @@ -80,7 +80,7 @@ func BuildBidInsert(slot int64, relayID int64, bid map[string]any) (database.Bid valStr, ok := parseBigString(get("value", "value_wei", "valueWei")) if !ok || valStr == "" { - return database.BidRow{}, false // skip if no value + return database.BidRow{}, false // skip if no value } var blockNum *int64 @@ -114,19 +114,16 @@ func BuildBidInsert(slot int64, relayID int64, bid map[string]any) (database.Bid } } - // ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) - // defer cancel() - return database.BidRow{ - Slot: slot, - RelayID: relayID, - Builder: hexutil.Encode(builder), - Proposer: hexutil.Encode(proposer), - FeeRec: hexutil.Encode(feeRec), - ValStr: valStr, - BlockNum: blockNum, - TsMS: tsMS, - }, true + Slot: slot, + RelayID: relayID, + Builder: hexutil.Encode(builder), + Proposer: hexutil.Encode(proposer), + FeeRec: hexutil.Encode(feeRec), + ValStr: valStr, + BlockNum: blockNum, + TsMS: tsMS, + }, true } func UpsertRelaysAndLoad(ctx context.Context, db *database.DB) ([]Row, error) { From 18b2f5c43526a1653ef3084cbc4b6db3f4091eec Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Mon, 6 Oct 2025 17:41:07 +0530 Subject: [PATCH 06/18] fixed lint issues --- tools/indexer/cmd/start.go | 12 ++++++++---- tools/indexer/pkg/beacon/client.go | 2 +- tools/indexer/pkg/database/starrock.go | 11 ++++++----- tools/indexer/pkg/ethereum/client.go | 2 +- tools/indexer/pkg/http/client.go | 2 +- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 88e59cd7d..abb2bc8e2 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -3,7 +3,7 @@ package main import ( "context" "log/slog" - "math/rand" + "os/signal" "syscall" "time" @@ -25,8 +25,6 @@ func startIndexer(c *cli.Context) error { dbURL := c.String(optionDatabaseURL.Name) infuraRPC := c.String(optionInfuraRPC.Name) beaconBase := c.String(optionBeaconBase.Name) - // Initialize random seed - rand.Seed(time.Now().UnixNano()) initLogger.Info("starting blockchain indexer with StarRocks database") initLogger.Info("configuration loaded", @@ -41,8 +39,14 @@ func startIndexer(c *cli.Context) error { db, err := database.MustConnect(ctx, dbURL, 20, 5) if err != nil { initLogger.Error("[DB] connection failed", "error", err) + return err } - defer db.Close() + + defer func() { + if cerr := db.Close(); cerr != nil { + initLogger.Error("[DB] close failed", "error", cerr) + } + }() initLogger.Info("[DB] connected to StarRocks database") // Ensure required tables exist diff --git a/tools/indexer/pkg/beacon/client.go b/tools/indexer/pkg/beacon/client.go index 15c3e4b6e..84aeabfcc 100644 --- a/tools/indexer/pkg/beacon/client.go +++ b/tools/indexer/pkg/beacon/client.go @@ -166,7 +166,7 @@ func fetchBlockFromRPC(httpc *retryablehttp.Client, rpcURL string, blockNumber i if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() var result struct { Result struct { diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go index 0d72cb71b..4f30f94ce 100644 --- a/tools/indexer/pkg/database/starrock.go +++ b/tools/indexer/pkg/database/starrock.go @@ -41,7 +41,8 @@ func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() if err := conn.PingContext(pingCtx); err != nil { - conn.Close() + _ = conn.Close() + return nil, fmt.Errorf("StarRocks ping failed: %v", err) } @@ -273,7 +274,7 @@ func (db *DB) GetActiveRelays(ctx context.Context) ([]struct { if err != nil { return nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() var results []struct { ID int64 @@ -326,7 +327,7 @@ func (db *DB) GetRecentMissingBlocks(ctx context.Context, lookback int64, batch if err != nil { return nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() var results []struct { Slot int64 @@ -361,7 +362,7 @@ LIMIT %d`, batch) if err != nil { return nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() var slots []int64 for rows.Next() { @@ -396,7 +397,7 @@ LIMIT %d`, batch) if err != nil { return nil, err } - defer rows.Close() + defer func() { _ = rows.Close() }() var results []struct { Slot int64 diff --git a/tools/indexer/pkg/ethereum/client.go b/tools/indexer/pkg/ethereum/client.go index ea4b601c7..31b3b8d71 100644 --- a/tools/indexer/pkg/ethereum/client.go +++ b/tools/indexer/pkg/ethereum/client.go @@ -60,7 +60,7 @@ func GetLatestBlockNumber(httpc *http.Client, rpcURL string) (int64, error) { if err != nil { return 0, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() var result struct { Result string `json:"result"` diff --git a/tools/indexer/pkg/http/client.go b/tools/indexer/pkg/http/client.go index 77515b8ec..945d6d451 100644 --- a/tools/indexer/pkg/http/client.go +++ b/tools/indexer/pkg/http/client.go @@ -29,7 +29,7 @@ func FetchJSON(ctx context.Context, client *retryablehttp.Client, url string, ou if err != nil { return err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { return fmt.Errorf("HTTP %d", resp.StatusCode) From 51afda452973146acc136e65f6383ffb716eb67c Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Mon, 6 Oct 2025 18:27:28 +0530 Subject: [PATCH 07/18] fixed formatting issues --- tools/indexer/cmd/main.go | 1 + tools/indexer/cmd/start.go | 47 +++++++++++++------------- tools/indexer/pkg/backfill/backfill.go | 10 +++--- tools/indexer/pkg/database/starrock.go | 2 +- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/tools/indexer/cmd/main.go b/tools/indexer/cmd/main.go index 9f28deb9c..f8df61cf5 100644 --- a/tools/indexer/cmd/main.go +++ b/tools/indexer/cmd/main.go @@ -133,6 +133,7 @@ func main() { }}, } ctx, cancel := context.WithCancel(context.Background()) + defer cancel() sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) go func() { diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index abb2bc8e2..65378ca09 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -3,9 +3,6 @@ package main import ( "context" "log/slog" - - "os/signal" - "syscall" "time" _ "github.com/go-sql-driver/mysql" @@ -30,13 +27,9 @@ func startIndexer(c *cli.Context) error { initLogger.Info("configuration loaded", "block_interval", c.Duration("block-interval"), "validator_delay", c.Duration("validator-delay")) - - // Setup graceful shutdown - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer cancel() - + ctx := c.Context // Connect to StarRocks database - db, err := database.MustConnect(ctx, dbURL, 20, 5) + db, err := database.Connect(ctx, dbURL, 20, 5) if err != nil { initLogger.Error("[DB] connection failed", "error", err) return err @@ -80,13 +73,13 @@ func startIndexer(c *cli.Context) error { } } - // Replace the hardcoded block search with: if lastBN == 0 { initLogger.Info("getting latest block from Ethereum RPC...") latestBlock, err := ethereum.GetLatestBlockNumber(httpc.HTTPClient, infuraRPC) if err != nil { initLogger.Error("failed to get latest block from RPC", "error", err) + return err } lastBN = latestBlock - 10 // Start 10 blocks behind to ensure data availability @@ -100,11 +93,15 @@ func startIndexer(c *cli.Context) error { initLogger.Info("[BACKFILL] running one-time backfill", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) - go backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays) - initLogger.Info("[BACKFILL] completed startup backfill") + if err := backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays); err != nil { + initLogger.Error("[BACKFILL] failed", "error", err) + } else { + initLogger.Info("[BACKFILL] completed startup backfill") + } } else { initLogger.Info("[BACKFILL] skipped", "reason", "backfill-lookback=0") } + mainTicker := time.NewTicker(c.Duration("block-interval")) defer mainTicker.Stop() @@ -126,28 +123,32 @@ func startIndexer(c *cli.Context) error { ei, err := beacon.FetchCombinedBlockData(ctx, httpc, infuraRPC, beaconBase, nextBN) if err != nil || ei == nil { initLogger.Warn("[BLOCK] not available yet", "block", nextBN, "error", err) - continue } - - // Log block details - initLogger.Info("[BLOCK] processing block", "block", nextBN, "slot", ei.Slot) - + fields := []any{ + "block", nextBN, + "slot", ei.Slot, + } if ei.Timestamp != nil { - initLogger.Info("[BLOCK] block timestamp", "block", nextBN, "timestamp", ei.Timestamp.Format(time.RFC3339)) + fields = append(fields, "timestamp", ei.Timestamp.Format(time.RFC3339)) } if ei.ProposerIdx != nil { - initLogger.Info("[VALIDATOR] proposer index", "index", *ei.ProposerIdx) + fields = append(fields, "proposer_index", *ei.ProposerIdx) } if ei.RelayTag != nil { - initLogger.Info("[RELAY] winning relay", "tag", *ei.RelayTag) + fields = append(fields, "winning_relay", *ei.RelayTag) } - if ei.BuilderHex != nil && len(*ei.BuilderHex) > 20 { - initLogger.Info("[BLOCK] builder pubkey", "pubkey_prefix", (*ei.BuilderHex)[:20]) + if ei.BuilderHex != nil { + pref := *ei.BuilderHex + if len(pref) > 20 { + pref = pref[:20] + } + fields = append(fields, "builder_pubkey_prefix", pref) } if ei.RewardEth != nil { - initLogger.Info("[BLOCK] producer reward", "reward_eth", *ei.RewardEth) + fields = append(fields, "producer_reward_eth", *ei.RewardEth) } + initLogger.Info("processing block", fields...) // Save block to database if err := db.UpsertBlockFromExec(ctx, ei); err != nil { diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index c997946f5..b4b18fad1 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -139,23 +139,23 @@ func validatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cf } // RunAll executes all backfill operations ensuring complete coverage -func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, relays []relay.Row) { +func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, relays []relay.Row) error { logger := slog.With("component", "backfill") logger.Info("Starting comprehensive backfill for ALL blocks (not just opted-in)") - // Run backfill operations if err := recentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { logger.Error("RecentMissing failed", "error", err) + return err } - if err := validatorOptIn(ctx, db, httpc.HTTPClient, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { logger.Error("ValidatorOptIn failed", "error", err) + return err } - - // This ensures bid data for ALL blocks, not just mev-commit opted-in blocks if err := recentBids(ctx, db, httpc, relays, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { logger.Error("RecentBids failed", "error", err) + return err } logger.Info("Backfill-done") + return nil } diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go index 4f30f94ce..d30a38438 100644 --- a/tools/indexer/pkg/database/starrock.go +++ b/tools/indexer/pkg/database/starrock.go @@ -27,7 +27,7 @@ type BidInsert struct { TsMS *int64 } -func MustConnect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, error) { +func Connect(ctx context.Context, dsn string, maxConns, minConns int) (*DB, error) { conn, err := sql.Open("mysql", dsn) if err != nil { return nil, fmt.Errorf("failed to open StarRocks connection: %w", err) From b9140d2aa4894ab7fe15720b9cf753607c26a1d2 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Mon, 6 Oct 2025 20:40:49 +0530 Subject: [PATCH 08/18] updated ctx for a func --- tools/indexer/cmd/start.go | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 65378ca09..7ddf02af2 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -125,30 +125,15 @@ func startIndexer(c *cli.Context) error { initLogger.Warn("[BLOCK] not available yet", "block", nextBN, "error", err) continue } - fields := []any{ + initLogger.Info("processing block", "block", nextBN, "slot", ei.Slot, - } - if ei.Timestamp != nil { - fields = append(fields, "timestamp", ei.Timestamp.Format(time.RFC3339)) - } - if ei.ProposerIdx != nil { - fields = append(fields, "proposer_index", *ei.ProposerIdx) - } - if ei.RelayTag != nil { - fields = append(fields, "winning_relay", *ei.RelayTag) - } - if ei.BuilderHex != nil { - pref := *ei.BuilderHex - if len(pref) > 20 { - pref = pref[:20] - } - fields = append(fields, "builder_pubkey_prefix", pref) - } - if ei.RewardEth != nil { - fields = append(fields, "producer_reward_eth", *ei.RewardEth) - } - initLogger.Info("processing block", fields...) + "timestamp", ei.Timestamp, + "proposer_index", ei.ProposerIdx, + "winning_relay", ei.RelayTag, + "builder_pubkey_prefix", ei.BuilderHex, + "producer_reward_eth", ei.RewardEth, + ) // Save block to database if err := db.UpsertBlockFromExec(ctx, ei); err != nil { @@ -206,11 +191,13 @@ func startIndexer(c *cli.Context) error { // final flush if len(batch) > 0 { - if err := db.InsertBidsBatch(ctx, batch); err != nil { + flushCtx, flushCancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := db.InsertBidsBatch(flushCtx, batch); err != nil { initLogger.Error("[DB] batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) } else { relayBids += len(batch) } + flushCancel() } if mainContextCanceled { @@ -274,11 +261,13 @@ func startIndexer(c *cli.Context) error { } lastBN = nextBN - if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { + gctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := db.SaveLastBlockNumber(gctx, lastBN); err != nil { initLogger.Error("[PROGRESS] failed to save block number", "block", lastBN, "error", err) } else { initLogger.Info("[PROGRESS] advanced to block", "block", lastBN) } + cancel() } } } From 30cfa45301592d15658834364a3f6588114305d6 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Mon, 6 Oct 2025 22:34:41 +0530 Subject: [PATCH 09/18] improved backfill logic andrefactorign of start.go --- tools/indexer/cmd/start.go | 424 ++++++++++++++----------- tools/indexer/pkg/backfill/backfill.go | 95 +++--- 2 files changed, 289 insertions(+), 230 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 7ddf02af2..8f1f0212c 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -6,268 +6,332 @@ import ( "time" _ "github.com/go-sql-driver/mysql" + "github.com/hashicorp/go-retryablehttp" "github.com/primev/mev-commit/tools/indexer/pkg/backfill" "github.com/primev/mev-commit/tools/indexer/pkg/beacon" "github.com/primev/mev-commit/tools/indexer/pkg/database" "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" httputil "github.com/primev/mev-commit/tools/indexer/pkg/http" "github.com/primev/mev-commit/tools/indexer/pkg/relay" + "github.com/urfave/cli/v2" ) -func startIndexer(c *cli.Context) error { - - initLogger := slog.With("component", "init") - - dbURL := c.String(optionDatabaseURL.Name) - infuraRPC := c.String(optionInfuraRPC.Name) - beaconBase := c.String(optionBeaconBase.Name) - - initLogger.Info("starting blockchain indexer with StarRocks database") - initLogger.Info("configuration loaded", - "block_interval", c.Duration("block-interval"), - "validator_delay", c.Duration("validator-delay")) - ctx := c.Context - // Connect to StarRocks database +func initializeDatabase(ctx context.Context, dbURL string, logger *slog.Logger) (*database.DB, error) { db, err := database.Connect(ctx, dbURL, 20, 5) if err != nil { - initLogger.Error("[DB] connection failed", "error", err) - return err + logger.Error("[DB] connection failed", "error", err) + return nil, err } + logger.Info("[DB] connected to StarRocks database") - defer func() { - if cerr := db.Close(); cerr != nil { - initLogger.Error("[DB] close failed", "error", cerr) - } - }() - initLogger.Info("[DB] connected to StarRocks database") - - // Ensure required tables exist if err := db.EnsureStateTable(ctx); err != nil { - initLogger.Error("[DB] failed to ensure state table", "error", err) - return err + logger.Error("[DB] failed to ensure state table", "error", err) + return nil, err } - initLogger.Info("[DB] state table ready") + logger.Info("[DB] state table ready") - // Initialize HTTP client - httpc := httputil.NewHTTPClient(c.Duration("http-timeout")) - initLogger.Info("[HTTP] client initialized", "timeout", c.Duration("http-timeout")) + return db, nil +} - // Load relay configurations +func closeDatabase(db *database.DB, logger *slog.Logger) { + if cerr := db.Close(); cerr != nil { + logger.Error("[DB] close failed", "error", cerr) + } +} + +func loadRelays(ctx context.Context, db *database.DB, logger *slog.Logger) ([]relay.Row, error) { relays, err := relay.UpsertRelaysAndLoad(ctx, db) if err != nil { - initLogger.Error("[RELAY] failed to load", "error", err) + logger.Error("[RELAY] failed to load", "error", err) + return nil, err } - initLogger.Info("[RELAY] loaded active relays", "count", len(relays)) + + logger.Info("[RELAY] loaded active relays", "count", len(relays)) for _, r := range relays { - initLogger.Info("[RELAY] relay found", "id", r.ID, "url", r.URL) + logger.Info("[RELAY] relay found", "id", r.ID, "url", r.URL) } - // Initialize starting block number + return relays, nil +} + +func getStartingBlockNumber(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, infuraRPC string, logger *slog.Logger) (int64, error) { lastBN, found := db.LoadLastBlockNumber(ctx) + if !found || lastBN == 0 { - initLogger.Info("no previous state found, checking database for latest block") + logger.Info("no previous state found, checking database for latest block") + var err error lastBN, err = db.GetMaxBlockNumber(ctx) if err != nil { - initLogger.Error("database query failed", "error", err) + logger.Error("database query failed", "error", err) } } if lastBN == 0 { - initLogger.Info("getting latest block from Ethereum RPC...") + logger.Info("getting latest block from Ethereum RPC...") latestBlock, err := ethereum.GetLatestBlockNumber(httpc.HTTPClient, infuraRPC) if err != nil { - initLogger.Error("failed to get latest block from RPC", "error", err) - return err + logger.Error("failed to get latest block from RPC", "error", err) + return 0, err } lastBN = latestBlock - 10 // Start 10 blocks behind to ensure data availability - initLogger.Info("starting from block", "block", lastBN, "latest", latestBlock) + logger.Info("starting from block", "block", lastBN, "latest", latestBlock) } + return lastBN, nil +} - initLogger.Info("starting from block number", "block", lastBN) - initLogger.Info("indexer configuration", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) +func runBackfillIfConfigured(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, logger *slog.Logger) { + logger.Info("indexer configuration", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) if c.Int("backfill-lookback") > 0 { - initLogger.Info("[BACKFILL] running one-time backfill", + logger.Info("[BACKFILL] running one-time backfill", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) if err := backfill.RunAll(ctx, db, httpc, createOptionsFromCLI(c), relays); err != nil { - initLogger.Error("[BACKFILL] failed", "error", err) + logger.Error("[BACKFILL] failed", "error", err) } else { - initLogger.Info("[BACKFILL] completed startup backfill") + logger.Info("[BACKFILL] completed startup backfill") } } else { - initLogger.Info("[BACKFILL] skipped", "reason", "backfill-lookback=0") + logger.Info("[BACKFILL] skipped", "reason", "backfill-lookback=0") } +} +func runMainLoop(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, infuraRPC, beaconBase string, startBN int64, logger *slog.Logger) error { mainTicker := time.NewTicker(c.Duration("block-interval")) defer mainTicker.Stop() - // Main processing loop + lastBN := startBN + for { select { case <-ctx.Done(): - initLogger.Info("[SHUTDOWN] graceful shutdown initiated", "reason", ctx.Err()) - if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { - initLogger.Error("[SHUTDOWN] failed to save last block number", "error", err) - } - initLogger.Info("[SHUTDOWN] indexer stopped", "block", lastBN) - return nil + return handleShutdown(ctx, db, lastBN, logger) case <-mainTicker.C: - nextBN := lastBN + 1 + lastBN = processNextBlock(ctx, c, db, httpc, relays, infuraRPC, beaconBase, lastBN, logger) + } + } +} - // Fetch execution block data - ei, err := beacon.FetchCombinedBlockData(ctx, httpc, infuraRPC, beaconBase, nextBN) - if err != nil || ei == nil { - initLogger.Warn("[BLOCK] not available yet", "block", nextBN, "error", err) - continue - } - initLogger.Info("processing block", - "block", nextBN, - "slot", ei.Slot, - "timestamp", ei.Timestamp, - "proposer_index", ei.ProposerIdx, - "winning_relay", ei.RelayTag, - "builder_pubkey_prefix", ei.BuilderHex, - "producer_reward_eth", ei.RewardEth, - ) - - // Save block to database - if err := db.UpsertBlockFromExec(ctx, ei); err != nil { - initLogger.Error("[DB] failed to save block", "block", nextBN, "error", err) - continue - } - initLogger.Info("[DB] block saved successfully", "block", nextBN) - - // Fetch and store bid data from all relays - totalBids := 0 - successfulRelays := 0 - mainContextCanceled := false - const batchSize = 500 - for _, rr := range relays { - // Check if main context is canceled before processing each relay - if ctx.Err() != nil { - initLogger.Warn("main context canceled, stopping relay processing") - mainContextCanceled = true - break - } +func handleShutdown(ctx context.Context, db *database.DB, lastBN int64, logger *slog.Logger) error { + logger.Info("[SHUTDOWN] graceful shutdown initiated", "reason", ctx.Err()) + if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { + logger.Error("[SHUTDOWN] failed to save last block number", "error", err) + } + logger.Info("[SHUTDOWN] indexer stopped", "block", lastBN) + return nil +} - bids, err := relay.FetchBuilderBlocksReceived(ctx, httpc, rr.URL, ei.Slot) - if err != nil { - initLogger.Error("[RELAY] failed to fetch bids", "relay_id", rr.ID, "url", rr.URL, "error", err) - continue - } +func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, infuraRPC, beaconBase string, lastBN int64, logger *slog.Logger) int64 { + nextBN := lastBN + 1 - relayBids := 0 - batch := make([]database.BidRow, 0, batchSize) + ei, err := beacon.FetchCombinedBlockData(ctx, httpc, infuraRPC, beaconBase, nextBN) + if err != nil || ei == nil { + logger.Warn("[BLOCK] not available yet", "block", nextBN, "error", err) + return lastBN + } - for _, bid := range bids { - // Check if main context is still valid - if ctx.Err() != nil { - initLogger.Warn("[BIDS] main context canceled, stopping bid insertion") - mainContextCanceled = true - break - } + logger.Info("processing block", + "block", nextBN, + "slot", ei.Slot, + "timestamp", ei.Timestamp, + "proposer_index", ei.ProposerIdx, + "winning_relay", ei.RelayTag, + "builder_pubkey_prefix", ei.BuilderHex, + "producer_reward_eth", ei.RewardEth, + ) + + if err := db.UpsertBlockFromExec(ctx, ei); err != nil { + logger.Error("[DB] failed to save block", "block", nextBN, "error", err) + return lastBN + } + logger.Info("[DB] block saved successfully", "block", nextBN) - if row, ok := relay.BuildBidInsert(ei.Slot, rr.ID, bid); ok { - batch = append(batch, row) + processBidsForBlock(ctx, db, httpc, relays, ei, logger) + launchAsyncValidatorTasks(ctx, c, db, httpc, ei, beaconBase, logger) - if len(batch) >= batchSize { - insCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - if err := db.InsertBidsBatch(insCtx, batch); err != nil { + saveBlockProgress(db, nextBN, logger) + return nextBN +} - initLogger.Error("[DB]batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) - } else { - relayBids += len(batch) - } - cancel() - batch = batch[:0] - } - } - } +func processBidsForBlock(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, ei *beacon.ExecInfo, logger *slog.Logger) { + + // Fetch and store bid data from all relays + totalBids := 0 + successfulRelays := 0 + mainContextCanceled := false + const batchSize = 500 + for _, rr := range relays { + // Check if main context is canceled before processing each relay + if ctx.Err() != nil { + logger.Warn("main context canceled, stopping relay processing") + mainContextCanceled = true + break + } + + bids, err := relay.FetchBuilderBlocksReceived(ctx, httpc, rr.URL, ei.Slot) + if err != nil { + logger.Error("[RELAY] failed to fetch bids", "relay_id", rr.ID, "url", rr.URL, "error", err) + continue + } + + relayBids := 0 + batch := make([]database.BidRow, 0, batchSize) - // final flush - if len(batch) > 0 { - flushCtx, flushCancel := context.WithTimeout(context.Background(), 5*time.Second) - if err := db.InsertBidsBatch(flushCtx, batch); err != nil { - initLogger.Error("[DB] batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) + for _, bid := range bids { + // Check if main context is still valid + if ctx.Err() != nil { + logger.Warn("[BIDS] main context canceled, stopping bid insertion") + mainContextCanceled = true + break + } + + if row, ok := relay.BuildBidInsert(ei.Slot, rr.ID, bid); ok { + batch = append(batch, row) + + if len(batch) >= batchSize { + insCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + if err := db.InsertBidsBatch(insCtx, batch); err != nil { + + logger.Error("[DB]batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) } else { relayBids += len(batch) } - flushCancel() - } - - if mainContextCanceled { - break + cancel() + batch = batch[:0] } + } + } - if relayBids > 0 { - initLogger.Info("[BIDS] bids collected", "relay_id", rr.ID, "count", relayBids) - totalBids += relayBids - successfulRelays++ - } + // final flush + if len(batch) > 0 { + flushCtx, flushCancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := db.InsertBidsBatch(flushCtx, batch); err != nil { + logger.Error("[DB] batch insert failed", "slot", ei.Slot, "relay_id", rr.ID, "count", len(batch), "error", err) + } else { + relayBids += len(batch) } + flushCancel() + } - initLogger.Info("[BIDS] summary", "block", nextBN, "total_bids", totalBids, "successful_relays", successfulRelays) - // Async validator pubkey fetch - if ei.ProposerIdx != nil { - go func(slot int64, proposerIdx int64) { - time.Sleep(c.Duration("validator-delay")) + if mainContextCanceled { + break + } - vpub, err := beacon.FetchValidatorPubkey(ctx, httpc, beaconBase, proposerIdx) - if err != nil { - initLogger.Error("[VALIDATOR] failed to fetch pubkey", "proposer", proposerIdx, "error", err) - return - } + if relayBids > 0 { + logger.Info("[BIDS] bids collected", "relay_id", rr.ID, "count", relayBids) + totalBids += relayBids + successfulRelays++ + } + } + logger.Info("[BIDS] summary", "block", ei.BlockNumber, "total_bids", totalBids, "successful_relays", successfulRelays) - if len(vpub) > 0 { - if err := db.UpdateValidatorPubkey(ctx, slot, vpub); err != nil { - initLogger.Error("[VALIDATOR] failed to save pubkey", "slot", slot, "error", err) - } else { - initLogger.Info("[VALIDATOR] pubkey saved", "proposer", proposerIdx, "slot", slot) - } - } - }(ei.Slot, *ei.ProposerIdx) +} + +func saveBlockProgress(db *database.DB, blockNum int64, logger *slog.Logger) { + gctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := db.SaveLastBlockNumber(gctx, blockNum); err != nil { + logger.Error("[PROGRESS] failed to save block number", "block", blockNum, "error", err) + } else { + logger.Info("[PROGRESS] advanced to block", "block", blockNum) + } + +} + +func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, ei *beacon.ExecInfo, beaconBase string, logger *slog.Logger) { // Async validator pubkey fetch + if ei.ProposerIdx != nil { + go func(slot int64, proposerIdx int64) { + time.Sleep(c.Duration("validator-delay")) + + vpub, err := beacon.FetchValidatorPubkey(ctx, httpc, beaconBase, proposerIdx) + if err != nil { + logger.Error("[VALIDATOR] failed to fetch pubkey", "proposer", proposerIdx, "error", err) + return } - // Async opt-in status check - if ei.ProposerIdx != nil { - go func(slot int64, blockNumber int64) { - time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) + if len(vpub) > 0 { + if err := db.UpdateValidatorPubkey(ctx, slot, vpub); err != nil { + logger.Error("[VALIDATOR] failed to save pubkey", "slot", slot, "error", err) + } else { + logger.Info("[VALIDATOR] pubkey saved", "proposer", proposerIdx, "slot", slot) + } + } + }(ei.Slot, *ei.ProposerIdx) + } - // Wait for validator pubkey to be available - vpk, err := db.GetValidatorPubkeyWithRetry(ctx, slot, 3, time.Second) - if err != nil { - initLogger.Error("[VALIDATOR] pubkey not available", "slot", slot, "error", err) - return - } + // Async opt-in status check + if ei.ProposerIdx != nil { + go func(slot int64, blockNumber int64) { + time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) - opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, createOptionsFromCLI(c), blockNumber, vpk) - if err != nil { - initLogger.Error("[OPT-IN] failed to check opt-in status", "slot", slot, "error", err) - return - } + // Wait for validator pubkey to be available + vpk, err := db.GetValidatorPubkeyWithRetry(ctx, slot, 3, time.Second) + if err != nil { + logger.Error("[VALIDATOR] pubkey not available", "slot", slot, "error", err) + return + } - err = db.UpdateValidatorOptInStatus(ctx, slot, opted) - if err != nil { - initLogger.Error("[OPT-IN] failed to save opt-in status", "slot", slot, "error", err) - } else { - initLogger.Info("[OPT-IN] validator opt-in status", "slot", slot, "opted_in", opted) - } - }(ei.Slot, ei.BlockNumber) + opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, createOptionsFromCLI(c), blockNumber, vpk) + if err != nil { + logger.Error("[OPT-IN] failed to check opt-in status", "slot", slot, "error", err) + return } - lastBN = nextBN - gctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - if err := db.SaveLastBlockNumber(gctx, lastBN); err != nil { - initLogger.Error("[PROGRESS] failed to save block number", "block", lastBN, "error", err) + err = db.UpdateValidatorOptInStatus(ctx, slot, opted) + if err != nil { + logger.Error("[OPT-IN] failed to save opt-in status", "slot", slot, "error", err) } else { - initLogger.Info("[PROGRESS] advanced to block", "block", lastBN) + logger.Info("[OPT-IN] validator opt-in status", "slot", slot, "opted_in", opted) } - cancel() - } + }(ei.Slot, ei.BlockNumber) } } + +func startIndexer(c *cli.Context) error { + + initLogger := slog.With("component", "init") + + dbURL := c.String(optionDatabaseURL.Name) + infuraRPC := c.String(optionInfuraRPC.Name) + beaconBase := c.String(optionBeaconBase.Name) + + initLogger.Info("starting blockchain indexer with StarRocks database") + initLogger.Info("configuration loaded", + "block_interval", c.Duration("block-interval"), + "validator_delay", c.Duration("validator-delay")) + ctx := c.Context + + db, err := initializeDatabase(ctx, dbURL, initLogger) + if err != nil { + return err + } + defer closeDatabase(db, initLogger) + + // Initialize HTTP client + httpc := httputil.NewHTTPClient(c.Duration("http-timeout")) + initLogger.Info("[HTTP] client initialized", "timeout", c.Duration("http-timeout")) + + // Load relay configurations + relays, err := loadRelays(ctx, db, initLogger) + if err != nil { + return err + } + + // Get starting block number + lastBN, err := getStartingBlockNumber(ctx, db, httpc, infuraRPC, initLogger) + if err != nil { + return err + } + + initLogger.Info("starting from block number", "block", lastBN) + initLogger.Info("indexer configuration", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) + + // Run backfill if configured + runBackfillIfConfigured(ctx, c, db, httpc, relays, initLogger) + return runMainLoop(ctx, c, db, httpc, relays, infuraRPC, beaconBase, lastBN, initLogger) +} diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index b4b18fad1..d9dc3db3e 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -3,19 +3,25 @@ package backfill import ( "context" "fmt" + "log/slog" + "time" + "github.com/hashicorp/go-retryablehttp" "github.com/primev/mev-commit/tools/indexer/pkg/beacon" "github.com/primev/mev-commit/tools/indexer/pkg/config" "github.com/primev/mev-commit/tools/indexer/pkg/database" "github.com/primev/mev-commit/tools/indexer/pkg/ethereum" "github.com/primev/mev-commit/tools/indexer/pkg/relay" - "log/slog" - "net/http" - "time" ) +type SlotData struct { + Slot int64 + BlockNumber int64 + ValidatorPubkey []byte +} + // RecentMissing backfills recent blocks that are missing data -func recentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, lookback int64, batch int) error { +func recentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, lookback int64, batch int, ch chan<- SlotData) error { logger := slog.With("component", "backfill") blocks, err := db.GetRecentMissingBlocks(ctx, lookback, batch) @@ -37,22 +43,27 @@ func recentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Cl continue } - + var vpub []byte // Schedule async validator pubkey fetch if ei.ProposerIdx != nil { vctx, vcancel := context.WithTimeout(ctx, 5*time.Second) - vpub, verr := beacon.FetchValidatorPubkey(vctx, httpc, cfg.BeaconBase, *ei.ProposerIdx) + v, verr := beacon.FetchValidatorPubkey(vctx, httpc, cfg.BeaconBase, *ei.ProposerIdx) vcancel() if verr != nil { return fmt.Errorf("validator pubkey fetch failed slot=%d: %w", ei.Slot, verr) - } - if len(vpub) > 0 { + } else if len(vpub) > 0 { + vpub = v if err := db.UpdateValidatorPubkey(ctx, ei.Slot, vpub); err != nil { return fmt.Errorf("validator pubkey update failed slot=%d: %w", ei.Slot, err) } } } + select { + case ch <- SlotData{Slot: ei.Slot, BlockNumber: ei.BlockNumber, ValidatorPubkey: vpub}: + case <-ctx.Done(): + return ctx.Err() + } } logger.Info("RecentMissing processed", "blocks", len(blocks)) @@ -60,17 +71,8 @@ func recentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Cl } // RecentBids backfills bid data for ALL recent slots (not just opted-in blocks) -func recentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, lookback int64, batch int) error { +func recentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, slots []int64) error { logger := slog.With("component", "backfill") - opCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - slots, err := db.GetRecentSlotsWithBlocks(opCtx, lookback, batch) - if err != nil { - logger.Error("RecentBids query failed", "error", err) - return err - } - - logger.Info("RecentBids fetched slots", "count", len(slots)) for _, slot := range slots { if ctx.Err() != nil { @@ -113,49 +115,42 @@ func recentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Clien return nil } -// ValidatorOptIn backfills validator opt-in status (this is opt-in specific data) -func validatorOptIn(ctx context.Context, db *database.DB, httpc *http.Client, cfg *config.Config, lookback int64, batch int) error { - logger := slog.With("component", "backfill") - validators, err := db.GetValidatorsNeedingOptInCheck(ctx, lookback, batch) - if err != nil { - logger.Error("ValidatorOptIn query failed", "error", err) - return err - } - - for _, v := range validators { - opted, err := ethereum.CallAreOptedInAtBlock(httpc, cfg, v.BlockNumber, v.ValidatorPubkey) - if err == nil { - if err := db.UpdateValidatorOptInStatus(ctx, v.Slot, opted); err != nil { - logger.Error("ValidatorOptIn update failed", "slot", v.Slot, "error", err) - } - - } else { - logger.Error("ValidatorOptIn check failed", "slot", v.Slot, "error", err) - } - } - - logger.Info("ValidatorOptIn processed", "validators", len(validators)) - return nil -} - // RunAll executes all backfill operations ensuring complete coverage func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, relays []relay.Row) error { logger := slog.With("component", "backfill") logger.Info("Starting comprehensive backfill for ALL blocks (not just opted-in)") - if err := recentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { - logger.Error("RecentMissing failed", "error", err) - return err + // Channel to pass slot data from stage 1 to stages 2 & 3 + slotChan := make(chan SlotData, cfg.BackfillBatch) + + // Run recentMissing and collect slot data + go func() { + recentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch, slotChan) + close(slotChan) + }() + + // Collect slots and validator data from channel + var slotsForBids []int64 + var validatorsToCheck []SlotData + + for data := range slotChan { + slotsForBids = append(slotsForBids, data.Slot) + if len(data.ValidatorPubkey) > 0 { + validatorsToCheck = append(validatorsToCheck, data) + } } - if err := validatorOptIn(ctx, db, httpc.HTTPClient, cfg, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { - logger.Error("ValidatorOptIn failed", "error", err) - return err + for _, v := range validatorsToCheck { + opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, cfg, v.BlockNumber, v.ValidatorPubkey) + if err == nil { + db.UpdateValidatorOptInStatus(ctx, v.Slot, opted) + } } - if err := recentBids(ctx, db, httpc, relays, cfg.BackfillLookback, cfg.BackfillBatch); err != nil { + if err := recentBids(ctx, db, httpc, relays, slotsForBids); err != nil { logger.Error("RecentBids failed", "error", err) return err } logger.Info("Backfill-done") return nil + } From d0c31a597ce4a35cd530a5245f7a67a0ed7a0045 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 00:38:46 +0530 Subject: [PATCH 10/18] fixed refactoring --- tools/indexer/cmd/start.go | 36 ++++++++++++-------------- tools/indexer/pkg/backfill/backfill.go | 15 +++++++++-- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 8f1f0212c..908ebb51f 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -34,12 +34,6 @@ func initializeDatabase(ctx context.Context, dbURL string, logger *slog.Logger) return db, nil } -func closeDatabase(db *database.DB, logger *slog.Logger) { - if cerr := db.Close(); cerr != nil { - logger.Error("[DB] close failed", "error", cerr) - } -} - func loadRelays(ctx context.Context, db *database.DB, logger *slog.Logger) ([]relay.Row, error) { relays, err := relay.UpsertRelaysAndLoad(ctx, db) if err != nil { @@ -108,7 +102,12 @@ func runMainLoop(ctx context.Context, c *cli.Context, db *database.DB, httpc *re for { select { case <-ctx.Done(): - return handleShutdown(ctx, db, lastBN, logger) + logger.Info("[SHUTDOWN] graceful shutdown initiated", "reason", ctx.Err()) + if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { + logger.Error("[SHUTDOWN] failed to save last block number", "error", err) + } + logger.Info("[SHUTDOWN] indexer stopped", "block", lastBN) + return nil case <-mainTicker.C: lastBN = processNextBlock(ctx, c, db, httpc, relays, infuraRPC, beaconBase, lastBN, logger) @@ -116,15 +115,6 @@ func runMainLoop(ctx context.Context, c *cli.Context, db *database.DB, httpc *re } } -func handleShutdown(ctx context.Context, db *database.DB, lastBN int64, logger *slog.Logger) error { - logger.Info("[SHUTDOWN] graceful shutdown initiated", "reason", ctx.Err()) - if err := db.SaveLastBlockNumber(ctx, lastBN); err != nil { - logger.Error("[SHUTDOWN] failed to save last block number", "error", err) - } - logger.Info("[SHUTDOWN] indexer stopped", "block", lastBN) - return nil -} - func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, infuraRPC, beaconBase string, lastBN int64, logger *slog.Logger) int64 { nextBN := lastBN + 1 @@ -248,14 +238,16 @@ func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database go func(slot int64, proposerIdx int64) { time.Sleep(c.Duration("validator-delay")) - vpub, err := beacon.FetchValidatorPubkey(ctx, httpc, beaconBase, proposerIdx) + vctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + vpub, err := beacon.FetchValidatorPubkey(vctx, httpc, beaconBase, proposerIdx) if err != nil { logger.Error("[VALIDATOR] failed to fetch pubkey", "proposer", proposerIdx, "error", err) return } if len(vpub) > 0 { - if err := db.UpdateValidatorPubkey(ctx, slot, vpub); err != nil { + if err := db.UpdateValidatorPubkey(vctx, slot, vpub); err != nil { logger.Error("[VALIDATOR] failed to save pubkey", "slot", slot, "error", err) } else { logger.Info("[VALIDATOR] pubkey saved", "proposer", proposerIdx, "slot", slot) @@ -310,7 +302,11 @@ func startIndexer(c *cli.Context) error { if err != nil { return err } - defer closeDatabase(db, initLogger) + defer func() { + if cerr := db.Close(); cerr != nil { + initLogger.Error("[DB] close failed", "error", cerr) + } + }() // Initialize HTTP client httpc := httputil.NewHTTPClient(c.Duration("http-timeout")) @@ -332,6 +328,6 @@ func startIndexer(c *cli.Context) error { initLogger.Info("indexer configuration", "lookback", c.Int("backfill-lookback"), "batch", c.Int("backfill-batch")) // Run backfill if configured - runBackfillIfConfigured(ctx, c, db, httpc, relays, initLogger) + go runBackfillIfConfigured(ctx, c, db, httpc, relays, initLogger) return runMainLoop(ctx, c, db, httpc, relays, infuraRPC, beaconBase, lastBN, initLogger) } diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index d9dc3db3e..f7e0ba0c6 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -52,7 +52,7 @@ func recentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Cl if verr != nil { return fmt.Errorf("validator pubkey fetch failed slot=%d: %w", ei.Slot, verr) - } else if len(vpub) > 0 { + } else if len(v) > 0 { vpub = v if err := db.UpdateValidatorPubkey(ctx, ei.Slot, vpub); err != nil { return fmt.Errorf("validator pubkey update failed slot=%d: %w", ei.Slot, err) @@ -122,10 +122,13 @@ func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, c // Channel to pass slot data from stage 1 to stages 2 & 3 slotChan := make(chan SlotData, cfg.BackfillBatch) + errCh := make(chan error, 1) // Run recentMissing and collect slot data go func() { - recentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch, slotChan) + if err := recentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch, slotChan); err != nil { + errCh <- err + } close(slotChan) }() @@ -139,6 +142,14 @@ func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, c validatorsToCheck = append(validatorsToCheck, data) } } + select { + case err := <-errCh: + if err != nil { + logger.Error("RecentMissing failed", "error", err) + return err + } + default: + } for _, v := range validatorsToCheck { opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, cfg, v.BlockNumber, v.ValidatorPubkey) if err == nil { From d95790882f0ee5e661797ef9c72709b5d8d4b789 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 00:55:13 +0530 Subject: [PATCH 11/18] fixes --- tools/indexer/cmd/start.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 908ebb51f..058436485 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -262,7 +262,9 @@ func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) // Wait for validator pubkey to be available - vpk, err := db.GetValidatorPubkeyWithRetry(ctx, slot, 3, time.Second) + getCtx, getCancel := context.WithTimeout(context.Background(), 5*time.Second) + vpk, err := db.GetValidatorPubkeyWithRetry(getCtx, slot, 3, time.Second) + if err != nil { logger.Error("[VALIDATOR] pubkey not available", "slot", slot, "error", err) return @@ -274,7 +276,8 @@ func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database return } - err = db.UpdateValidatorOptInStatus(ctx, slot, opted) + err = db.UpdateValidatorOptInStatus(getCtx, slot, opted) + getCancel() if err != nil { logger.Error("[OPT-IN] failed to save opt-in status", "slot", slot, "error", err) } else { From 936ac83f7de5ea1577654df97f76cd2f56cfd58f Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 00:59:34 +0530 Subject: [PATCH 12/18] fixed backfill error logging --- tools/indexer/pkg/backfill/backfill.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index f7e0ba0c6..e0915b50d 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -152,8 +152,12 @@ func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, c } for _, v := range validatorsToCheck { opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, cfg, v.BlockNumber, v.ValidatorPubkey) - if err == nil { - db.UpdateValidatorOptInStatus(ctx, v.Slot, opted) + if err != nil { + logger.Error("opt-in check failed", "slot", v.Slot, "error", err) + continue + } + if uerr := db.UpdateValidatorOptInStatus(ctx, v.Slot, opted); uerr != nil { + logger.Error("opt-in status update failed", "slot", v.Slot, "error", uerr) } } if err := recentBids(ctx, db, httpc, relays, slotsForBids); err != nil { From 369f882a0671bcc9a21961fde6cc25285c803fd8 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 01:11:55 +0530 Subject: [PATCH 13/18] lint error fixes --- tools/indexer/cmd/start.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 058436485..cb83fdebb 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -155,10 +155,8 @@ func processBidsForBlock(ctx context.Context, db *database.DB, httpc *retryableh mainContextCanceled := false const batchSize = 500 for _, rr := range relays { - // Check if main context is canceled before processing each relay if ctx.Err() != nil { logger.Warn("main context canceled, stopping relay processing") - mainContextCanceled = true break } From aa9c997af7a67c8d13f982fba76e64f9fd34bbd7 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 01:21:05 +0530 Subject: [PATCH 14/18] lint error fixes --- tools/indexer/cmd/start.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index cb83fdebb..996a270c1 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -262,6 +262,7 @@ func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database // Wait for validator pubkey to be available getCtx, getCancel := context.WithTimeout(context.Background(), 5*time.Second) vpk, err := db.GetValidatorPubkeyWithRetry(getCtx, slot, 3, time.Second) + getCancel() if err != nil { logger.Error("[VALIDATOR] pubkey not available", "slot", slot, "error", err) @@ -274,8 +275,9 @@ func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database return } - err = db.UpdateValidatorOptInStatus(getCtx, slot, opted) - getCancel() + updCtx, updCancel := context.WithTimeout(context.Background(), 3*time.Second) + err = db.UpdateValidatorOptInStatus(updCtx, slot, opted) + updCancel() if err != nil { logger.Error("[OPT-IN] failed to save opt-in status", "slot", slot, "error", err) } else { From 650b11111bee0cdd899cf83101bee92ec574d84a Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 14:29:05 +0530 Subject: [PATCH 15/18] fixed the start.go changes --- tools/indexer/cmd/start.go | 39 +++++++++++++++--------------- tools/indexer/pkg/beacon/client.go | 4 ++- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 996a270c1..5609ae8af 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "log/slog" "time" @@ -128,10 +129,10 @@ func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, http "block", nextBN, "slot", ei.Slot, "timestamp", ei.Timestamp, - "proposer_index", ei.ProposerIdx, - "winning_relay", ei.RelayTag, - "builder_pubkey_prefix", ei.BuilderHex, - "producer_reward_eth", ei.RewardEth, + "proposer_index", *ei.ProposerIdx, + "winning_relay", *ei.RelayTag, + "builder_pubkey_prefix", *ei.BuilderHex, + "producer_reward_eth", *ei.RewardEth, ) if err := db.UpsertBlockFromExec(ctx, ei); err != nil { @@ -140,41 +141,43 @@ func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, http } logger.Info("[DB] block saved successfully", "block", nextBN) - processBidsForBlock(ctx, db, httpc, relays, ei, logger) + if err := processBidsForBlock(ctx, db, httpc, relays, ei, logger); err != nil { + logger.Error("failed to process bids", "error", err) + return lastBN + } launchAsyncValidatorTasks(ctx, c, db, httpc, ei, beaconBase, logger) saveBlockProgress(db, nextBN, logger) return nextBN } -func processBidsForBlock(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, ei *beacon.ExecInfo, logger *slog.Logger) { +func processBidsForBlock(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, ei *beacon.ExecInfo, logger *slog.Logger) error { // Fetch and store bid data from all relays totalBids := 0 successfulRelays := 0 - mainContextCanceled := false const batchSize = 500 for _, rr := range relays { - if ctx.Err() != nil { + if err := ctx.Err(); err != nil { logger.Warn("main context canceled, stopping relay processing") - break + return err } bids, err := relay.FetchBuilderBlocksReceived(ctx, httpc, rr.URL, ei.Slot) if err != nil { - logger.Error("[RELAY] failed to fetch bids", "relay_id", rr.ID, "url", rr.URL, "error", err) - continue + // logger.Error("[RELAY] failed to fetch bids", "relay_id", rr.ID, "url", rr.URL, "error", err) + return fmt.Errorf("fetch bids: relay_id=%d url=%s slot=%d: %w", rr.ID, rr.URL, ei.Slot, err) + } relayBids := 0 batch := make([]database.BidRow, 0, batchSize) for _, bid := range bids { - // Check if main context is still valid - if ctx.Err() != nil { + + if err := ctx.Err(); err != nil { logger.Warn("[BIDS] main context canceled, stopping bid insertion") - mainContextCanceled = true - break + return err } if row, ok := relay.BuildBidInsert(ei.Slot, rr.ID, bid); ok { @@ -205,10 +208,6 @@ func processBidsForBlock(ctx context.Context, db *database.DB, httpc *retryableh flushCancel() } - if mainContextCanceled { - break - } - if relayBids > 0 { logger.Info("[BIDS] bids collected", "relay_id", rr.ID, "count", relayBids) totalBids += relayBids @@ -216,7 +215,7 @@ func processBidsForBlock(ctx context.Context, db *database.DB, httpc *retryableh } } logger.Info("[BIDS] summary", "block", ei.BlockNumber, "total_bids", totalBids, "successful_relays", successfulRelays) - + return nil } func saveBlockProgress(db *database.DB, blockNum int64, logger *slog.Logger) { diff --git a/tools/indexer/pkg/beacon/client.go b/tools/indexer/pkg/beacon/client.go index 84aeabfcc..8252bb9cb 100644 --- a/tools/indexer/pkg/beacon/client.go +++ b/tools/indexer/pkg/beacon/client.go @@ -204,7 +204,7 @@ func FetchCombinedBlockData(ctx context.Context, httpc *retryablehttp.Client, rp // Convert block number to slot for beacon chain query slotNumber := ethereum.BlockNumberToSlot(blockNumber) - beaconData, _ := FetchBeaconExecutionBlock(ctx, httpc, beaconBase, slotNumber) + beaconData, _ := FetchBeaconExecutionBlock(ctx, httpc, beaconBase, blockNumber) // Merge data - use Alchemy as primary, beacon as supplement if beaconData != nil { @@ -212,6 +212,8 @@ func FetchCombinedBlockData(ctx context.Context, httpc *retryablehttp.Client, rp execBlock.ProposerIdx = beaconData.ProposerIdx execBlock.RelayTag = beaconData.RelayTag execBlock.RewardEth = beaconData.RewardEth + execBlock.BuilderHex = beaconData.BuilderHex + execBlock.FeeRecHex = beaconData.FeeRecHex } else { execBlock.Slot = slotNumber From afbe5df910204b902d8a030d36c725908af9ba2e Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 14:55:00 +0530 Subject: [PATCH 16/18] fixed the start.go changes --- tools/indexer/cmd/main.go | 2 +- tools/indexer/cmd/start.go | 105 +++++++++++++++-------------- tools/indexer/pkg/beacon/client.go | 4 +- 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/tools/indexer/cmd/main.go b/tools/indexer/cmd/main.go index f8df61cf5..5cc43e8fc 100644 --- a/tools/indexer/cmd/main.go +++ b/tools/indexer/cmd/main.go @@ -69,7 +69,7 @@ var ( Name: "backfill-lookback", Usage: "number of slots to look back for backfill", EnvVars: []string{"INDEXER_BACKFILL_LOOKBACK"}, - Value: 512, + Value: 10000000, }) optionBackfillBatch = altsrc.NewIntFlag(&cli.IntFlag{ diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 5609ae8af..7d144315b 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "reflect" "time" _ "github.com/go-sql-driver/mysql" @@ -115,7 +116,13 @@ func runMainLoop(ctx context.Context, c *cli.Context, db *database.DB, httpc *re } } } - +func safe(p interface{}) interface{} { + v := reflect.ValueOf(p) + if !v.IsValid() || v.IsNil() { + return nil + } + return v.Elem().Interface() +} func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, infuraRPC, beaconBase string, lastBN int64, logger *slog.Logger) int64 { nextBN := lastBN + 1 @@ -129,10 +136,10 @@ func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, http "block", nextBN, "slot", ei.Slot, "timestamp", ei.Timestamp, - "proposer_index", *ei.ProposerIdx, - "winning_relay", *ei.RelayTag, - "builder_pubkey_prefix", *ei.BuilderHex, - "producer_reward_eth", *ei.RewardEth, + "proposer_index", safe(ei.ProposerIdx), + "winning_relay", safe(ei.RelayTag), + "builder_pubkey_prefix", safe(ei.BuilderHex), + "producer_reward_eth", safe(ei.RewardEth), ) if err := db.UpsertBlockFromExec(ctx, ei); err != nil { @@ -145,7 +152,10 @@ func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, http logger.Error("failed to process bids", "error", err) return lastBN } - launchAsyncValidatorTasks(ctx, c, db, httpc, ei, beaconBase, logger) + if err := launchAsyncValidatorTasks(ctx, c, db, httpc, ei, beaconBase, logger); err != nil { + logger.Error("[VALIDATOR] failed to launch async tasks", "slot", ei.Slot, "error", err) + return lastBN + } saveBlockProgress(db, nextBN, logger) return nextBN @@ -230,60 +240,51 @@ func saveBlockProgress(db *database.DB, blockNum int64, logger *slog.Logger) { } -func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, ei *beacon.ExecInfo, beaconBase string, logger *slog.Logger) { // Async validator pubkey fetch - if ei.ProposerIdx != nil { - go func(slot int64, proposerIdx int64) { - time.Sleep(c.Duration("validator-delay")) - - vctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - vpub, err := beacon.FetchValidatorPubkey(vctx, httpc, beaconBase, proposerIdx) - if err != nil { - logger.Error("[VALIDATOR] failed to fetch pubkey", "proposer", proposerIdx, "error", err) - return - } +func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, ei *beacon.ExecInfo, beaconBase string, logger *slog.Logger) error { // Async validator pubkey fetch + if ei.ProposerIdx == nil { + return nil + } - if len(vpub) > 0 { - if err := db.UpdateValidatorPubkey(vctx, slot, vpub); err != nil { - logger.Error("[VALIDATOR] failed to save pubkey", "slot", slot, "error", err) - } else { - logger.Info("[VALIDATOR] pubkey saved", "proposer", proposerIdx, "slot", slot) - } - } - }(ei.Slot, *ei.ProposerIdx) + vctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + vpub, err := beacon.FetchValidatorPubkey(vctx, httpc, beaconBase, *ei.ProposerIdx) + if err != nil { + return fmt.Errorf("fetch validator pubkey: %w", err) } - // Async opt-in status check - if ei.ProposerIdx != nil { - go func(slot int64, blockNumber int64) { - time.Sleep(c.Duration("validator-delay") + 500*time.Millisecond) + if len(vpub) > 0 { + if err := db.UpdateValidatorPubkey(vctx, ei.Slot, vpub); err != nil { + logger.Error("[VALIDATOR] failed to save pubkey", "slot", ei.Slot, "error", err) + } else { + logger.Info("[VALIDATOR] pubkey saved", "proposer", *ei.ProposerIdx, "slot", ei.Slot) + } + } - // Wait for validator pubkey to be available - getCtx, getCancel := context.WithTimeout(context.Background(), 5*time.Second) - vpk, err := db.GetValidatorPubkeyWithRetry(getCtx, slot, 3, time.Second) - getCancel() + // Wait for validator pubkey to be available + getCtx, getCancel := context.WithTimeout(context.Background(), 5*time.Second) + vpk, err := db.GetValidatorPubkeyWithRetry(getCtx, ei.Slot, 3, time.Second) + getCancel() - if err != nil { - logger.Error("[VALIDATOR] pubkey not available", "slot", slot, "error", err) - return - } + if err != nil { + logger.Error("[VALIDATOR] pubkey not available", "slot", ei.Slot, "error", err) + return fmt.Errorf("save validator pubkey: %w", err) + } - opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, createOptionsFromCLI(c), blockNumber, vpk) - if err != nil { - logger.Error("[OPT-IN] failed to check opt-in status", "slot", slot, "error", err) - return - } + opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, createOptionsFromCLI(c), ei.BlockNumber, vpk) + if err != nil { + return fmt.Errorf("check opt-in status: %w", err) + } - updCtx, updCancel := context.WithTimeout(context.Background(), 3*time.Second) - err = db.UpdateValidatorOptInStatus(updCtx, slot, opted) - updCancel() - if err != nil { - logger.Error("[OPT-IN] failed to save opt-in status", "slot", slot, "error", err) - } else { - logger.Info("[OPT-IN] validator opt-in status", "slot", slot, "opted_in", opted) - } - }(ei.Slot, ei.BlockNumber) + updCtx, updCancel := context.WithTimeout(context.Background(), 3*time.Second) + err = db.UpdateValidatorOptInStatus(updCtx, ei.Slot, opted) + updCancel() + if err != nil { + return fmt.Errorf("save opt-in status: %w", err) + } else { + logger.Info("[OPT-IN] validator opt-in status", "slot", ei.Slot, "opted_in", opted) } + return nil + } func startIndexer(c *cli.Context) error { diff --git a/tools/indexer/pkg/beacon/client.go b/tools/indexer/pkg/beacon/client.go index 8252bb9cb..2b7011143 100644 --- a/tools/indexer/pkg/beacon/client.go +++ b/tools/indexer/pkg/beacon/client.go @@ -212,8 +212,8 @@ func FetchCombinedBlockData(ctx context.Context, httpc *retryablehttp.Client, rp execBlock.ProposerIdx = beaconData.ProposerIdx execBlock.RelayTag = beaconData.RelayTag execBlock.RewardEth = beaconData.RewardEth - execBlock.BuilderHex = beaconData.BuilderHex - execBlock.FeeRecHex = beaconData.FeeRecHex + execBlock.BuilderHex = beaconData.BuilderHex + execBlock.FeeRecHex = beaconData.FeeRecHex } else { execBlock.Slot = slotNumber From 634be45d32d5c5af570cb28e18a46d45cd779790 Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 17:43:35 +0530 Subject: [PATCH 17/18] fixed the start.go changes --- tools/indexer/pkg/backfill/backfill.go | 161 +++++++++---------------- tools/indexer/pkg/database/starrock.go | 1 - 2 files changed, 56 insertions(+), 106 deletions(-) diff --git a/tools/indexer/pkg/backfill/backfill.go b/tools/indexer/pkg/backfill/backfill.go index e0915b50d..a9d31c23c 100644 --- a/tools/indexer/pkg/backfill/backfill.go +++ b/tools/indexer/pkg/backfill/backfill.go @@ -18,154 +18,105 @@ type SlotData struct { Slot int64 BlockNumber int64 ValidatorPubkey []byte + ProposerIdx *int64 } -// RecentMissing backfills recent blocks that are missing data -func recentMissing(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, lookback int64, batch int, ch chan<- SlotData) error { +func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, relays []relay.Row) error { logger := slog.With("component", "backfill") + logger.Info("Starting streaming backfill") - blocks, err := db.GetRecentMissingBlocks(ctx, lookback, batch) - if err != nil { - logger.Error("RecentMissing query failed", "error", err) + if err := ctx.Err(); err != nil { return err } - for _, block := range blocks { + blocks, err := db.GetRecentMissingBlocks(ctx, cfg.BackfillLookback, cfg.BackfillBatch) + if err != nil { + return fmt.Errorf("get missing blocks: %w", err) + } + + for _, b := range blocks { + if err := ctx.Err(); err != nil { + return err + } + fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - // Fetch beacon execution block data - ei, ferr := beacon.FetchBeaconExecutionBlock(fetchCtx, httpc, cfg.BeaconBase, block.BlockNumber) + ei, ferr := beacon.FetchBeaconExecutionBlock(fetchCtx, httpc, cfg.BeaconBase, b.BlockNumber) cancel() if ferr != nil || ei == nil { - return fmt.Errorf("beacon fetch failed for block=%d: %w", block.BlockNumber, ferr) + logger.Error("beacon fetch failed", "block", b.BlockNumber, "error", ferr) + continue } - if err := db.UpsertBlockFromExec(ctx, ei); err != nil { - logger.Error("RecentMissing upsert failed", "slot", block.Slot, "error", err) + if err := db.UpsertBlockFromExec(ctx, ei); err != nil { + logger.Error("block upsert failed", "slot", ei.Slot, "error", err) continue } + var vpub []byte - // Schedule async validator pubkey fetch if ei.ProposerIdx != nil { vctx, vcancel := context.WithTimeout(ctx, 5*time.Second) v, verr := beacon.FetchValidatorPubkey(vctx, httpc, cfg.BeaconBase, *ei.ProposerIdx) vcancel() - if verr != nil { - return fmt.Errorf("validator pubkey fetch failed slot=%d: %w", ei.Slot, verr) + logger.Error("validator fetch failed", "slot", ei.Slot, "error", verr) } else if len(v) > 0 { vpub = v + + // Save validator pubkey if err := db.UpdateValidatorPubkey(ctx, ei.Slot, vpub); err != nil { - return fmt.Errorf("validator pubkey update failed slot=%d: %w", ei.Slot, err) + logger.Error("validator update failed", "slot", ei.Slot, "error", err) + } else { + + opted, oerr := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, cfg, ei.BlockNumber, vpub) + + if oerr != nil { + logger.Error("opt-in check failed", "slot", ei.Slot, "error", oerr) + } else { + updCtx, updCancel := context.WithTimeout(ctx, 3*time.Second) + if uerr := db.UpdateValidatorOptInStatus(updCtx, ei.Slot, opted); uerr != nil { + logger.Error("opt-in update failed", "slot", ei.Slot, "error", uerr) + } + updCancel() + } } } } - select { - case ch <- SlotData{Slot: ei.Slot, BlockNumber: ei.BlockNumber, ValidatorPubkey: vpub}: - case <-ctx.Done(): - return ctx.Err() - } - } - - logger.Info("RecentMissing processed", "blocks", len(blocks)) - return nil -} - -// RecentBids backfills bid data for ALL recent slots (not just opted-in blocks) -func recentBids(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, relays []relay.Row, slots []int64) error { - logger := slog.With("component", "backfill") - for _, slot := range slots { - if ctx.Err() != nil { - break - } + for _, r := range relays { + if err := ctx.Err(); err != nil { + return err + } - for _, rr := range relays { - if ctx.Err() != nil { - break + bctx, bcancel := context.WithTimeout(ctx, 5*time.Second) + bids, berr := relay.FetchBuilderBlocksReceived(bctx, httpc, r.URL, ei.Slot) + bcancel() + if berr != nil { + logger.Debug("bid fetch failed", "slot", ei.Slot, "relay", r.ID, "error", berr) + continue } - fetchCtx, fcancel := context.WithTimeout(ctx, 5*time.Second) - bids, err := relay.FetchBuilderBlocksReceived(fetchCtx, httpc, rr.URL, slot) - fcancel() - if err != nil { - logger.Error("RecentBids fetch failed", "slot", slot, "relay_id", rr.ID, "relay_url", rr.URL, "error", err) + if len(bids) == 0 { continue } rows := make([]database.BidRow, 0, len(bids)) - for _, b := range bids { - if row, ok := relay.BuildBidInsert(slot, rr.ID, b); ok { + for _, bid := range bids { + if row, ok := relay.BuildBidInsert(ei.Slot, r.ID, bid); ok { rows = append(rows, row) } } if len(rows) > 0 { - insCtx, icancel := context.WithTimeout(ctx, 5*time.Second) - if err := db.InsertBidsBatch(insCtx, rows); err != nil { - icancel() - return fmt.Errorf("bids insert failed slot=%d relay_id=%d: %w", slot, rr.ID, err) + insCtx, insCancel := context.WithTimeout(ctx, 5*time.Second) + if ierr := db.InsertBidsBatch(insCtx, rows); ierr != nil { + logger.Error("bid insert failed", "slot", ei.Slot, "relay", r.ID, "error", ierr) } - icancel() + insCancel() } } - - } - - logger.Info("RecentBids processed", "slots", len(slots)) - return nil -} - -// RunAll executes all backfill operations ensuring complete coverage -func RunAll(ctx context.Context, db *database.DB, httpc *retryablehttp.Client, cfg *config.Config, relays []relay.Row) error { - logger := slog.With("component", "backfill") - logger.Info("Starting comprehensive backfill for ALL blocks (not just opted-in)") - - // Channel to pass slot data from stage 1 to stages 2 & 3 - slotChan := make(chan SlotData, cfg.BackfillBatch) - errCh := make(chan error, 1) - - // Run recentMissing and collect slot data - go func() { - if err := recentMissing(ctx, db, httpc, cfg, cfg.BackfillLookback, cfg.BackfillBatch, slotChan); err != nil { - errCh <- err - } - close(slotChan) - }() - - // Collect slots and validator data from channel - var slotsForBids []int64 - var validatorsToCheck []SlotData - - for data := range slotChan { - slotsForBids = append(slotsForBids, data.Slot) - if len(data.ValidatorPubkey) > 0 { - validatorsToCheck = append(validatorsToCheck, data) - } - } - select { - case err := <-errCh: - if err != nil { - logger.Error("RecentMissing failed", "error", err) - return err - } - default: - } - for _, v := range validatorsToCheck { - opted, err := ethereum.CallAreOptedInAtBlock(httpc.HTTPClient, cfg, v.BlockNumber, v.ValidatorPubkey) - if err != nil { - logger.Error("opt-in check failed", "slot", v.Slot, "error", err) - continue - } - if uerr := db.UpdateValidatorOptInStatus(ctx, v.Slot, opted); uerr != nil { - logger.Error("opt-in status update failed", "slot", v.Slot, "error", uerr) - } - } - if err := recentBids(ctx, db, httpc, relays, slotsForBids); err != nil { - logger.Error("RecentBids failed", "error", err) - return err + logger.Debug("slot processed", "slot", ei.Slot) } - logger.Info("Backfill-done") + logger.Info("Backfill completed", "blocks_processed", len(blocks)) return nil - } diff --git a/tools/indexer/pkg/database/starrock.go b/tools/indexer/pkg/database/starrock.go index d30a38438..da4929d89 100644 --- a/tools/indexer/pkg/database/starrock.go +++ b/tools/indexer/pkg/database/starrock.go @@ -305,7 +305,6 @@ func (db *DB) GetRecentMissingBlocks(ctx context.Context, lookback int64, batch return nil, fmt.Errorf("invalid parameters: lookback=%d, batch=%d", lookback, batch) } - // Build query with literal values query := fmt.Sprintf(` WITH recent AS ( SELECT COALESCE(MAX(slot), 0) AS s FROM blocks From 83c1a4d83f3f7e2849964fea1e441b6a152bd0ec Mon Sep 17 00:00:00 2001 From: Rose Jethani Date: Tue, 7 Oct 2025 19:49:58 +0530 Subject: [PATCH 18/18] fixed the naming of the fucntions --- tools/indexer/cmd/start.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/indexer/cmd/start.go b/tools/indexer/cmd/start.go index 7d144315b..1f3a2b5d4 100644 --- a/tools/indexer/cmd/start.go +++ b/tools/indexer/cmd/start.go @@ -152,7 +152,7 @@ func processNextBlock(ctx context.Context, c *cli.Context, db *database.DB, http logger.Error("failed to process bids", "error", err) return lastBN } - if err := launchAsyncValidatorTasks(ctx, c, db, httpc, ei, beaconBase, logger); err != nil { + if err := launchValidatorTasks(ctx, c, db, httpc, ei, beaconBase, logger); err != nil { logger.Error("[VALIDATOR] failed to launch async tasks", "slot", ei.Slot, "error", err) return lastBN } @@ -240,7 +240,7 @@ func saveBlockProgress(db *database.DB, blockNum int64, logger *slog.Logger) { } -func launchAsyncValidatorTasks(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, ei *beacon.ExecInfo, beaconBase string, logger *slog.Logger) error { // Async validator pubkey fetch +func launchValidatorTasks(ctx context.Context, c *cli.Context, db *database.DB, httpc *retryablehttp.Client, ei *beacon.ExecInfo, beaconBase string, logger *slog.Logger) error { // Async validator pubkey fetch if ei.ProposerIdx == nil { return nil }