feat: beacon-storage-part1

portalnetwork/beacon/sql.go

@@ -0,0 +1,45 @@
+package beacon
+
+const CreateQueryDBBeacon = `CREATE TABLE IF NOT EXISTS beacon (
+	content_id blob PRIMARY KEY,
+    content_key blob NOT NULL,
+    content_value blob NOT NULL,
+    content_size INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS beacon_content_size_idx ON beacon(content_size);`
+
+const InsertQueryBeacon = `INSERT OR IGNORE INTO beacon (content_id, content_key, content_value, content_size)
+                            VALUES (?1, ?2, ?3, ?4)`
+
+const DeleteQueryBeacon = `DELETE FROM beacon
+    WHERE content_id = (?1)`
+
+const ContentKeyLookupQueryBeacon = `SELECT content_key FROM beacon WHERE content_id = (?1) LIMIT 1`
+
+const ContentValueLookupQueryBeacon = `SELECT content_value FROM beacon WHERE content_id = (?1) LIMIT 1`
+
+const TotalDataSizeQueryBeacon = "SELECT TOTAL(content_size) FROM beacon"
+
+const TotalEntryCountQueryBeacon = "SELECT COUNT(*) FROM beacon"
+
+const ContentSizeLookupQueryBeacon = "SELECT content_size FROM beacon WHERE content_id = (?1)"
+
+const LCUpdateCreateTable = `CREATE TABLE IF NOT EXISTS lc_update (
+	period INTEGER PRIMARY KEY,
+	value BLOB NOT NULL,
+	score INTEGER NOT NULL,
+	update_size INTEGER
+);
+CREATE INDEX IF NOT EXISTS update_size_idx ON lc_update(update_size);
+DROP INDEX IF EXISTS period_idx;`
+
+const InsertLCUpdateQuery = `INSERT OR IGNORE INTO lc_update (period, value, score, update_size)
+				  VALUES (?1, ?2, ?3, ?4)`
+
+const LCUpdateLookupQuery = `SELECT value FROM lc_update WHERE period = (?1) LIMIT 1`
+
+const LCUpdateLookupQueryByRange = `SELECT value FROM lc_update WHERE period >= (?1) AND period < (?2)`
+
+const LCUpdatePeriodLookupQuery = `SELECT period FROM lc_update WHERE period = (?1) LIMIT 1`
+
+const LCUpdateTotalSizeQuery = `SELECT TOTAL(update_size) FROM lc_update`

portalnetwork/beacon/storage.go

@@ -0,0 +1,152 @@
+package beacon
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/protolambda/zrnt/eth2/beacon/common"
+	"github.com/protolambda/ztyp/codec"
+
+	"github.com/ethereum/go-ethereum/portalnetwork/storage"
+)
+
+const BytesInMB uint64 = 1000 * 1000
+
+type BeaconStorage struct {
+	storageCapacityInBytes uint64
+	db                     *sql.DB
+	log                    log.Logger
+	spec                   *common.Spec
+}
+
+var _ storage.ContentStorage = &BeaconStorage{}
+
+func NewBeaconStorage(config storage.PortalStorageConfig) (storage.ContentStorage, error) {
+	bs := &BeaconStorage{
+		storageCapacityInBytes: config.StorageCapacityMB * BytesInMB,
+		db:                     config.DB,
+		log:                    log.New("beacon_storage"),
+		spec:                   config.Spec,
+	}
+	if err := bs.setup(); err != nil {
+		return nil, err
+	}
+	return bs, nil
+}
+
+func (bs *BeaconStorage) setup() error {
+	if _, err := bs.db.Exec(CreateQueryDBBeacon); err != nil {
+		return err
+	}
+	if _, err := bs.db.Exec(LCUpdateCreateTable); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (bs *BeaconStorage) Get(contentKey []byte, contentId []byte) ([]byte, error) {
+	switch storage.ContentType(contentKey[0]) {
+	case LightClientBootstrap:
+		return bs.getContentValue(contentId)
+	case LightClientUpdate:
+		lightClientUpdateKey := new(LightClientUpdateKey)
+		err := lightClientUpdateKey.UnmarshalSSZ(contentKey[1:])
+		if err != nil {
+			return nil, err
+		}
+		return bs.getLcUpdateValueByRange(lightClientUpdateKey.StartPeriod, lightClientUpdateKey.StartPeriod+lightClientUpdateKey.Count)
+	case LightClientFinalityUpdate:
+	case LightClientOptimisticUpdate:
+	}
+	return nil, nil
+}
+
+func (bs *BeaconStorage) Put(contentKey []byte, contentId []byte, content []byte) error {
+	switch storage.ContentType(contentKey[0]) {
+	case LightClientBootstrap:
+		return bs.putContentValue(contentId, contentKey, content)
+	case LightClientUpdate:
+		lightClientUpdateKey := new(LightClientUpdateKey)
+		err := lightClientUpdateKey.UnmarshalSSZ(contentKey[1:])
+		if err != nil {
+			return err
+		}
+		lightClientUpdateRange := new(LightClientUpdateRange)
+		reader := codec.NewDecodingReader(bytes.NewReader(content), uint64(len(content)))
+		err = lightClientUpdateRange.Deserialize(bs.spec, reader)
+		if err != nil {
+			return err
+		}
+		for index, update := range *lightClientUpdateRange {
+			var buf bytes.Buffer
+			writer := codec.NewEncodingWriter(&buf)
+			err := update.Serialize(bs.spec, writer)
+			if err != nil {
+				return err
+			}
+			period := lightClientUpdateKey.StartPeriod + uint64(index)
+			err = bs.putLcUpdate(period, buf.Bytes())
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	case LightClientFinalityUpdate:
+	case LightClientOptimisticUpdate:
+	}
+	return nil
+}
+
+func (bs *BeaconStorage) getContentValue(contentId []byte) ([]byte, error) {
+	res := make([]byte, 0)
+	err := bs.db.QueryRowContext(context.Background(), ContentValueLookupQueryBeacon, contentId).Scan(&res)
+	if err == sql.ErrNoRows {
+		return nil, storage.ErrContentNotFound
+	}
+	return res, err
+}
+
+func (bs *BeaconStorage) getLcUpdateValueByRange(start, end uint64) ([]byte, error) {
+	// LightClientUpdateRange := make([]ForkedLightClientUpdate, 0)
+	var lightClientUpdateRange LightClientUpdateRange
+	rows, err := bs.db.QueryContext(context.Background(), LCUpdateLookupQueryByRange, start, end)
+	if err != nil {
+		return nil, err
+	}
+	hasData := false
+	defer rows.Close()
+	for rows.Next() {
+		hasData = true
+		var val []byte
+		err := rows.Scan(&val)
+		if err != nil {
+			return nil, err
+		}
+		update := new(ForkedLightClientUpdate)
+		dec := codec.NewDecodingReader(bytes.NewReader(val), uint64(len(val)))
+		update.Deserialize(bs.spec, dec)
+		lightClientUpdateRange = append(lightClientUpdateRange, *update)
+	}
+	if !hasData {
+		return nil, storage.ErrContentNotFound
+	}
+	var buf bytes.Buffer
+	err = lightClientUpdateRange.Serialize(bs.spec, codec.NewEncodingWriter(&buf))
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func (bs *BeaconStorage) putContentValue(contentId, contentKey, value []byte) error {
+	length := 32 + len(contentKey) + len(value)
+	_, err := bs.db.ExecContext(context.Background(), InsertQueryBeacon, contentId, contentKey, value, length)
+	return err
+}
+
+func (bs *BeaconStorage) putLcUpdate(period uint64, value []byte) error {
+	_, err := bs.db.ExecContext(context.Background(), InsertLCUpdateQuery, period, value, 0, len(value))
+	return err
+}

portalnetwork/beacon/storage_test.go

@@ -0,0 +1,119 @@
+package beacon
+
+import (
+	"crypto/sha256"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/portalnetwork/storage"
+	"github.com/ethereum/go-ethereum/portalnetwork/utils"
+	"github.com/holiman/uint256"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/protolambda/zrnt/eth2/configs"
+	"github.com/stretchr/testify/require"
+)
+
+var zeroNodeId = uint256.NewInt(0).Bytes32()
+
+const dbName = "beacon.sqlite"
+
+func defaultContentIdFunc(contentKey []byte) []byte {
+	digest := sha256.Sum256(contentKey)
+	return digest[:]
+}
+
+func TestGetAndPut(t *testing.T) {
+	testDir := "./"
+	beaconStorage, err := genStorage(testDir)
+	require.NoError(t, err)
+	defer clearNodeData(testDir)
+	key, value, err := getClientBootstrap()
+	require.NoError(t, err)
+
+	contentId := defaultContentIdFunc(key)
+	_, err = beaconStorage.Get(key, contentId)
+	require.Equal(t, storage.ErrContentNotFound, err)
+
+	err = beaconStorage.Put(key, contentId, value)
+	require.NoError(t, err)
+
+	res, err := beaconStorage.Get(key, contentId)
+	require.NoError(t, err)
+	require.Equal(t, value, res)
+
+	key, value, err = getClientUpdatesByRange()
+	require.NoError(t, err)
+
+	contentId = defaultContentIdFunc(key)
+	_, err = beaconStorage.Get(key, contentId)
+	require.Equal(t, storage.ErrContentNotFound, err)
+
+	err = beaconStorage.Put(key, contentId, value)
+	require.NoError(t, err)
+
+	res, err = beaconStorage.Get(key, contentId)
+	require.NoError(t, err)
+	require.Equal(t, value, res)
+}
+
+func genStorage(testDir string) (storage.ContentStorage, error) {
+	utils.EnsureDir(testDir)
+	db, err := sql.Open("sqlite3", path.Join(testDir, dbName))
+	if err != nil {
+		return nil, err
+	}
+	config := &storage.PortalStorageConfig{
+		StorageCapacityMB: 1000,
+		DB:                db,
+		NodeId:            enode.ID(zeroNodeId),
+		Spec:              configs.Mainnet,
+	}
+	return NewBeaconStorage(*config)
+}
+
+func getClientBootstrap() ([]byte, []byte, error) {
+	filePath := "testdata/light_client_bootstrap.json"
+
+	f, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, nil, err
+	}
+	var result map[string]map[string]string
+	err = json.Unmarshal(f, &result)
+	if err != nil {
+		return nil, nil, err
+	}
+	contentKey := hexutil.MustDecode(result["6718368"]["content_key"])
+	contentValue := hexutil.MustDecode(result["6718368"]["content_value"])
+	return contentKey, contentValue, nil
+}
+
+func getClientUpdatesByRange() ([]byte, []byte, error) {
+	filePath := "testdata/light_client_updates_by_range.json"
+
+	f, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, nil, err
+	}
+	var result map[string]map[string]string
+	err = json.Unmarshal(f, &result)
+	if err != nil {
+		return nil, nil, err
+	}
+	contentKey := hexutil.MustDecode(result["6684738"]["content_key"])
+	contentValue := hexutil.MustDecode(result["6684738"]["content_value"])
+	return contentKey, contentValue, nil
+}
+
+func clearNodeData(nodeDataDir string) {
+	err := os.Remove(fmt.Sprintf("%s%s", nodeDataDir, dbName))
+	if err != nil {
+		fmt.Println(err)
+	}
+}

portalnetwork/beacon/testdata/light_client_finality_update.json

@@ -1,6 +1,6 @@
 {
   "6718463": {
-    "content_key": "0x020000000000000000",
+    "content_key": "0x12a083660000000000",
     "content_value": "0xbba4da9670010000b30400001d34030000000000000000000000000000000000000000000000000000000000a4f8b2415bbca66d73597343afead504bd8282523df27153543b2de8973b7474ace652bb73991c3b63f7185a17333c8aea619a69b69863a5d79810dad6c363d3df460765de20d3c8f56bcd9490f15e2d9e122b3f6173cea76f6d0d16074ce0f0ecde5df1b81f36c08e91ce803c25add312bfd8cb759a81b7cfd158d724959cf22e83908e043f77bdc5992806617ef1fa0a4f1985d9aa1b67371d3580be8c2c64ffffffffffffffffffffffffffffbfffff7fffffffffffffffffffffffffffffffffffffffffffffffffffdefffffffffffffffffffeffffffffffffffffefffa480f81d481c5aa8439f8c65f86c69d3b1570cfd95b61300f7e29c5b0954c0c0fb080bba0e8ba60dead40c45355332ed06982fcaf53ad1ad4a7e6e39091ee7e5602d4a30b6d46e12047c31451f246bf43c4a3fbdb1d1de2f2180a776ef71fc3d0084660000000000ff83660000000000e5f4020000000000007b8211614b9c5331a1de2691ad7cdde1ca113b4e51b9f2757a4b502833f9234a06eb807d82b9175dd085748ade76aaa86fb4eca48bc8deb01b9c56a267a896d98469820ef7d34f8610e134c6eaf8d379e83cf6c3ab0663fba828edab344dc5f40000008e2a532cfdfdce86b27b26f611f1df0e2b00a54d18c6980e69ad28a4b70bf480336488033fe5f3ef4ccc12af07b9370b92e553e35ecb4a337a1b1c0e4afe1e0edb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71b4c4e01065cd22a0d2f27a544ff8315b9e28f6270a63f543831778f32648ac3a15ecbea44f91b9b4a28ec8308a30a4b6252f554a2088bb1c73ea01013ba4963dfeebabe6b0418ec13b30aadf129f5dcdd4f70ceaa56499df0b3ba9d8fe6ca410132a29cf387371294137003adb139ce9f06b6f7dbf32ee01da91016a485a8aab5669168efc3d3a200b2ee0f25d937a413dfc38e44e211f02432a01a83088e700808c302002139aad8496010300cd220b14288429d8a6049945000a3b60f00f00109019341a0129a08906314848650817002cb98894a328084521a87e6d0a64e8080004ac0805000800e29808294016508a2004001304002a02320c4705501ac008a62848701040012a08e5c8b69010f1009f80408894857053e1b940e2c9004912860e26112080610d002098242000ca001420209a42c141d0c86087509344ced8a32e0204410a0196025a12106104e20008ad4449d5043208090008810201820209108607a0fa1004aa323410050222867024e0d172a82821b02c00020513152006484c800b10f0420802c0244d0850288c6541dc34cabc32f5d355aef4017425cab95c1de0474d483623e23c9bf13d9a18f1597e930b010000000080c3c901000000000ca88800000000004b6094640000000038020000ab9bf66d04000000000000000000000000000000000000000000000000000000797be971185ac699fa7288c87457c3611b443fb98483fddf282dfa7c5b4b27058eddbcd8b5de9b747c57c827ceba820d19e987ff6740eed38ccbc947696d71e3e2c3b96c9200d7d5c66fb4e8458067d6420e64647682ca2f0798ae817d2bc19468747470733a2f2f6574682d6275696c6465722e636f6da08366000000000037b10700000000002a7315c8ddfc25dc2266a6b221cb8f9fdf641970ab1f65a2754df4e14c432b9c446c604131913bed45976c4a8ea27df843a72d622f80ef15da622f0d56ec1ffe3021190177b0405c4fa1a0311a493c4dd095f24d386be6a03f5838a6b1008665f4000000f5c98fc152bf40e1ce216c8839b8ddd42ea5b355f0b5c700fc9b2cb7c802e1c8336488033fe5f3ef4ccc12af07b9370b92e553e35ecb4a337a1b1c0e4afe1e0edb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71a533e05d648cb9635382f38b778dc3c76b18e4eab2d017fdb24c1c0e32d2991707a45ce3f26e443dafa697a76bf24122c9b19b57eebe798668dbd1a0da7c900795222290dd7278aa3ddd389cc1e1d165cc4bafe5c658153cdef5850f97c4d4fb6706b82310414a00fe7dc34b3043e0c7a8a24e810e3cec9de3e1dd7ce06bc191efd2273a357da4436fc4bf82069f6cb87d3d605aeda1336ac513f358b4296b8da51232896ad271063d93404eaa6b80960fa2d4b41d74a544eef94731f0b83ba2701e33b49e3191098c3369bc4658d82f536cb9d38285c51c7844292ced06dd2e8148f0bcd585dc1104e64aa930045e83d828ab436a0e587637e5ef83044d1c2918da284d7072847132885d443640979907485847368e32501345b0dc24c904c06732cc17d944b26181b0c05c6461bccbe4f7b2180a282b6871aa6472fa36cdc49ec50cf69d3eee4b26fe02bf48f39efb549d3963d19e4112c7250c580543516204ff1b0675680cfea02ac09f0b23d6e28c14b08421b3a10a9788442457d732b120c5c466c54ad938dedd3cd3701d9b32fa15dc8db3d6b1ea5d8495732477a72b34efcaf8c349aa40d8e1f21f226ca11182cff00921930b010000000080c3c901000000009104e90000000000d75b9464000000003802000076ab1c8604000000000000000000000000000000000000000000000000000000844f7a7e7585362114fc88a70654146215ab4eaad65ad54b64975840b78d365e8b75e78616a92294e747589c3ee4a845c2962e1cf6cd7fe8bfbd09439e5e6d5e4d4efdf181473803e734d97891388e1b50628658ac818599cf2cb069c8e6b38b6265617665726275696c642e6f7267"
   }
 }

portalnetwork/beacon/testdata/light_client_optimistic_update.json

@@ -1,6 +1,6 @@
 {
   "6718463": {
-    "content_key": "0x030000000000000000",
+    "content_key": "0x130084660000000000",
     "content_value": "0xbba4da96ac000000ffffffffffffffffffffffffffffbfffff7fffffffffffffffffffffffffffffffffffffffffffffffffffdefffffffffffffffffffeffffffffffffffffefffa480f81d481c5aa8439f8c65f86c69d3b1570cfd95b61300f7e29c5b0954c0c0fb080bba0e8ba60dead40c45355332ed06982fcaf53ad1ad4a7e6e39091ee7e5602d4a30b6d46e12047c31451f246bf43c4a3fbdb1d1de2f2180a776ef71fc3d0084660000000000ff83660000000000e5f4020000000000007b8211614b9c5331a1de2691ad7cdde1ca113b4e51b9f2757a4b502833f9234a06eb807d82b9175dd085748ade76aaa86fb4eca48bc8deb01b9c56a267a896d98469820ef7d34f8610e134c6eaf8d379e83cf6c3ab0663fba828edab344dc5f40000008e2a532cfdfdce86b27b26f611f1df0e2b00a54d18c6980e69ad28a4b70bf480336488033fe5f3ef4ccc12af07b9370b92e553e35ecb4a337a1b1c0e4afe1e0edb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71b4c4e01065cd22a0d2f27a544ff8315b9e28f6270a63f543831778f32648ac3a15ecbea44f91b9b4a28ec8308a30a4b6252f554a2088bb1c73ea01013ba4963dfeebabe6b0418ec13b30aadf129f5dcdd4f70ceaa56499df0b3ba9d8fe6ca410132a29cf387371294137003adb139ce9f06b6f7dbf32ee01da91016a485a8aab5669168efc3d3a200b2ee0f25d937a413dfc38e44e211f02432a01a83088e700808c302002139aad8496010300cd220b14288429d8a6049945000a3b60f00f00109019341a0129a08906314848650817002cb98894a328084521a87e6d0a64e8080004ac0805000800e29808294016508a2004001304002a02320c4705501ac008a62848701040012a08e5c8b69010f1009f80408894857053e1b940e2c9004912860e26112080610d002098242000ca001420209a42c141d0c86087509344ced8a32e0204410a0196025a12106104e20008ad4449d5043208090008810201820209108607a0fa1004aa323410050222867024e0d172a82821b02c00020513152006484c800b10f0420802c0244d0850288c6541dc34cabc32f5d355aef4017425cab95c1de0474d483623e23c9bf13d9a18f1597e930b010000000080c3c901000000000ca88800000000004b6094640000000038020000ab9bf66d04000000000000000000000000000000000000000000000000000000797be971185ac699fa7288c87457c3611b443fb98483fddf282dfa7c5b4b27058eddbcd8b5de9b747c57c827ceba820d19e987ff6740eed38ccbc947696d71e3e2c3b96c9200d7d5c66fb4e8458067d6420e64647682ca2f0798ae817d2bc19468747470733a2f2f6574682d6275696c6465722e636f6d"
   }
 }

portalnetwork/storage/config.go

@@ -0,0 +1,15 @@
+package storage
+
+import (
+	"database/sql"
+
+	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/protolambda/zrnt/eth2/beacon/common"
+)
+
+type PortalStorageConfig struct {
+	StorageCapacityMB uint64
+	DB                *sql.DB
+	NodeId            enode.ID
+	Spec              *common.Spec
+}

portalnetwork/utils/util.go

@@ -0,0 +1,24 @@
+package utils
+
+import (
+	"errors"
+	"os"
+)
+
+func EnsureDir(dir string) error {
+	stat, err := os.Stat(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(dir, 0755)
+			if err != nil {
+				return err
+			}
+		}
+		return err
+	}
+
+	if !stat.IsDir() {
+		return errors.New("node dir should be a dir")
+	}
+	return nil
+}