test: add stateful end to end test

Makefile

@@ -255,6 +255,11 @@ start-localnet-skip-build:
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-setup-only.yml up -d
 
 stop-localnet:
+start-e2e-import-mainnet-test: zetanode
+	@echo "--> Starting e2e import-data test"
+	cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-import-data.yml up -d
+
+stop-test:
 	cd contrib/localnet/ && $(DOCKER) compose down --remove-orphans
 
 ###############################################################################

changelog.md

@@ -63,6 +63,7 @@
 * [2329](https://github.com/zeta-chain/node/pull/2329) - fix TODOs in rpc unit tests
 * [2342](https://github.com/zeta-chain/node/pull/2342) - extend rpc unit tests with testing extension to include synthetic ethereum txs
 * [2299](https://github.com/zeta-chain/node/pull/2299) - add `zetae2e` command to deploy test contracts
+* [2360](https://github.com/zeta-chain/node/pull/2360) - add stateful e2e tests.
 * [2349](https://github.com/zeta-chain/node/pull/2349) - add TestBitcoinDepositRefund and WithdrawBitcoinMultipleTimes E2E tests
 
 ### Fixes

cmd/zetacored/parse_genesis.go

@@ -41,7 +41,6 @@ const MaxItemsForList = 10
 // Copy represents a set of modules for which, the entire state is copied without any modifications
 var Copy = map[string]bool{
 	slashingtypes.ModuleName:  true,
-	govtypes.ModuleName:       true,
 	crisistypes.ModuleName:    true,
 	feemarkettypes.ModuleName: true,
 	paramstypes.ModuleName:    true,
@@ -50,24 +49,37 @@ var Copy = map[string]bool{
 	vestingtypes.ModuleName:   true,
 	fungibletypes.ModuleName:  true,
 	emissionstypes.ModuleName: true,
-	authz.ModuleName:          true,
 }
 
 // Skip represents a set of modules for which, the entire state is skipped and nothing gets imported
 var Skip = map[string]bool{
-	evmtypes.ModuleName:          true,
-	stakingtypes.ModuleName:      true,
-	genutiltypes.ModuleName:      true,
-	authtypes.ModuleName:         true,
-	banktypes.ModuleName:         true,
+	// Skipping evm this is done to reduce the size of the genesis file evm module uses the majority of the space due to smart contract data
+	evmtypes.ModuleName: true,
+	// Skipping staking as new validators would be created for the new chain
+	stakingtypes.ModuleName: true,
+	// Skipping genutil as new gentxs would be created
+	genutiltypes.ModuleName: true,
+	// Skipping auth as new accounts would be created for the new chain. This also needs to be done as we are skipping evm module
+	authtypes.ModuleName: true,
+	// Skipping bank module as it is not used when starting a new chain this is done to make sure the total supply invariant is maintained.
+	// This would need modification but might be possible to add in non evm based modules in the future
+	banktypes.ModuleName: true,
+	// Skipping distribution module as it is not used when starting a new chain , rewards are based on validators and delegators , and so rewards from a different chain do not hold any value
 	distributiontypes.ModuleName: true,
-	group.ModuleName:             true,
+	// Skipping group module as it is not used when starting a new chain, new groups should be created based on the validator operator keys
+	group.ModuleName: true,
+	// Skipping authz as it is not used when starting a new chain, new grants should be created based on the validator hotkeys abd operator keys
+	authz.ModuleName: true,
+	// Skipping fungible module as new fungible tokens would be created and system contract would be deployed
+	fungibletypes.ModuleName: true,
+	// Skipping gov types as new parameters are set for the new chain
+	govtypes.ModuleName: true,
 }
 
 // Modify represents a set of modules for which, the state is modified before importing. Each Module should have a corresponding Modify function
 var Modify = map[string]bool{
-	crosschaintypes.ModuleName: true,
 	observertypes.ModuleName:   true,
+	crosschaintypes.ModuleName: true,
 }
 
 func CmdParseGenesisFile() *cobra.Command {
@@ -78,6 +90,10 @@ func CmdParseGenesisFile() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			clientCtx := client.GetClientContextFromCmd(cmd)
 			cdc := clientCtx.Codec
+			modifyEnabled, err := cmd.Flags().GetBool("modify")
+			if err != nil {
+				return err
+			}
 			genesisFilePath := filepath.Join(app.DefaultNodeHome, "config", "genesis.json")
 			if len(args) == 2 {
 				genesisFilePath = args[1]
@@ -90,7 +106,7 @@ func CmdParseGenesisFile() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			err = ImportDataIntoFile(genDoc, importData, cdc)
+			err = ImportDataIntoFile(genDoc, importData, cdc, modifyEnabled)
 			if err != nil {
 				return err
 			}
@@ -103,10 +119,16 @@ func CmdParseGenesisFile() *cobra.Command {
 			return nil
 		},
 	}
+	cmd.PersistentFlags().Bool("modify", false, "Modify the genesis file before importing")
 	return cmd
 }
 
-func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc, cdc codec.Codec) error {
+func ImportDataIntoFile(
+	genDoc *types.GenesisDoc,
+	importFile *types.GenesisDoc,
+	cdc codec.Codec,
+	modifyEnabled bool,
+) error {
 
 	appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc)
 	if err != nil {
@@ -124,7 +146,7 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 		if Copy[m] {
 			appState[m] = importAppState[m]
 		}
-		if Modify[m] {
+		if Modify[m] && modifyEnabled {
 			switch m {
 			case crosschaintypes.ModuleName:
 				err := ModifyCrosschainState(appState, importAppState, cdc)

cmd/zetacored/parse_genesis_test.go

@@ -92,43 +92,86 @@ func Test_ModifyObserverState(t *testing.T) {
 }
 
 func Test_ImportDataIntoFile(t *testing.T) {
-	setConfig(t)
-	cdc := keepertest.NewCodec()
-	genDoc := sample.GenDoc(t)
-	importGenDoc := ImportGenDoc(t, cdc, 100)
+	t.Run("successfully import data into file and modify data", func(t *testing.T) {
+		setConfig(t)
+		cdc := keepertest.NewCodec()
+		genDoc := sample.GenDoc(t)
+		importGenDoc := ImportGenDoc(t, cdc, 100)
 
-	err := zetacored.ImportDataIntoFile(genDoc, importGenDoc, cdc)
-	require.NoError(t, err)
+		err := zetacored.ImportDataIntoFile(genDoc, importGenDoc, cdc, true)
+		require.NoError(t, err)
 
-	appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc)
-	require.NoError(t, err)
+		appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc)
+		require.NoError(t, err)
 
-	// Crosschain module is in Modify list
-	crosschainStateAfterImport := crosschaintypes.GetGenesisStateFromAppState(cdc, appState)
-	require.Len(t, crosschainStateAfterImport.CrossChainTxs, zetacored.MaxItemsForList)
-	require.Len(t, crosschainStateAfterImport.InboundHashToCctxList, zetacored.MaxItemsForList)
-	require.Len(t, crosschainStateAfterImport.FinalizedInbounds, zetacored.MaxItemsForList)
+		// Crosschain module is in Modify list
+		crosschainStateAfterImport := crosschaintypes.GetGenesisStateFromAppState(cdc, appState)
+		require.Len(t, crosschainStateAfterImport.CrossChainTxs, zetacored.MaxItemsForList)
+		require.Len(t, crosschainStateAfterImport.InboundHashToCctxList, zetacored.MaxItemsForList)
+		require.Len(t, crosschainStateAfterImport.FinalizedInbounds, zetacored.MaxItemsForList)
 
-	// Bank module is in Skip list
-	var bankStateAfterImport banktypes.GenesisState
-	if appState[banktypes.ModuleName] != nil {
-		err := cdc.UnmarshalJSON(appState[banktypes.ModuleName], &bankStateAfterImport)
-		if err != nil {
-			panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error()))
+		// Bank module is in Skip list
+		var bankStateAfterImport banktypes.GenesisState
+		if appState[banktypes.ModuleName] != nil {
+			err := cdc.UnmarshalJSON(appState[banktypes.ModuleName], &bankStateAfterImport)
+			if err != nil {
+				panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error()))
+			}
 		}
-	}
-	// 4 balances were present in the original genesis state
-	require.Len(t, bankStateAfterImport.Balances, 4)
+		// 4 balances were present in the original genesis state
+		require.Len(t, bankStateAfterImport.Balances, 4)
 
-	// Emissions module is in Copy list
-	var emissionStateAfterImport emissionstypes.GenesisState
-	if appState[emissionstypes.ModuleName] != nil {
-		err := cdc.UnmarshalJSON(appState[emissionstypes.ModuleName], &emissionStateAfterImport)
-		if err != nil {
-			panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error()))
+		// Emissions module is in Copy list
+		var emissionStateAfterImport emissionstypes.GenesisState
+		if appState[emissionstypes.ModuleName] != nil {
+			err := cdc.UnmarshalJSON(appState[emissionstypes.ModuleName], &emissionStateAfterImport)
+			if err != nil {
+				panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error()))
+			}
 		}
-	}
-	require.Len(t, emissionStateAfterImport.WithdrawableEmissions, 100)
+		require.Len(t, emissionStateAfterImport.WithdrawableEmissions, 100)
+	})
+
+	t.Run("successfully import data into file without modifying data", func(t *testing.T) {
+		setConfig(t)
+		cdc := keepertest.NewCodec()
+		genDoc := sample.GenDoc(t)
+		importGenDoc := ImportGenDoc(t, cdc, 8)
+
+		err := zetacored.ImportDataIntoFile(genDoc, importGenDoc, cdc, false)
+		require.NoError(t, err)
+
+		appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc)
+		require.NoError(t, err)
+
+		// Crosschain module is in Modify list
+		crosschainStateAfterImport := crosschaintypes.GetGenesisStateFromAppState(cdc, appState)
+		require.Len(t, crosschainStateAfterImport.CrossChainTxs, 0)
+		require.Len(t, crosschainStateAfterImport.InboundHashToCctxList, 0)
+		require.Len(t, crosschainStateAfterImport.FinalizedInbounds, 0)
+
+		// Bank module is in Skip list
+		var bankStateAfterImport banktypes.GenesisState
+		if appState[banktypes.ModuleName] != nil {
+			err := cdc.UnmarshalJSON(appState[banktypes.ModuleName], &bankStateAfterImport)
+			if err != nil {
+				panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error()))
+			}
+		}
+		// 4 balances were present in the original genesis state
+		require.Len(t, bankStateAfterImport.Balances, 4)
+
+		// Emissions module is in Copy list
+		var emissionStateAfterImport emissionstypes.GenesisState
+		if appState[emissionstypes.ModuleName] != nil {
+			err := cdc.UnmarshalJSON(appState[emissionstypes.ModuleName], &emissionStateAfterImport)
+			if err != nil {
+				panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error()))
+			}
+		}
+		require.Len(t, emissionStateAfterImport.WithdrawableEmissions, 8)
+
+	})
 }
 
 func Test_GetGenDoc(t *testing.T) {

contrib/localnet/docker-compose-import-data.yml

@@ -0,0 +1,30 @@
+version: "3"
+
+# This docker-compose file configures the localnet environment
+# it contains the following services:
+# - ZetaChain with 2 nodes (zetacore0, zetacore1)
+# - A observer set with 2 clients (zetaclient0, zetaclient1)
+# - An Ethereum node (eth)
+# - A Bitcoin node (bitcoin)
+# - A Rosetta API (rosetta)
+# - An orchestrator to manage interaction with the localnet (orchestrator)
+services:
+  rosetta:
+    entrypoint: ["/root/start-rosetta.sh"]
+
+  zetacore0:
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+    volumes:
+      - ~/genesis_export/:/root/genesis_data
+
+  zetacore1:
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+
+  zetaclient0:
+    entrypoint: ["/root/start-zetaclientd.sh"]
+
+  zetaclient1:
+    entrypoint: ["/root/start-zetaclientd.sh"]
+
+  orchestrator:
+    entrypoint: ["/work/start-zetae2e.sh", "local"]

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -29,6 +29,12 @@ while [ ! -f ~/.ssh/authorized_keys ]; do
     sleep 1
 done
 
+# need to wait for zetacore0 to be up
+while ! curl -s -o /dev/null zetacore0:26657/status ; do
+    echo "Waiting for zetacore0 rpc"
+    sleep 10
+done
+
 echo "waiting for geth RPC to start..."
 sleep 2
 

contrib/localnet/scripts/import-data.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+if [ $# -lt 1 ]
+then
+  echo "Usage: import-data.sh [network]"
+  exit 1
+fi
+
+NETWORK=$1
+echo "NETWORK: ${NETWORK}"
+rm -rf ~/genesis_export/
+mkdir ~/genesis_export/
+echo "Download Latest State Export"
+LATEST_EXPORT_URL=$(curl https://snapshots.zetachain.com/latest-state-export | jq -r ."${NETWORK}")
+echo "LATEST EXPORT URL: ${LATEST_EXPORT_URL}"
+wget -q ${LATEST_EXPORT_URL} -O ~/genesis_export/exported-genesis.json

contrib/localnet/scripts/start-rosetta.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This script is used to start the Rosetta API server for the Zetacore network.
+
+echo "Waiting for network to start producing blocks"
+CURRENT_HEIGHT=0
+WAIT_HEIGHT=1
+while [[ $CURRENT_HEIGHT -lt $WAIT_HEIGHT ]]
+do
+    CURRENT_HEIGHT=$(curl -s zetacore0:26657/status | jq '.result.sync_info.latest_block_height' | tr -d '"')
+    sleep 5
+done
+
+zetacored rosetta --tendermint zetacore0:26657 --grpc zetacore0:9090 --network athens_101-1 --blockchain zetacore

contrib/localnet/scripts/start-zetaclientd.sh

@@ -28,6 +28,15 @@ while [ ! -f ~/.ssh/authorized_keys ]; do
     sleep 1
 done
 
+
+
+# need to wait for zetacore0 to be up
+while ! curl -s -o /dev/null zetacore0:26657/status ; do
+    echo "Waiting for zetacore0 rpc"
+    sleep 10
+done
+
+
 # read HOTKEY_BACKEND env var for hotkey keyring backend and set default to test
 BACKEND="test"
 if [ "$HOTKEY_BACKEND" == "file" ]; then

contrib/localnet/scripts/start-zetacored.sh

@@ -4,6 +4,8 @@
 # It initializes the nodes and creates the genesis.json file
 # It also starts the nodes
 # The number of nodes is passed as an first argument to the script
+# The second argument is optional and can have the following value:
+#  - import-data: import data into the genesis file
 
 /usr/sbin/sshd
 
@@ -71,6 +73,7 @@ then
   exit 1
 fi
 NUMOFNODES=$1
+OPTION=$2
 
 # create keys
 CHAINID="athens_101-1"
@@ -254,6 +257,11 @@ then
       scp $NODE:~/.zetacored/config/gentx/* ~/.zetacored/config/gentx/z2gentx/
   done
 
+  if [[ "$OPTION" == "import-data" || "$OPTION" == "import-data-upgrade" ]]; then
+    echo "Importing data"
+    zetacored parse-genesis-file /root/genesis_data/exported-genesis.json
+  fi
+
 # 4. Collect all the gentx files in zetacore0 and create the final genesis.json
   zetacored collect-gentxs
   zetacored validate-genesis

docs/cli/zetacored/zetacored_parse-genesis-file.md

@@ -9,7 +9,8 @@ zetacored parse-genesis-file [import-genesis-file] [optional-genesis-file] [flag
 ### Options
 
 ```
-  -h, --help   help for parse-genesis-file
+  -h, --help     help for parse-genesis-file
+      --modify   Modify the genesis file before importing
 ```
 
 ### Options inherited from parent commands