From ab3f7783cd52ab14df8da2186110d91b5a46741c Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Thu, 29 Sep 2022 22:34:39 -0700 Subject: [PATCH 01/14] Parallel TX Synchrnous Testing --- loadtest/scripts/populate_genesis_accounts.py | 12 ++-- scripts/old_initialize_local.sh | 60 ++++++++----------- 2 files changed, 33 insertions(+), 39 deletions(-) mode change 100644 => 100755 scripts/old_initialize_local.sh diff --git a/loadtest/scripts/populate_genesis_accounts.py b/loadtest/scripts/populate_genesis_accounts.py index 6f92f1276a..4aab62463d 100644 --- a/loadtest/scripts/populate_genesis_accounts.py +++ b/loadtest/scripts/populate_genesis_accounts.py @@ -14,11 +14,13 @@ def add_genesis_account(account_name, local=False): add_key_cmd = f"yes | ~/go/bin/seid keys add {account_name} --keyring-backend test" else: add_key_cmd = f"printf '12345678\n' | ~/go/bin/seid keys add {account_name}" - add_key_output = subprocess.check_output( - [add_key_cmd], - stderr=subprocess.STDOUT, - shell=True, - ).decode() + + with LOCK: + add_key_output = subprocess.check_output( + [add_key_cmd], + stderr=subprocess.STDOUT, + shell=True, + ).decode() splitted_outputs = add_key_output.split('\n') address = splitted_outputs[3].split(': ')[1] mnemonic = splitted_outputs[11] diff --git a/scripts/old_initialize_local.sh b/scripts/old_initialize_local.sh old mode 100644 new mode 100755 index ea0167b0cb..b274461496 --- a/scripts/old_initialize_local.sh +++ b/scripts/old_initialize_local.sh @@ -1,43 +1,35 @@ -#!/bin/bash -echo -n OS Password: -read -s password -echo -echo -n Key Name: -read keyname -echo -echo -n Number of Test Accounts: -read numtestaccount -echo -docker stop jaeger -docker rm jaeger -docker run -d --name jaeger \ - -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \ - -p 5775:5775/udp \ - -p 6831:6831/udp \ - -p 6832:6832/udp \ - -p 5778:5778 \ - -p 16686:16686 \ - -p 14250:14250 \ - -p 14268:14268 \ - -p 14269:14269 \ - -p 9411:9411 \ - jaegertracing/all-in-one:1.33 +#!/bin/bash + +keyname=admin +#docker stop jaeger +#docker rm jaeger +#docker run -d --name jaeger \ +# -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \ +# -p 5775:5775/udp \ +# -p 6831:6831/udp \ +# -p 6832:6832/udp \ +# -p 5778:5778 \ +# -p 16686:16686 \ +# -p 14250:14250 \ +# -p 14268:14268 \ +# -p 14269:14269 \ +# -p 9411:9411 \ +# jaegertracing/all-in-one:1.33 +rm -rf ~/.sei echo "Building..." make install -echo $password | sudo -S rm -r ~/.sei/ -echo $password | sudo -S rm -r ~/test_accounts/ +#echo $password | sudo -S rm -r ~/.sei/ +#echo $password | sudo -S rm -r ~/test_accounts/ ~/go/bin/seid init demo --chain-id sei-chain -yes | ~/go/bin/seid keys add $keyname -yes | ~/go/bin/seid keys add faucet -printf '12345678\n' | ~/go/bin/seid add-genesis-account $(~/go/bin/seid keys show $keyname -a) 100000000000000000000usei,100000000000000000000uusdc,100000000000000000000uatom -printf '12345678\n' | ~/go/bin/seid add-genesis-account $(~/go/bin/seid keys show faucet -a) 100000000000000000000usei,100000000000000000000uusdc,100000000000000000000uatom -python3 ./loadtest/scripts/populate_genesis_accounts.py $numtestaccount loc -printf '12345678\n' | ~/go/bin/seid gentx $keyname 70000000000000000000usei --chain-id sei-chain -sed -i 's/mode = "full"/mode = "validator"/g' $HOME/.sei/config/config.toml -sed -i 's/indexer = \["null"\]/indexer = \["kv"\]/g' $HOME/.sei/config/config.toml +~/go/bin/seid keys add $keyname --keyring-backend test +#yes | ~/go/bin/seid keys add faucet +~/go/bin/seid add-genesis-account $(~/go/bin/seid keys show $keyname -a --keyring-backend test) 100000000000000000000usei,100000000000000000000uusdc,100000000000000000000uatom +~/go/bin/seid gentx $keyname 70000000000000000000usei --chain-id sei-chain --keyring-backend test +sed -i '' 's/mode = "full"/mode = "validator"/g' $HOME/.sei/config/config.toml +sed -i '' 's/indexer = \["null"\]/indexer = \["kv"\]/g' $HOME/.sei/config/config.toml KEY=$(jq '.pub_key' ~/.sei/config/priv_validator_key.json -c) jq '.validators = [{}]' ~/.sei/config/genesis.json > ~/.sei/config/tmp_genesis.json jq '.validators[0] += {"power":"70000000000000"}' ~/.sei/config/tmp_genesis.json > ~/.sei/config/tmp_genesis_2.json From 0e0d47ed97f656f370e765ed122ab356a5d31434 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Thu, 29 Sep 2022 22:51:50 -0700 Subject: [PATCH 02/14] Revert parallell populate --- loadtest/scripts/populate_genesis_accounts.py | 54 +++++-------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/loadtest/scripts/populate_genesis_accounts.py b/loadtest/scripts/populate_genesis_accounts.py index 4aab62463d..9338f2db07 100644 --- a/loadtest/scripts/populate_genesis_accounts.py +++ b/loadtest/scripts/populate_genesis_accounts.py @@ -1,26 +1,18 @@ import json import os -import multiprocessing import subprocess import sys -import threading -import time - -PARALLEISM=32 -LOCK=threading.Lock() def add_genesis_account(account_name, local=False): if local: add_key_cmd = f"yes | ~/go/bin/seid keys add {account_name} --keyring-backend test" else: add_key_cmd = f"printf '12345678\n' | ~/go/bin/seid keys add {account_name}" - - with LOCK: - add_key_output = subprocess.check_output( - [add_key_cmd], - stderr=subprocess.STDOUT, - shell=True, - ).decode() + add_key_output = subprocess.check_output( + [add_key_cmd], + stderr=subprocess.STDOUT, + shell=True, + ).decode() splitted_outputs = add_key_output.split('\n') address = splitted_outputs[3].split(': ')[1] mnemonic = splitted_outputs[11] @@ -38,26 +30,13 @@ def add_genesis_account(account_name, local=False): "mnemonic": mnemonic, } json.dump(data, f) - success = False - retry_counter = 5 - sleep_time = 1 - while not success and retry_counter > 0: - try: - with LOCK: - subprocess.check_call( - [add_account_cmd], - shell=True, - ) - success = True - except subprocess.CalledProcessError as e: - print(f"Encountered error {e}, retrying {retry_counter - 1} times") - retry_counter -= 1 - sleep_time += 0.5 - time.sleep(sleep_time) - + subprocess.check_call( + [add_account_cmd], + shell=True, + ) -def bulk_create_genesis_accounts(number_of_accounts, start_idx, is_local=False): - for i in range(start_idx, start_idx + number_of_accounts): +def bulk_create_genesis_accounts(number_of_accounts, is_local=False): + for i in range(number_of_accounts): print(f"Creating account {i}") add_genesis_account(f"ta{i}", is_local) @@ -67,14 +46,7 @@ def main(): is_local = False if len(args) > 1 and args[1] == "loc": is_local = True - num_processes = number_of_accounts // PARALLEISM - processes = [] - for i in range(0, number_of_accounts, num_processes): - processes.append(multiprocessing.Process(target=bulk_create_genesis_accounts, args=(num_processes, i, is_local))) - for p in processes: - p.start() - for p in processes: - p.join() + bulk_create_genesis_accounts(number_of_accounts, is_local) if __name__ == "__main__": - main() + main() \ No newline at end of file From 3d31bd1b09b74dd3c8a4086770dd3b45fc1c744d Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Thu, 29 Sep 2022 22:58:18 -0700 Subject: [PATCH 03/14] try psu new change --- loadtest/scripts/populate_genesis_accounts.py | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/loadtest/scripts/populate_genesis_accounts.py b/loadtest/scripts/populate_genesis_accounts.py index 9338f2db07..12caa0aee5 100644 --- a/loadtest/scripts/populate_genesis_accounts.py +++ b/loadtest/scripts/populate_genesis_accounts.py @@ -2,8 +2,12 @@ import os import subprocess import sys +import threading +import time -def add_genesis_account(account_name, local=False): +PARALLEISM=32 + +def add_genesis_account(account_name, lock, local=False): if local: add_key_cmd = f"yes | ~/go/bin/seid keys add {account_name} --keyring-backend test" else: @@ -30,15 +34,28 @@ def add_genesis_account(account_name, local=False): "mnemonic": mnemonic, } json.dump(data, f) - subprocess.check_call( - [add_account_cmd], - shell=True, - ) + success = False + retry_counter = 5 + sleep_time = 1 + while not success and retry_counter > 0: + try: + with lock: + subprocess.check_call( + [add_account_cmd], + shell=True, + ) + success = True + except subprocess.CalledProcessError as e: + print(f"Encountered error {e}, retrying {retry_counter - 1} times") + retry_counter -= 1 + sleep_time += 0.5 + time.sleep(sleep_time) + -def bulk_create_genesis_accounts(number_of_accounts, is_local=False): - for i in range(number_of_accounts): +def bulk_create_genesis_accounts(number_of_accounts, start_idx, lock, is_local=False): + for i in range(start_idx, start_idx + number_of_accounts): print(f"Creating account {i}") - add_genesis_account(f"ta{i}", is_local) + add_genesis_account(f"ta{i}", lock, is_local) def main(): args = sys.argv[1:] @@ -46,7 +63,15 @@ def main(): is_local = False if len(args) > 1 and args[1] == "loc": is_local = True - bulk_create_genesis_accounts(number_of_accounts, is_local) + num_threads = number_of_accounts // PARALLEISM + threads = [] + lock=threading.Lock() + for i in range(0, number_of_accounts, num_threads): + threads.append(threading.Thread(target=bulk_create_genesis_accounts, args=(num_threads, i, lock, is_local))) + for t in threads: + t.start() + for t in threads: + t.join() if __name__ == "__main__": main() \ No newline at end of file From 2c70e1459b011c79fc8659ac2a00408315ad863f Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Fri, 30 Sep 2022 08:37:48 -0700 Subject: [PATCH 04/14] fees --- loadtest/contracts/deploy_ten_contracts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loadtest/contracts/deploy_ten_contracts.sh b/loadtest/contracts/deploy_ten_contracts.sh index 9250f78bc0..27ababf7cf 100644 --- a/loadtest/contracts/deploy_ten_contracts.sh +++ b/loadtest/contracts/deploy_ten_contracts.sh @@ -70,7 +70,7 @@ marsaddr4=$(python3 parser.py contract_address $marsinsres4) echo "Registering..." valaddr=$(printf "12345678\n" | $seidbin keys show $(printf "12345678\n" | $seidbin keys show node_admin --output json | jq -r .address) --bech=val --output json | jq -r '.address') -printf "12345678\n" | $seidbin tx staking delegate $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -b block -y +printf "12345678\n" | $seidbin tx staking delegate $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -—fees 2000use -b block -y printf "12345678\n" | $seidbin tx dex register-contract $marsaddr $marsid false true -y --from=$keyname --chain-id=$chainid --fees=10000000usei --gas=500000 --broadcast-mode=block printf "12345678\n" | $seidbin tx dex register-contract $saturnaddr $saturnid false true -y --from=$keyname --chain-id=$chainid --fees=10000000usei --gas=500000 --broadcast-mode=block @@ -145,7 +145,7 @@ printf "12345678\n" | $seidbin tx gov vote $marsproposalid4 yes -y --from=$keyna sleep 90 -printf "12345678\n" | $seidbin tx staking unbond $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -b block -y +printf "12345678\n" | $seidbin tx staking unbond $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -b -—fees 2000use block -y echo $marsaddr echo $saturnaddr From 22506499a67baa83fd894b3d412583f6a7f3b8eb Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Fri, 30 Sep 2022 08:50:21 -0700 Subject: [PATCH 05/14] 0.12.6 --- loadtest/contracts/deploy_ten_contracts.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/loadtest/contracts/deploy_ten_contracts.sh b/loadtest/contracts/deploy_ten_contracts.sh index 27ababf7cf..ea985be2cd 100644 --- a/loadtest/contracts/deploy_ten_contracts.sh +++ b/loadtest/contracts/deploy_ten_contracts.sh @@ -15,17 +15,17 @@ echo cd $seihome/loadtest/contracts/mars && cargo build && docker run --rm -v "$(pwd)":/code \ --mount type=volume,source="$(basename "$(pwd)")_cache",target=/code/target \ --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \ - cosmwasm/rust-optimizer:0.12.5 + cosmwasm/rust-optimizer:0.12.6 cd $seihome/loadtest/contracts/saturn && cargo build && docker run --rm -v "$(pwd)":/code \ --mount type=volume,source="$(basename "$(pwd)")_cache",target=/code/target \ --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \ - cosmwasm/rust-optimizer:0.12.5 + cosmwasm/rust-optimizer:0.12.6 cd $seihome/loadtest/contracts/venus && cargo build && docker run --rm -v "$(pwd)":/code \ --mount type=volume,source="$(basename "$(pwd)")_cache",target=/code/target \ --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \ - cosmwasm/rust-optimizer:0.12.5 + cosmwasm/rust-optimizer:0.12.6 # Deploy all contracts echo "Deploying contracts..." From ea353053279b25c944acb96ec73a8a058100cdab Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Fri, 30 Sep 2022 08:58:59 -0700 Subject: [PATCH 06/14] logging --- app/app.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/app.go b/app/app.go index d1c212a335..44602aad32 100644 --- a/app/app.go +++ b/app/app.go @@ -983,6 +983,8 @@ func (app *App) ProcessTxConcurrent( txBlockingSignalsMap MessageCompletionSignalMapping, ) { defer wg.Done() + ctx.Logger().Info(fmt.Sprintf("ProcessTxConcurrent:: Processing: %d", txIndex)) + // Store the Channels in the Context Object for each transaction ctx.WithTxBlockingChannels(getChannelsFromSignalMapping(txBlockingSignalsMap)) ctx.WithTxCompletionChannels(getChannelsFromSignalMapping(txCompletionSignalingMap)) @@ -1003,6 +1005,7 @@ func (app *App) ProcessBlockConcurrent( // If there's no transactions then return empty results if len(txs) == 0 { + ctx.Logger().Info("ProcessBlockConcurrent:: No Transactions!") return txResults } @@ -1021,6 +1024,7 @@ func (app *App) ProcessBlockConcurrent( } // Waits for all the transactions to complete + ctx.Logger().Info("ProcessBlockConcurrent:: Waiting for TXs!") waitGroup.Wait() // Gather Results and store it based on txIndex @@ -1030,6 +1034,7 @@ func (app *App) ProcessBlockConcurrent( txResultsMap[result.txIndex] = result.result } + ctx.Logger().Info("ProcessBlockConcurrent:: Gathered Results!") // Gather Results and store in array based on txIndex to preserve ordering for txIndex := range txs { txResults = append(txResults, txResultsMap[txIndex]) @@ -1089,6 +1094,7 @@ func (app *App) ProcessBlock(ctx sdk.Context, txs [][]byte, req BlockProcessRequ txResults = app.ProcessBlockSynchronous(ctx, txs) } } else { + ctx.Logger().Info("ProcessBlock:: Building Mapping") completionSignalingMap, blockingSignalsMap := dependencyDag.BuildCompletionSignalMaps() txResults = app.ProcessBlockConcurrent(ctx, txs, completionSignalingMap, blockingSignalsMap) } From d08bfcdb9cdca3821beacaf8c9282d6e3f6f51fe Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Fri, 30 Sep 2022 09:03:33 -0700 Subject: [PATCH 07/14] logs --- app/app.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/app.go b/app/app.go index 44602aad32..d286108e88 100644 --- a/app/app.go +++ b/app/app.go @@ -990,6 +990,7 @@ func (app *App) ProcessTxConcurrent( ctx.WithTxCompletionChannels(getChannelsFromSignalMapping(txCompletionSignalingMap)) // Deliver the transaction and store the result in the channel + ctx.Logger().Info(fmt.Sprintf("ProcessTxConcurrent:: Waiting for: %d", txIndex)) resultChan <- ChannelResult{txIndex, app.DeliverTxWithResult(ctx, txBytes)} } @@ -1009,6 +1010,8 @@ func (app *App) ProcessBlockConcurrent( return txResults } + ctx.Logger().Info(fmt.Sprintf("ProcessBlockConcurrent:: Got transactions: %d", len(txs))) + // For each transaction, start goroutine and deliver TX for txIndex, txBytes := range txs { waitGroup.Add(1) From fb1dacd154051d5f48aee96e0bfa28fec2e1c8e7 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Fri, 30 Sep 2022 09:12:17 -0700 Subject: [PATCH 08/14] new version --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d25f13cdc5..984623bf9c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/cosmos/cosmos-sdk v0.45.4 github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/ibc-go/v3 v3.0.0 - github.com/deckarep/golang-set v1.8.0 github.com/gogo/protobuf v1.3.3 github.com/golang/protobuf v1.5.2 github.com/gorilla/mux v1.8.0 @@ -131,7 +130,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.1.73 + github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.1.70 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.1.56 From 8da953b30e7b16847d8935ee95da0672168fad34 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Fri, 30 Sep 2022 09:13:49 -0700 Subject: [PATCH 09/14] tidy --- app/app.go | 26 ++++- app/graph.go | 127 +++++++++------------ go.mod | 4 +- go.sum | 10 +- loadtest/contracts/deploy_ten_contracts.sh | 4 +- utils/metrics/labels.go | 6 + utils/metrics/metrics_util.go | 18 +++ x/dex/keeper/epoch.go | 2 +- 8 files changed, 113 insertions(+), 84 deletions(-) create mode 100644 utils/metrics/labels.go create mode 100644 utils/metrics/metrics_util.go diff --git a/app/app.go b/app/app.go index d286108e88..0df6ae836f 100644 --- a/app/app.go +++ b/app/app.go @@ -13,10 +13,12 @@ import ( "time" storetypes "github.com/cosmos/cosmos-sdk/store/types" + "github.com/k0kubun/pp" graph "github.com/yourbasic/graph" appparams "github.com/sei-protocol/sei-chain/app/params" "github.com/sei-protocol/sei-chain/utils" + "github.com/sei-protocol/sei-chain/utils/metrics" "github.com/sei-protocol/sei-chain/wasmbinding" wasmkeeper "github.com/CosmWasm/wasmd/x/wasm/keeper" @@ -885,9 +887,13 @@ func (app *App) BuildDependencyDag(ctx sdk.Context, txs [][]byte) (*Dag, error) } msgs := tx.GetMsgs() for messageIndex, msg := range msgs { - msgDependencies := app.AccessControlKeeper.GetResourceDependencyMapping(ctx, acltypes.GenerateMessageKey(msg)) + msgKey := acltypes.GenerateMessageKey(msg) + msgDependencies := app.AccessControlKeeper.GetResourceDependencyMapping(ctx, msgKey) + ctx.Logger().Info(fmt.Sprintf("BuildDependencyDag:Getting Dependencies for %s", msgKey)) + pp.Println(msgDependencies) for _, accessOp := range msgDependencies.GetAccessOps() { // make a new node in the dependency dag + ctx.Logger().Info(fmt.Sprintf("BuildDependencyDag:Get AccessOps Dependencies for %s", msgKey)) dependencyDag.AddNodeBuildDependency(messageIndex, txIndex, accessOp) } } @@ -933,6 +939,7 @@ func (app *App) DeliverTxWithResult(ctx sdk.Context, tx []byte) *abci.ExecTxResu deliverTxResp := app.DeliverTx(ctx, abci.RequestDeliverTx{ Tx: tx, }) + ctx.Logger().Info(fmt.Sprintf("DeliverTxWithResult:: Delievered TX")) return &abci.ExecTxResult{ Code: deliverTxResp.Code, Data: deliverTxResp.Data, @@ -950,6 +957,7 @@ func (app *App) ProcessBlockSynchronous(ctx sdk.Context, txs [][]byte) []*abci.E for _, tx := range txs { txResults = append(txResults, app.DeliverTxWithResult(ctx, tx)) } + metrics.IncrTransactionProcessTypeCounter(metrics.SYNCHRONOUS) return txResults } @@ -991,7 +999,12 @@ func (app *App) ProcessTxConcurrent( // Deliver the transaction and store the result in the channel ctx.Logger().Info(fmt.Sprintf("ProcessTxConcurrent:: Waiting for: %d", txIndex)) - resultChan <- ChannelResult{txIndex, app.DeliverTxWithResult(ctx, txBytes)} + results := app.DeliverTxWithResult(ctx, txBytes) + ctx.Logger().Info(fmt.Sprintf("Got Results: %d", txIndex)) + channelResult := ChannelResult{txIndex, results} + resultChan <- channelResult + ctx.Logger().Info(fmt.Sprintf("ProcessTxConcurrent:: Sent to channel: %d", txIndex)) + metrics.IncrTransactionProcessTypeCounter(metrics.CONCURRENT) } func (app *App) ProcessBlockConcurrent( @@ -1011,6 +1024,11 @@ func (app *App) ProcessBlockConcurrent( } ctx.Logger().Info(fmt.Sprintf("ProcessBlockConcurrent:: Got transactions: %d", len(txs))) + go func() { + waitGroup.Wait() + ctx.Logger().Info("Closing Channels!") + close(resultChan) + }() // For each transaction, start goroutine and deliver TX for txIndex, txBytes := range txs { @@ -1027,8 +1045,8 @@ func (app *App) ProcessBlockConcurrent( } // Waits for all the transactions to complete - ctx.Logger().Info("ProcessBlockConcurrent:: Waiting for TXs!") - waitGroup.Wait() + // ctx.Logger().Info("ProcessBlockConcurrent:: Waiting for TXs!") + // waitGroup.Wait() // Gather Results and store it based on txIndex // Concurrent results may be in different order than the original txIndex diff --git a/app/graph.go b/app/graph.go index 8301de3094..59e20a383a 100644 --- a/app/graph.go +++ b/app/graph.go @@ -5,6 +5,7 @@ import ( acltypes "github.com/cosmos/cosmos-sdk/types/accesscontrol" mapset "github.com/deckarep/golang-set" + "github.com/k0kubun/pp" ) type DagNodeID int @@ -136,11 +137,11 @@ func (dag *Dag) AddNodeBuildDependency(messageIndex int, txIndex int, accessOp a dagNode := dag.AddNode(messageIndex, txIndex, accessOp) // update tx index map dag.TxIndexMap[txIndex] = dagNode.NodeID - nodeDependencies := dag.GetNodeDependencies(dagNode) // build edges for each of the dependencies for _, nodeDependency := range nodeDependencies { - dag.AddEdge(nodeDependency, dagNode.NodeID) + println("AddNodeBuildDependency:Adding Node Edge!") + dag.AddEdge(nodeDependency.NodeID, dagNode.NodeID) } // update access ops map with the latest node id using a specific access op @@ -158,81 +159,56 @@ func getAllNodeIDsFromIdentifierMapping(mapping ResourceIdentifierNodeIDMapping) return } -func (dag *Dag) getDependencyWrites(node DagNode, dependentResource acltypes.ResourceType) mapset.Set { - nodeIDs := mapset.NewSet() - writeResourceAccess := ResourceAccess{ - dependentResource, - acltypes.AccessType_WRITE, - } - if identifierNodeMapping, ok := dag.ResourceAccessMap[writeResourceAccess]; ok { - var nodeIDsMaybeDependency []DagNodeID - if dependentResource != node.AccessOperation.ResourceType { - // we can add all node IDs as dependencies if applicable - nodeIDsMaybeDependency = getAllNodeIDsFromIdentifierMapping(identifierNodeMapping) - } else { - // TODO: otherwise we need to have partial filtering on identifiers - // for now, lets just perform exact matching on identifiers - nodeIDsMaybeDependency = identifierNodeMapping[node.AccessOperation.IdentifierTemplate] +// This helper will identify nodes that are dependencies for the current node, and can then be used for creating edges between then for future completion signals +func (dag *Dag) GetNodeDependencies(node DagNode) (nodeDependencies []DagNode) { + accessOp := node.AccessOperation + // if the blocking access ops are in access ops map, make an edge + switch accessOp.AccessType { + case acltypes.AccessType_READ: + println("GetNodeDependencies: AccessType_READ") + // if we need to do a read, we need latest write as a dependency + // TODO: replace hardcoded access op dependencies with helper that generates (and also generates superseding resources too eg. Resource.ALL is blocking for Resource.KV) + writeAccessOp := acltypes.AccessOperation{ + AccessType: acltypes.AccessType_WRITE, + ResourceType: accessOp.GetResourceType(), + IdentifierTemplate: accessOp.GetIdentifierTemplate(), } - for _, wn := range nodeIDsMaybeDependency { - writeNode := dag.NodeMap[wn] - // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) - // if from a previous transaction, we need to create an edge - if writeNode.TxIndex < node.TxIndex { - // this should be the COMMIT access op for the tx - lastTxNode := dag.NodeMap[dag.TxIndexMap[writeNode.TxIndex]] - nodeIDs.Add(lastTxNode.NodeID) + if writeNodeIDs, ok := dag.AccessOpsMap[writeAccessOp]; ok { + for _, wn := range writeNodeIDs { + writeNode := dag.NodeMap[wn] + // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) + // if from a previous transaction, we need to create an edge + if writeNode.TxIndex < node.TxIndex { + // this should be the COMMIT access op for the tx + lastTxNode := dag.NodeMap[dag.TxIndexMap[writeNode.TxIndex]] + nodeDependencies = append(nodeDependencies, lastTxNode) + } } } - } - return nodeIDs -} - -func (dag *Dag) getDependencyUnknowns(node DagNode, dependentResource acltypes.ResourceType) mapset.Set { - nodeIDs := mapset.NewSet() - unknownResourceAccess := ResourceAccess{ - dependentResource, - acltypes.AccessType_UNKNOWN, - } - if identifierNodeMapping, ok := dag.ResourceAccessMap[unknownResourceAccess]; ok { - var nodeIDsMaybeDependency []DagNodeID - if dependentResource != node.AccessOperation.ResourceType { - // we can add all node IDs as dependencies if applicable - nodeIDsMaybeDependency = getAllNodeIDsFromIdentifierMapping(identifierNodeMapping) - } else { - // TODO: otherwise we need to have partial filtering on identifiers - // for now, lets just perform exact matching on identifiers - nodeIDsMaybeDependency = identifierNodeMapping[node.AccessOperation.IdentifierTemplate] + case acltypes.AccessType_WRITE, acltypes.AccessType_UNKNOWN: + println("GetNodeDependencies: AccessType_WRITE/AccessType_UNKNOWN") + // if we need to do a write, we need read and write as dependencies + writeAccessOp := acltypes.AccessOperation{ + AccessType: acltypes.AccessType_WRITE, + ResourceType: accessOp.GetResourceType(), + IdentifierTemplate: accessOp.GetIdentifierTemplate(), } - for _, un := range nodeIDsMaybeDependency { - uNode := dag.NodeMap[un] - // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) - // if from a previous transaction, we need to create an edge - if uNode.TxIndex < node.TxIndex { - // this should be the COMMIT access op for the tx - lastTxNode := dag.NodeMap[dag.TxIndexMap[uNode.TxIndex]] - nodeIDs.Add(lastTxNode.NodeID) + if writeNodeIDs, ok := dag.AccessOpsMap[writeAccessOp]; ok { + for _, wn := range writeNodeIDs { + // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) + writeNode := dag.NodeMap[wn] + // if from a previous transaction, we need to create an edge + if writeNode.TxIndex < node.TxIndex { + // we need to get the last node from that tx + lastTxNode := dag.NodeMap[dag.TxIndexMap[writeNode.TxIndex]] + nodeDependencies = append(nodeDependencies, lastTxNode) + } } } - } - return nodeIDs -} - -func (dag *Dag) getDependencyReads(node DagNode, dependentResource acltypes.ResourceType) mapset.Set { - nodeIDs := mapset.NewSet() - readResourceAccess := ResourceAccess{ - dependentResource, - acltypes.AccessType_READ, - } - if identifierNodeMapping, ok := dag.ResourceAccessMap[readResourceAccess]; ok { - var nodeIDsMaybeDependency []DagNodeID - if dependentResource != node.AccessOperation.ResourceType { - // we can add all node IDs as dependencies if applicable - nodeIDsMaybeDependency = getAllNodeIDsFromIdentifierMapping(identifierNodeMapping) - } else { - // TODO: otherwise we need to have partial filtering on identifiers - // for now, lets just perform exact matching on identifiers - nodeIDsMaybeDependency = identifierNodeMapping[node.AccessOperation.IdentifierTemplate] + readAccessOp := acltypes.AccessOperation{ + AccessType: acltypes.AccessType_READ, + ResourceType: accessOp.GetResourceType(), + IdentifierTemplate: accessOp.GetIdentifierTemplate(), } for _, rn := range nodeIDsMaybeDependency { readNode := dag.NodeMap[rn] @@ -282,7 +258,8 @@ func (dag *Dag) GetNodeDependencies(node DagNode) []DagNodeID { func (dag *Dag) BuildCompletionSignalMaps() ( completionSignalingMap map[int]MessageCompletionSignalMapping, blockingSignalsMap map[int]MessageCompletionSignalMapping, -) { +) { + pp.Println(dag) completionSignalingMap = make(map[int]MessageCompletionSignalMapping) blockingSignalsMap = make(map[int]MessageCompletionSignalMapping) // go through every node @@ -292,6 +269,8 @@ func (dag *Dag) BuildCompletionSignalMaps() ( for _, edge := range outgoingEdges { maybeCompletionSignal := dag.GetCompletionSignal(edge) if maybeCompletionSignal != nil { + println("CompletionSignal") + pp.Println(maybeCompletionSignal) completionSignal := *maybeCompletionSignal toNode := dag.NodeMap[edge.ToNodeID] @@ -319,6 +298,10 @@ func (dag *Dag) BuildCompletionSignalMaps() ( } } } + println("CompletionSignalMapping") + pp.Println(completionSignalingMap) + println("BlockingSignalMapping") + pp.Println(blockingSignalsMap) return completionSignalingMap, blockingSignalsMap } diff --git a/go.mod b/go.mod index 984623bf9c..6559ef56e6 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/golang/protobuf v1.5.2 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 + github.com/k0kubun/pp v3.0.1+incompatible github.com/pkg/errors v0.9.1 github.com/regen-network/cosmos-proto v0.3.1 github.com/spf13/cast v1.5.0 @@ -88,6 +89,7 @@ require ( github.com/lib/pq v1.10.6 // indirect github.com/libp2p/go-buffer-pool v0.0.2 // indirect github.com/magiconair/properties v1.8.6 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -130,7 +132,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.1.70 + github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.1.72 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.1.56 diff --git a/go.sum b/go.sum index 5c2e2530e1..d2b24d213e 100644 --- a/go.sum +++ b/go.sum @@ -291,8 +291,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= -github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denis-tingaikin/go-header v0.4.3/go.mod h1:0wOCWuN71D5qIgE2nz9KrKmuYBAC2Mra5RassOIQ2/c= github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU= @@ -733,7 +731,10 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/julz/importas v0.1.0/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -799,6 +800,7 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= @@ -1097,8 +1099,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+fscA+Pulbpo= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= -github.com/sei-protocol/sei-cosmos v0.1.73 h1:yFG32zV8BGCbsS/AQYUNc2ZFWWZ2lXv505Gkf6vauKE= -github.com/sei-protocol/sei-cosmos v0.1.73/go.mod h1:Oaj7toqHCkwEEb+sDIWxtfTkPZxOpMXBXDMvIIqUjpw= +github.com/sei-protocol/sei-cosmos v0.1.72 h1:G7FopuMhoblNXHdpjZeMSxZ+CoaHzT8iWY2gaU5j7g4= +github.com/sei-protocol/sei-cosmos v0.1.72/go.mod h1:Oaj7toqHCkwEEb+sDIWxtfTkPZxOpMXBXDMvIIqUjpw= github.com/sei-protocol/sei-tendermint v0.1.56 h1:iRVhiIetj+GSwpBzaR9lqgvmgcOqmShfACS6x3tuBIw= github.com/sei-protocol/sei-tendermint v0.1.56/go.mod h1:Olwbjyagrpoxj5DAUhHxMTWDVEfQ3FYdpypaJ3+6Hs8= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= diff --git a/loadtest/contracts/deploy_ten_contracts.sh b/loadtest/contracts/deploy_ten_contracts.sh index ea985be2cd..6adb8edf9b 100644 --- a/loadtest/contracts/deploy_ten_contracts.sh +++ b/loadtest/contracts/deploy_ten_contracts.sh @@ -70,7 +70,7 @@ marsaddr4=$(python3 parser.py contract_address $marsinsres4) echo "Registering..." valaddr=$(printf "12345678\n" | $seidbin keys show $(printf "12345678\n" | $seidbin keys show node_admin --output json | jq -r .address) --bech=val --output json | jq -r '.address') -printf "12345678\n" | $seidbin tx staking delegate $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -—fees 2000use -b block -y +printf "12345678\n" | $seidbin tx staking delegate $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -—fees 2000usei -b block -y printf "12345678\n" | $seidbin tx dex register-contract $marsaddr $marsid false true -y --from=$keyname --chain-id=$chainid --fees=10000000usei --gas=500000 --broadcast-mode=block printf "12345678\n" | $seidbin tx dex register-contract $saturnaddr $saturnid false true -y --from=$keyname --chain-id=$chainid --fees=10000000usei --gas=500000 --broadcast-mode=block @@ -145,7 +145,7 @@ printf "12345678\n" | $seidbin tx gov vote $marsproposalid4 yes -y --from=$keyna sleep 90 -printf "12345678\n" | $seidbin tx staking unbond $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -b -—fees 2000use block -y +printf "12345678\n" | $seidbin tx staking unbond $valaddr 1000000000usei --from=$keyname --chain-id=$chainid -b block -y -—fees 2000usei echo $marsaddr echo $saturnaddr diff --git a/utils/metrics/labels.go b/utils/metrics/labels.go new file mode 100644 index 0000000000..aacc6c1e3b --- /dev/null +++ b/utils/metrics/labels.go @@ -0,0 +1,6 @@ +package metrics + +const ( + CONCURRENT = "concurrent" + SYNCHRONOUS = "synchronous" +) \ No newline at end of file diff --git a/utils/metrics/metrics_util.go b/utils/metrics/metrics_util.go new file mode 100644 index 0000000000..5f7a28e841 --- /dev/null +++ b/utils/metrics/metrics_util.go @@ -0,0 +1,18 @@ +package metrics + +import ( + metrics "github.com/armon/go-metrics" + "github.com/cosmos/cosmos-sdk/telemetry" +) + +// Measures the time taken to execute a sudo msg +// Metric Names: +// +// sei_tx_process_type_count +func IncrTransactionProcessTypeCounter(processType string) { + metrics.IncrCounterWithLabels( + []string{"sei", "tx", "process", "type"}, + 1, + []metrics.Label{telemetry.NewLabel("type", processType)}, + ) +} diff --git a/x/dex/keeper/epoch.go b/x/dex/keeper/epoch.go index d9ca566b7d..33f8012af4 100644 --- a/x/dex/keeper/epoch.go +++ b/x/dex/keeper/epoch.go @@ -14,6 +14,7 @@ func (k Keeper) SetEpoch(ctx sdk.Context, epoch uint64) { bz := make([]byte, 8) binary.BigEndian.PutUint64(bz, epoch) store.Set([]byte(EpochKey), bz) + ctx.Logger().Info(fmt.Sprintf("Current epoch %d", epoch)) } func (k Keeper) IsNewEpoch(ctx sdk.Context) (bool, uint64) { @@ -21,6 +22,5 @@ func (k Keeper) IsNewEpoch(ctx sdk.Context) (bool, uint64) { b := store.Get([]byte(EpochKey)) lastEpoch := binary.BigEndian.Uint64(b) currentEpoch := k.EpochKeeper.GetEpoch(ctx).CurrentEpoch - ctx.Logger().Info(fmt.Sprintf("Current epoch %d", currentEpoch)) return currentEpoch > lastEpoch, currentEpoch } From 0e40c4fe90cef8efb8e0a829042bb8f20db0ffd0 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Sun, 2 Oct 2022 21:18:53 -0700 Subject: [PATCH 10/14] rebase on main --- app/graph.go | 127 +++++++++++++++++++++++++++++---------------------- go.mod | 3 +- go.sum | 5 +- 3 files changed, 77 insertions(+), 58 deletions(-) diff --git a/app/graph.go b/app/graph.go index 59e20a383a..8301de3094 100644 --- a/app/graph.go +++ b/app/graph.go @@ -5,7 +5,6 @@ import ( acltypes "github.com/cosmos/cosmos-sdk/types/accesscontrol" mapset "github.com/deckarep/golang-set" - "github.com/k0kubun/pp" ) type DagNodeID int @@ -137,11 +136,11 @@ func (dag *Dag) AddNodeBuildDependency(messageIndex int, txIndex int, accessOp a dagNode := dag.AddNode(messageIndex, txIndex, accessOp) // update tx index map dag.TxIndexMap[txIndex] = dagNode.NodeID + nodeDependencies := dag.GetNodeDependencies(dagNode) // build edges for each of the dependencies for _, nodeDependency := range nodeDependencies { - println("AddNodeBuildDependency:Adding Node Edge!") - dag.AddEdge(nodeDependency.NodeID, dagNode.NodeID) + dag.AddEdge(nodeDependency, dagNode.NodeID) } // update access ops map with the latest node id using a specific access op @@ -159,56 +158,81 @@ func getAllNodeIDsFromIdentifierMapping(mapping ResourceIdentifierNodeIDMapping) return } -// This helper will identify nodes that are dependencies for the current node, and can then be used for creating edges between then for future completion signals -func (dag *Dag) GetNodeDependencies(node DagNode) (nodeDependencies []DagNode) { - accessOp := node.AccessOperation - // if the blocking access ops are in access ops map, make an edge - switch accessOp.AccessType { - case acltypes.AccessType_READ: - println("GetNodeDependencies: AccessType_READ") - // if we need to do a read, we need latest write as a dependency - // TODO: replace hardcoded access op dependencies with helper that generates (and also generates superseding resources too eg. Resource.ALL is blocking for Resource.KV) - writeAccessOp := acltypes.AccessOperation{ - AccessType: acltypes.AccessType_WRITE, - ResourceType: accessOp.GetResourceType(), - IdentifierTemplate: accessOp.GetIdentifierTemplate(), +func (dag *Dag) getDependencyWrites(node DagNode, dependentResource acltypes.ResourceType) mapset.Set { + nodeIDs := mapset.NewSet() + writeResourceAccess := ResourceAccess{ + dependentResource, + acltypes.AccessType_WRITE, + } + if identifierNodeMapping, ok := dag.ResourceAccessMap[writeResourceAccess]; ok { + var nodeIDsMaybeDependency []DagNodeID + if dependentResource != node.AccessOperation.ResourceType { + // we can add all node IDs as dependencies if applicable + nodeIDsMaybeDependency = getAllNodeIDsFromIdentifierMapping(identifierNodeMapping) + } else { + // TODO: otherwise we need to have partial filtering on identifiers + // for now, lets just perform exact matching on identifiers + nodeIDsMaybeDependency = identifierNodeMapping[node.AccessOperation.IdentifierTemplate] } - if writeNodeIDs, ok := dag.AccessOpsMap[writeAccessOp]; ok { - for _, wn := range writeNodeIDs { - writeNode := dag.NodeMap[wn] - // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) - // if from a previous transaction, we need to create an edge - if writeNode.TxIndex < node.TxIndex { - // this should be the COMMIT access op for the tx - lastTxNode := dag.NodeMap[dag.TxIndexMap[writeNode.TxIndex]] - nodeDependencies = append(nodeDependencies, lastTxNode) - } + for _, wn := range nodeIDsMaybeDependency { + writeNode := dag.NodeMap[wn] + // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) + // if from a previous transaction, we need to create an edge + if writeNode.TxIndex < node.TxIndex { + // this should be the COMMIT access op for the tx + lastTxNode := dag.NodeMap[dag.TxIndexMap[writeNode.TxIndex]] + nodeIDs.Add(lastTxNode.NodeID) } } - case acltypes.AccessType_WRITE, acltypes.AccessType_UNKNOWN: - println("GetNodeDependencies: AccessType_WRITE/AccessType_UNKNOWN") - // if we need to do a write, we need read and write as dependencies - writeAccessOp := acltypes.AccessOperation{ - AccessType: acltypes.AccessType_WRITE, - ResourceType: accessOp.GetResourceType(), - IdentifierTemplate: accessOp.GetIdentifierTemplate(), + } + return nodeIDs +} + +func (dag *Dag) getDependencyUnknowns(node DagNode, dependentResource acltypes.ResourceType) mapset.Set { + nodeIDs := mapset.NewSet() + unknownResourceAccess := ResourceAccess{ + dependentResource, + acltypes.AccessType_UNKNOWN, + } + if identifierNodeMapping, ok := dag.ResourceAccessMap[unknownResourceAccess]; ok { + var nodeIDsMaybeDependency []DagNodeID + if dependentResource != node.AccessOperation.ResourceType { + // we can add all node IDs as dependencies if applicable + nodeIDsMaybeDependency = getAllNodeIDsFromIdentifierMapping(identifierNodeMapping) + } else { + // TODO: otherwise we need to have partial filtering on identifiers + // for now, lets just perform exact matching on identifiers + nodeIDsMaybeDependency = identifierNodeMapping[node.AccessOperation.IdentifierTemplate] } - if writeNodeIDs, ok := dag.AccessOpsMap[writeAccessOp]; ok { - for _, wn := range writeNodeIDs { - // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) - writeNode := dag.NodeMap[wn] - // if from a previous transaction, we need to create an edge - if writeNode.TxIndex < node.TxIndex { - // we need to get the last node from that tx - lastTxNode := dag.NodeMap[dag.TxIndexMap[writeNode.TxIndex]] - nodeDependencies = append(nodeDependencies, lastTxNode) - } + for _, un := range nodeIDsMaybeDependency { + uNode := dag.NodeMap[un] + // if accessOp exists already (and from a previous transaction), we need to define a dependency on the previous message (and make a edge between the two) + // if from a previous transaction, we need to create an edge + if uNode.TxIndex < node.TxIndex { + // this should be the COMMIT access op for the tx + lastTxNode := dag.NodeMap[dag.TxIndexMap[uNode.TxIndex]] + nodeIDs.Add(lastTxNode.NodeID) } } - readAccessOp := acltypes.AccessOperation{ - AccessType: acltypes.AccessType_READ, - ResourceType: accessOp.GetResourceType(), - IdentifierTemplate: accessOp.GetIdentifierTemplate(), + } + return nodeIDs +} + +func (dag *Dag) getDependencyReads(node DagNode, dependentResource acltypes.ResourceType) mapset.Set { + nodeIDs := mapset.NewSet() + readResourceAccess := ResourceAccess{ + dependentResource, + acltypes.AccessType_READ, + } + if identifierNodeMapping, ok := dag.ResourceAccessMap[readResourceAccess]; ok { + var nodeIDsMaybeDependency []DagNodeID + if dependentResource != node.AccessOperation.ResourceType { + // we can add all node IDs as dependencies if applicable + nodeIDsMaybeDependency = getAllNodeIDsFromIdentifierMapping(identifierNodeMapping) + } else { + // TODO: otherwise we need to have partial filtering on identifiers + // for now, lets just perform exact matching on identifiers + nodeIDsMaybeDependency = identifierNodeMapping[node.AccessOperation.IdentifierTemplate] } for _, rn := range nodeIDsMaybeDependency { readNode := dag.NodeMap[rn] @@ -258,8 +282,7 @@ func (dag *Dag) GetNodeDependencies(node DagNode) []DagNodeID { func (dag *Dag) BuildCompletionSignalMaps() ( completionSignalingMap map[int]MessageCompletionSignalMapping, blockingSignalsMap map[int]MessageCompletionSignalMapping, -) { - pp.Println(dag) +) { completionSignalingMap = make(map[int]MessageCompletionSignalMapping) blockingSignalsMap = make(map[int]MessageCompletionSignalMapping) // go through every node @@ -269,8 +292,6 @@ func (dag *Dag) BuildCompletionSignalMaps() ( for _, edge := range outgoingEdges { maybeCompletionSignal := dag.GetCompletionSignal(edge) if maybeCompletionSignal != nil { - println("CompletionSignal") - pp.Println(maybeCompletionSignal) completionSignal := *maybeCompletionSignal toNode := dag.NodeMap[edge.ToNodeID] @@ -298,10 +319,6 @@ func (dag *Dag) BuildCompletionSignalMaps() ( } } } - println("CompletionSignalMapping") - pp.Println(completionSignalingMap) - println("BlockingSignalMapping") - pp.Println(blockingSignalsMap) return completionSignalingMap, blockingSignalsMap } diff --git a/go.mod b/go.mod index 6559ef56e6..cc75ec4c1b 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/cosmos/cosmos-sdk v0.45.4 github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/ibc-go/v3 v3.0.0 + github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea github.com/gogo/protobuf v1.3.3 github.com/golang/protobuf v1.5.2 github.com/gorilla/mux v1.8.0 @@ -132,7 +133,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.1.72 + github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.1.74 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.1.56 diff --git a/go.sum b/go.sum index d2b24d213e..a96db634e9 100644 --- a/go.sum +++ b/go.sum @@ -290,6 +290,7 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denis-tingaikin/go-header v0.4.3/go.mod h1:0wOCWuN71D5qIgE2nz9KrKmuYBAC2Mra5RassOIQ2/c= @@ -1099,8 +1100,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+fscA+Pulbpo= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= -github.com/sei-protocol/sei-cosmos v0.1.72 h1:G7FopuMhoblNXHdpjZeMSxZ+CoaHzT8iWY2gaU5j7g4= -github.com/sei-protocol/sei-cosmos v0.1.72/go.mod h1:Oaj7toqHCkwEEb+sDIWxtfTkPZxOpMXBXDMvIIqUjpw= +github.com/sei-protocol/sei-cosmos v0.1.74 h1:issmbjmVpD/HMF4OO+0rJLwGWgONsgfCQGxn52ANsy8= +github.com/sei-protocol/sei-cosmos v0.1.74/go.mod h1:Oaj7toqHCkwEEb+sDIWxtfTkPZxOpMXBXDMvIIqUjpw= github.com/sei-protocol/sei-tendermint v0.1.56 h1:iRVhiIetj+GSwpBzaR9lqgvmgcOqmShfACS6x3tuBIw= github.com/sei-protocol/sei-tendermint v0.1.56/go.mod h1:Olwbjyagrpoxj5DAUhHxMTWDVEfQ3FYdpypaJ3+6Hs8= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= From a6087da6123d74ec8b877ae0a091c92f65594741 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Sun, 2 Oct 2022 21:30:44 -0700 Subject: [PATCH 11/14] init channel list --- app/app.go | 1 + 1 file changed, 1 insertion(+) diff --git a/app/app.go b/app/app.go index 0df6ae836f..c8d8e95845 100644 --- a/app/app.go +++ b/app/app.go @@ -967,6 +967,7 @@ func getChannelsFromSignalMapping(signalMapping MessageCompletionSignalMapping) for messageIndex, accessOperationsToSignal := range signalMapping { for accessOperation, completionSignals := range accessOperationsToSignal { var channels []chan interface{} + channelsMapping[messageIndex] = make(sdkacltypes.AccessOpsChannelMapping) for _, completionSignal := range completionSignals { channels = append(channels, completionSignal.Channel) } From 6a9c2b4b44d81f56905ad2160814dcc21c5e3f65 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Sun, 2 Oct 2022 21:33:04 -0700 Subject: [PATCH 12/14] more logs --- app/app.go | 2 -- app/graph.go | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/app.go b/app/app.go index c8d8e95845..86d482bdd6 100644 --- a/app/app.go +++ b/app/app.go @@ -13,7 +13,6 @@ import ( "time" storetypes "github.com/cosmos/cosmos-sdk/store/types" - "github.com/k0kubun/pp" graph "github.com/yourbasic/graph" appparams "github.com/sei-protocol/sei-chain/app/params" @@ -890,7 +889,6 @@ func (app *App) BuildDependencyDag(ctx sdk.Context, txs [][]byte) (*Dag, error) msgKey := acltypes.GenerateMessageKey(msg) msgDependencies := app.AccessControlKeeper.GetResourceDependencyMapping(ctx, msgKey) ctx.Logger().Info(fmt.Sprintf("BuildDependencyDag:Getting Dependencies for %s", msgKey)) - pp.Println(msgDependencies) for _, accessOp := range msgDependencies.GetAccessOps() { // make a new node in the dependency dag ctx.Logger().Info(fmt.Sprintf("BuildDependencyDag:Get AccessOps Dependencies for %s", msgKey)) diff --git a/app/graph.go b/app/graph.go index 8301de3094..baeff00a47 100644 --- a/app/graph.go +++ b/app/graph.go @@ -5,6 +5,7 @@ import ( acltypes "github.com/cosmos/cosmos-sdk/types/accesscontrol" mapset "github.com/deckarep/golang-set" + "github.com/k0kubun/pp" ) type DagNodeID int @@ -319,6 +320,13 @@ func (dag *Dag) BuildCompletionSignalMaps() ( } } } + println("DAG") + pp.Println(dag) + println("completionSignalingMap") + pp.Println(completionSignalingMap) + println("blockingSignalsMap") + pp.Println(blockingSignalsMap) + return completionSignalingMap, blockingSignalsMap } From e1e17a893f0ee00086559a342069127be3e8a7f8 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Sun, 2 Oct 2022 23:34:41 -0700 Subject: [PATCH 13/14] fixo --- app/abci.go | 3 +++ utils/metrics/metrics_util.go | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/app/abci.go b/app/abci.go index 35b2a84ae5..10a841bd1e 100644 --- a/app/abci.go +++ b/app/abci.go @@ -2,8 +2,10 @@ package app import ( "context" + "time" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/sei-protocol/sei-chain/utils/metrics" abci "github.com/tendermint/tendermint/abci/types" "go.opentelemetry.io/otel/attribute" ) @@ -31,6 +33,7 @@ func (app *App) CheckTx(ctx context.Context, req *abci.RequestCheckTx) (*abci.Re } func (app *App) DeliverTx(ctx sdk.Context, req abci.RequestDeliverTx) abci.ResponseDeliverTx { + defer metrics.MeasureDeliverTxDuration(time.Now()) // tracectx, span := (*app.tracingInfo.Tracer).Start(app.tracingInfo.TracerContext, "DeliverTx") // oldCtx := app.tracingInfo.TracerContext // app.tracingInfo.TracerContext = tracectx diff --git a/utils/metrics/metrics_util.go b/utils/metrics/metrics_util.go index 5f7a28e841..838ecbde7c 100644 --- a/utils/metrics/metrics_util.go +++ b/utils/metrics/metrics_util.go @@ -1,6 +1,8 @@ package metrics import ( + "time" + metrics "github.com/armon/go-metrics" "github.com/cosmos/cosmos-sdk/telemetry" ) @@ -16,3 +18,16 @@ func IncrTransactionProcessTypeCounter(processType string) { []metrics.Label{telemetry.NewLabel("type", processType)}, ) } + +// Measures the time taken to execute a sudo msg +// Metric Names: +// +// sei_deliver_tx_duration_miliseconds +// sei_deliver_tx_duration_miliseconds_count +// sei_deliver_tx_duration_miliseconds_sum +func MeasureDeliverTxDuration(start time.Time) { + metrics.MeasureSince( + []string{"sei", "deliver", "tx", "milliseconds"}, + start.UTC(), + ) +} \ No newline at end of file From 102703873a0956cad14e8d7f764f41794583165d Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Mon, 3 Oct 2022 01:16:13 -0700 Subject: [PATCH 14/14] linting --- app/app.go | 6 +++--- utils/metrics/labels.go | 2 +- x/oracle/simulation/operations.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/app.go b/app/app.go index 86d482bdd6..0d84fa3760 100644 --- a/app/app.go +++ b/app/app.go @@ -1024,10 +1024,10 @@ func (app *App) ProcessBlockConcurrent( ctx.Logger().Info(fmt.Sprintf("ProcessBlockConcurrent:: Got transactions: %d", len(txs))) go func() { - waitGroup.Wait() + waitGroup.Wait() ctx.Logger().Info("Closing Channels!") - close(resultChan) - }() + close(resultChan) + }() // For each transaction, start goroutine and deliver TX for txIndex, txBytes := range txs { diff --git a/utils/metrics/labels.go b/utils/metrics/labels.go index aacc6c1e3b..b09db68c27 100644 --- a/utils/metrics/labels.go +++ b/utils/metrics/labels.go @@ -1,6 +1,6 @@ package metrics const ( - CONCURRENT = "concurrent" + CONCURRENT = "concurrent" SYNCHRONOUS = "synchronous" ) \ No newline at end of file diff --git a/x/oracle/simulation/operations.go b/x/oracle/simulation/operations.go index e7af90f2a9..6b3dc6c946 100644 --- a/x/oracle/simulation/operations.go +++ b/x/oracle/simulation/operations.go @@ -83,7 +83,7 @@ func WeightedOperations( } // SimulateMsgAggregateExchangeRatePrevote generates a MsgAggregateExchangeRatePrevote with random values. -// nolint: funlen +//nolint: funlen func SimulateMsgAggregateExchangeRatePrevote(ak types.AccountKeeper, bk types.BankKeeper, k keeper.Keeper) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string, @@ -146,7 +146,7 @@ func SimulateMsgAggregateExchangeRatePrevote(ak types.AccountKeeper, bk types.Ba } // SimulateMsgAggregateExchangeRateVote generates a MsgAggregateExchangeRateVote with random values. -// nolint: funlen +//nolint: funlen func SimulateMsgAggregateExchangeRateVote(ak types.AccountKeeper, bk types.BankKeeper, k keeper.Keeper) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string, @@ -213,7 +213,7 @@ func SimulateMsgAggregateExchangeRateVote(ak types.AccountKeeper, bk types.BankK } // SimulateMsgDelegateFeedConsent generates a MsgDelegateFeedConsent with random values. -// nolint: funlen +//nolint: funlen func SimulateMsgDelegateFeedConsent(ak types.AccountKeeper, bk types.BankKeeper, k keeper.Keeper) simtypes.Operation { return func( r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simtypes.Account, chainID string,