From 1f5461479614ffda6e1ba6427b6d9ab404ff7cde Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Fri, 8 Oct 2021 20:16:01 +0530 Subject: [PATCH 1/9] Implement notify.finalized Telemetry Interface Fixes #1835 --- dot/state/block_finalisation.go | 2 +- dot/state/block_notify.go | 7 ++ dot/telemetry/block_import.go | 45 ++++++++++ dot/telemetry/network_state.go | 66 ++++++++++++++ dot/telemetry/notify_finalized.go | 42 +++++++++ dot/telemetry/system_connected.go | 52 +++++++++++ dot/telemetry/system_interval.go | 65 ++++++++++++++ dot/telemetry/telemetry.go | 141 ------------------------------ 8 files changed, 278 insertions(+), 142 deletions(-) create mode 100644 dot/telemetry/block_import.go create mode 100644 dot/telemetry/network_state.go create mode 100644 dot/telemetry/notify_finalized.go create mode 100644 dot/telemetry/system_connected.go create mode 100644 dot/telemetry/system_interval.go diff --git a/dot/state/block_finalisation.go b/dot/state/block_finalisation.go index 14b05cfacf..cfab576bf7 100644 --- a/dot/state/block_finalisation.go +++ b/dot/state/block_finalisation.go @@ -126,7 +126,7 @@ func (bs *BlockState) GetHighestFinalisedHeader() (*types.Header, error) { return header, nil } -// SetFinalisedHash sets the latest finalised block header +// SetFinalisedHash sets the latest finalised block hash func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) error { bs.Lock() defer bs.Unlock() diff --git a/dot/state/block_notify.go b/dot/state/block_notify.go index d0f7e4181c..aeb48cee92 100644 --- a/dot/state/block_notify.go +++ b/dot/state/block_notify.go @@ -20,6 +20,7 @@ import ( "errors" "sync" + "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/runtime" @@ -111,6 +112,12 @@ func (bs *BlockState) notifyFinalized(hash common.Hash, round, setID uint64) { go func(ch chan *types.FinalisationInfo) { select { case ch <- info: + telemetry.GetInstance().SendMessage( + telemetry.NewNotifyFinalizedTM( + info.Header.Hash(), + info.Header.Number, + ), + ) default: } }(ch) diff --git a/dot/telemetry/block_import.go b/dot/telemetry/block_import.go new file mode 100644 index 0000000000..b8af7cda7f --- /dev/null +++ b/dot/telemetry/block_import.go @@ -0,0 +1,45 @@ +// Copyright 2021 ChainSafe Systems (ON) Corp. +// This file is part of gossamer. +// +// The gossamer library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The gossamer library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the gossamer library. If not, see . + +package telemetry + +import ( + "math/big" + + "github.com/ChainSafe/gossamer/lib/common" +) + +// BlockImportTM struct to hold block import telemetry messages +type BlockImportTM struct { + BestHash *common.Hash `json:"best"` + Height *big.Int `json:"height"` + Msg string `json:"msg"` + Origin string `json:"origin"` +} + +// NewBlockImportTM function to create new Block Import Telemetry Message +func NewBlockImportTM(bestHash *common.Hash, height *big.Int, origin string) *BlockImportTM { + return &BlockImportTM{ + BestHash: bestHash, + Height: height, + Msg: "block.import", + Origin: origin, + } +} + +func (tm *BlockImportTM) messageType() string { + return tm.Msg +} diff --git a/dot/telemetry/network_state.go b/dot/telemetry/network_state.go new file mode 100644 index 0000000000..66c9298c45 --- /dev/null +++ b/dot/telemetry/network_state.go @@ -0,0 +1,66 @@ +// Copyright 2021 ChainSafe Systems (ON) Corp. +// This file is part of gossamer. +// +// The gossamer library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The gossamer library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the gossamer library. If not, see . + +package telemetry + +import ( + "fmt" + + "github.com/ChainSafe/gossamer/lib/common" + libp2phost "github.com/libp2p/go-libp2p-core/host" +) + +// NetworkStateTM struct to hold network state telemetry messages +type NetworkStateTM struct { + Msg string `json:"msg"` + State map[string]interface{} `json:"state"` +} + +// NewNetworkStateTM function to create new Network State Telemetry Message +func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) *NetworkStateTM { + netState := make(map[string]interface{}) + netState["peerId"] = host.ID() + hostAddrs := []string{} + for _, v := range host.Addrs() { + hostAddrs = append(hostAddrs, v.String()) + } + netState["externalAddressess"] = hostAddrs + listAddrs := []string{} + for _, v := range host.Network().ListenAddresses() { + listAddrs = append(listAddrs, fmt.Sprintf("%s/p2p/%s", v, host.ID())) + } + netState["listenedAddressess"] = listAddrs + + peers := make(map[string]interface{}) + for _, v := range peerInfos { + p := &peerInfo{ + Roles: v.Roles, + BestHash: v.BestHash.String(), + BestNumber: v.BestNumber, + } + peers[v.PeerID] = *p + } + netState["connectedPeers"] = peers + + return &NetworkStateTM{ + Msg: "system.network_state", + State: netState, + } +} + +func (tm *NetworkStateTM) messageType() string { + return tm.Msg +} diff --git a/dot/telemetry/notify_finalized.go b/dot/telemetry/notify_finalized.go new file mode 100644 index 0000000000..e756c35aea --- /dev/null +++ b/dot/telemetry/notify_finalized.go @@ -0,0 +1,42 @@ +// Copyright 2021 ChainSafe Systems (ON) Corp. +// This file is part of gossamer. +// +// The gossamer library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The gossamer library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the gossamer library. If not, see . + +package telemetry + +import ( + "math/big" + + "github.com/ChainSafe/gossamer/lib/common" +) + +type NotifyFinalizedTM struct { + Best *common.Hash `json:"best"` + // Height is same as block.Header.Number + Height *big.Int `json:"height"` + Msg string `json:"msg"` +} + +func NewNotifyFinalizedTM(best common.Hash, height *big.Int) *NotifyFinalizedTM { + return &NotifyFinalizedTM{ + Best: &best, + Height: height, + Msg: "notify.finalized", + } +} + +func (tm *NotifyFinalizedTM) messageType() string { + return tm.Msg +} diff --git a/dot/telemetry/system_connected.go b/dot/telemetry/system_connected.go new file mode 100644 index 0000000000..a8a2986ce8 --- /dev/null +++ b/dot/telemetry/system_connected.go @@ -0,0 +1,52 @@ +// Copyright 2021 ChainSafe Systems (ON) Corp. +// This file is part of gossamer. +// +// The gossamer library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The gossamer library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the gossamer library. If not, see . + +package telemetry + +import "github.com/ChainSafe/gossamer/lib/common" + +// SystemConnectedTM struct to hold system connected telemetry messages +type SystemConnectedTM struct { + Authority bool `json:"authority"` + Chain string `json:"chain"` + GenesisHash *common.Hash `json:"genesis_hash"` + Implementation string `json:"implementation"` + Msg string `json:"msg"` + Name string `json:"name"` + NetworkID string `json:"network_id"` + StartupTime string `json:"startup_time"` + Version string `json:"version"` +} + +// NewSystemConnectedTM function to create new System Connected Telemetry Message +func NewSystemConnectedTM(authority bool, chain string, genesisHash *common.Hash, + implementation, name, networkID, startupTime, version string) *SystemConnectedTM { + return &SystemConnectedTM{ + Authority: authority, + Chain: chain, + GenesisHash: genesisHash, + Implementation: implementation, + Msg: "system.connected", + Name: name, + NetworkID: networkID, + StartupTime: startupTime, + Version: version, + } +} + +func (tm *SystemConnectedTM) messageType() string { + return tm.Msg +} diff --git a/dot/telemetry/system_interval.go b/dot/telemetry/system_interval.go new file mode 100644 index 0000000000..a44e99ea73 --- /dev/null +++ b/dot/telemetry/system_interval.go @@ -0,0 +1,65 @@ +// Copyright 2021 ChainSafe Systems (ON) Corp. +// This file is part of gossamer. +// +// The gossamer library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The gossamer library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the gossamer library. If not, see . + +package telemetry + +import ( + "math/big" + + "github.com/ChainSafe/gossamer/lib/common" +) + +// SystemIntervalTM struct to hold system interval telemetry messages +type SystemIntervalTM struct { + BandwidthDownload float64 `json:"bandwidth_download,omitempty"` + BandwidthUpload float64 `json:"bandwidth_upload,omitempty"` + Msg string `json:"msg"` + Peers int `json:"peers,omitempty"` + BestHash *common.Hash `json:"best,omitempty"` + BestHeight *big.Int `json:"height,omitempty"` + FinalisedHash *common.Hash `json:"finalized_hash,omitempty"` // nolint + FinalisedHeight *big.Int `json:"finalized_height,omitempty"` // nolint + TxCount *big.Int `json:"txcount,omitempty"` + UsedStateCacheSize *big.Int `json:"used_state_cache_size,omitempty"` +} + +// NewBandwidthTM function to create new Bandwidth Telemetry Message +func NewBandwidthTM(bandwidthDownload, bandwidthUpload float64, peers int) *SystemIntervalTM { + return &SystemIntervalTM{ + BandwidthDownload: bandwidthDownload, + BandwidthUpload: bandwidthUpload, + Msg: "system.interval", + Peers: peers, + } +} + +// NewBlockIntervalTM function to create new Block Interval Telemetry Message +func NewBlockIntervalTM(beshHash *common.Hash, bestHeight *big.Int, finalisedHash *common.Hash, + finalisedHeight, txCount, usedStateCacheSize *big.Int) *SystemIntervalTM { + return &SystemIntervalTM{ + Msg: "system.interval", + BestHash: beshHash, + BestHeight: bestHeight, + FinalisedHash: finalisedHash, + FinalisedHeight: finalisedHeight, + TxCount: txCount, + UsedStateCacheSize: usedStateCacheSize, + } +} + +func (tm *SystemIntervalTM) messageType() string { + return tm.Msg +} diff --git a/dot/telemetry/telemetry.go b/dot/telemetry/telemetry.go index 873ed4273a..b3bfe1df62 100644 --- a/dot/telemetry/telemetry.go +++ b/dot/telemetry/telemetry.go @@ -19,16 +19,12 @@ package telemetry import ( "encoding/json" "errors" - "fmt" - "math/big" "sync" "time" - "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" log "github.com/ChainSafe/log15" "github.com/gorilla/websocket" - libp2phost "github.com/libp2p/go-libp2p-core/host" ) type telemetryConnection struct { @@ -171,149 +167,12 @@ type Message interface { messageType() string } -// SystemConnectedTM struct to hold system connected telemetry messages -type SystemConnectedTM struct { - Authority bool `json:"authority"` - Chain string `json:"chain"` - GenesisHash *common.Hash `json:"genesis_hash"` - Implementation string `json:"implementation"` - Msg string `json:"msg"` - Name string `json:"name"` - NetworkID string `json:"network_id"` - StartupTime string `json:"startup_time"` - Version string `json:"version"` -} - -// NewSystemConnectedTM function to create new System Connected Telemetry Message -func NewSystemConnectedTM(authority bool, chain string, genesisHash *common.Hash, - implementation, name, networkID, startupTime, version string) *SystemConnectedTM { - return &SystemConnectedTM{ - Authority: authority, - Chain: chain, - GenesisHash: genesisHash, - Implementation: implementation, - Msg: "system.connected", - Name: name, - NetworkID: networkID, - StartupTime: startupTime, - Version: version, - } -} -func (tm *SystemConnectedTM) messageType() string { - return tm.Msg -} - -// BlockImportTM struct to hold block import telemetry messages -type BlockImportTM struct { - BestHash *common.Hash `json:"best"` - Height *big.Int `json:"height"` - Msg string `json:"msg"` - Origin string `json:"origin"` -} - -// NewBlockImportTM function to create new Block Import Telemetry Message -func NewBlockImportTM(bestHash *common.Hash, height *big.Int, origin string) *BlockImportTM { - return &BlockImportTM{ - BestHash: bestHash, - Height: height, - Msg: "block.import", - Origin: origin, - } -} - -func (tm *BlockImportTM) messageType() string { - return tm.Msg -} - -// SystemIntervalTM struct to hold system interval telemetry messages -type SystemIntervalTM struct { - BandwidthDownload float64 `json:"bandwidth_download,omitempty"` - BandwidthUpload float64 `json:"bandwidth_upload,omitempty"` - Msg string `json:"msg"` - Peers int `json:"peers,omitempty"` - BestHash *common.Hash `json:"best,omitempty"` - BestHeight *big.Int `json:"height,omitempty"` - FinalisedHash *common.Hash `json:"finalized_hash,omitempty"` // nolint - FinalisedHeight *big.Int `json:"finalized_height,omitempty"` // nolint - TxCount *big.Int `json:"txcount,omitempty"` - UsedStateCacheSize *big.Int `json:"used_state_cache_size,omitempty"` -} - -// NewBandwidthTM function to create new Bandwidth Telemetry Message -func NewBandwidthTM(bandwidthDownload, bandwidthUpload float64, peers int) *SystemIntervalTM { - return &SystemIntervalTM{ - BandwidthDownload: bandwidthDownload, - BandwidthUpload: bandwidthUpload, - Msg: "system.interval", - Peers: peers, - } -} - -// NewBlockIntervalTM function to create new Block Interval Telemetry Message -func NewBlockIntervalTM(beshHash *common.Hash, bestHeight *big.Int, finalisedHash *common.Hash, - finalisedHeight, txCount, usedStateCacheSize *big.Int) *SystemIntervalTM { - return &SystemIntervalTM{ - Msg: "system.interval", - BestHash: beshHash, - BestHeight: bestHeight, - FinalisedHash: finalisedHash, - FinalisedHeight: finalisedHeight, - TxCount: txCount, - UsedStateCacheSize: usedStateCacheSize, - } -} - -func (tm *SystemIntervalTM) messageType() string { - return tm.Msg -} - type peerInfo struct { Roles byte `json:"roles"` BestHash string `json:"bestHash"` BestNumber uint64 `json:"bestNumber"` } -// NetworkStateTM struct to hold network state telemetry messages -type NetworkStateTM struct { - Msg string `json:"msg"` - State map[string]interface{} `json:"state"` -} - -// NewNetworkStateTM function to create new Network State Telemetry Message -func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) *NetworkStateTM { - netState := make(map[string]interface{}) - netState["peerId"] = host.ID() - hostAddrs := []string{} - for _, v := range host.Addrs() { - hostAddrs = append(hostAddrs, v.String()) - } - netState["externalAddressess"] = hostAddrs - listAddrs := []string{} - for _, v := range host.Network().ListenAddresses() { - listAddrs = append(listAddrs, fmt.Sprintf("%s/p2p/%s", v, host.ID())) - } - netState["listenedAddressess"] = listAddrs - - peers := make(map[string]interface{}) - for _, v := range peerInfos { - p := &peerInfo{ - Roles: v.Roles, - BestHash: v.BestHash.String(), - BestNumber: v.BestNumber, - } - peers[v.PeerID] = *p - } - netState["connectedPeers"] = peers - - return &NetworkStateTM{ - Msg: "system.network_state", - State: netState, - } -} -func (tm *NetworkStateTM) messageType() string { - return tm.Msg -} - // NoopHandler struct no op handling (ignoring) telemetry messages type NoopHandler struct { } From 11f262656e448435fb82c333d09e7423a87c475e Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Mon, 11 Oct 2021 15:21:35 +0530 Subject: [PATCH 2/9] Added tests for notify.finalized telemetry message --- dot/telemetry/notify_finalized.go | 4 ++-- dot/telemetry/telemetry_test.go | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dot/telemetry/notify_finalized.go b/dot/telemetry/notify_finalized.go index e756c35aea..46fc4128bd 100644 --- a/dot/telemetry/notify_finalized.go +++ b/dot/telemetry/notify_finalized.go @@ -23,7 +23,7 @@ import ( ) type NotifyFinalizedTM struct { - Best *common.Hash `json:"best"` + Best common.Hash `json:"best"` // Height is same as block.Header.Number Height *big.Int `json:"height"` Msg string `json:"msg"` @@ -31,7 +31,7 @@ type NotifyFinalizedTM struct { func NewNotifyFinalizedTM(best common.Hash, height *big.Int) *NotifyFinalizedTM { return &NotifyFinalizedTM{ - Best: &best, + Best: best, Height: height, Msg: "notify.finalized", } diff --git a/dot/telemetry/telemetry_test.go b/dot/telemetry/telemetry_test.go index 7e89665149..dc25496bdf 100644 --- a/dot/telemetry/telemetry_test.go +++ b/dot/telemetry/telemetry_test.go @@ -43,7 +43,7 @@ func TestMain(m *testing.M) { func TestHandler_SendMulti(t *testing.T) { var wg sync.WaitGroup - wg.Add(4) + wg.Add(5) resultCh = make(chan []byte) @@ -78,19 +78,27 @@ func TestHandler_SendMulti(t *testing.T) { wg.Done() }() + go func() { + bestHash := common.MustHexToHash("0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6") + GetInstance().SendMessage(NewNotifyFinalizedTM(bestHash, big.NewInt(32375))) + + wg.Done() + }() + wg.Wait() expected1 := []byte(`{"authority":false,"chain":"chain","genesis_hash":"0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3","implementation":"systemName","msg":"system.connected","name":"nodeName","network_id":"netID","startup_time":"startTime","ts":`) expected2 := []byte(`{"best":"0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6","height":2,"msg":"block.import","origin":"NetworkInitialSync","ts":`) expected3 := []byte(`{"bandwidth_download":2,"bandwidth_upload":3,"msg":"system.interval","peers":1,"ts":`) expected4 := []byte(`{"best":"0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6","finalized_hash":"0x687197c11b4cf95374159843e7f46fbcd63558db981aaef01a8bac2a44a1d6b2","finalized_height":32256,"height":32375,"msg":"system.interval","ts":`) // nolint + expected5 := []byte(`{"best":"0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6","height":32375,"msg":"notify.finalized","ts":`) - expected := [][]byte{expected1, expected3, expected4, expected2} + expected := [][]byte{expected1, expected3, expected4, expected2, expected5} var actual [][]byte for data := range resultCh { actual = append(actual, data) - if len(actual) == 4 { + if len(actual) == 5 { break } } @@ -98,10 +106,12 @@ func TestHandler_SendMulti(t *testing.T) { sort.Slice(actual, func(i, j int) bool { return bytes.Compare(actual[i], actual[j]) < 0 }) + require.Contains(t, string(actual[0]), string(expected[0])) require.Contains(t, string(actual[1]), string(expected[1])) require.Contains(t, string(actual[2]), string(expected[2])) require.Contains(t, string(actual[3]), string(expected[3])) + require.Contains(t, string(actual[4]), string(expected[4])) } func TestListenerConcurrency(t *testing.T) { From f6b1f10e496428123d095dfa5c718557ca4e4b55 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Mon, 11 Oct 2021 20:59:39 +0530 Subject: [PATCH 3/9] Fixing some linting issues --- dot/state/block_notify.go | 5 ++++- dot/telemetry/notify_finalized.go | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dot/state/block_notify.go b/dot/state/block_notify.go index aeb48cee92..4237f645b2 100644 --- a/dot/state/block_notify.go +++ b/dot/state/block_notify.go @@ -112,12 +112,15 @@ func (bs *BlockState) notifyFinalized(hash common.Hash, round, setID uint64) { go func(ch chan *types.FinalisationInfo) { select { case ch <- info: - telemetry.GetInstance().SendMessage( + err := telemetry.GetInstance().SendMessage( telemetry.NewNotifyFinalizedTM( info.Header.Hash(), info.Header.Number, ), ) + if err != nil { + logger.Error("could not send 'notify.finalized' telemetry message", "error", err) + } default: } }(ch) diff --git a/dot/telemetry/notify_finalized.go b/dot/telemetry/notify_finalized.go index 46fc4128bd..581f795b7c 100644 --- a/dot/telemetry/notify_finalized.go +++ b/dot/telemetry/notify_finalized.go @@ -22,6 +22,9 @@ import ( "github.com/ChainSafe/gossamer/lib/common" ) +//nolint +// NotifyFinalizedTM holds `notify.finalized` telemetry message, which is +// supposed to be send when a new block gets finalized. type NotifyFinalizedTM struct { Best common.Hash `json:"best"` // Height is same as block.Header.Number @@ -29,6 +32,7 @@ type NotifyFinalizedTM struct { Msg string `json:"msg"` } +// NewNotifyFinalizedTM gets a new NotifyFinalizedTM struct. func NewNotifyFinalizedTM(best common.Hash, height *big.Int) *NotifyFinalizedTM { return &NotifyFinalizedTM{ Best: best, From 98279c158bc287c99fb5fa4e648fc5cf04e5028d Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Fri, 15 Oct 2021 14:25:11 +0530 Subject: [PATCH 4/9] Send telemetry message from SetFinalisedHash instead --- dot/state/block_finalisation.go | 28 ++++++++++++++++++++++------ dot/state/block_notify.go | 10 ---------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/dot/state/block_finalisation.go b/dot/state/block_finalisation.go index cfab576bf7..b23af57bc9 100644 --- a/dot/state/block_finalisation.go +++ b/dot/state/block_finalisation.go @@ -21,6 +21,7 @@ import ( "fmt" "math/big" + "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" ) @@ -154,20 +155,20 @@ func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) er } pruned := bs.bt.Prune(hash) - for _, hash := range pruned { - header, err := bs.GetHeader(hash) + for _, prunedHash := range pruned { + header, err := bs.GetHeader(prunedHash) if err != nil { - logger.Debug("failed to get pruned header", "hash", hash, "error", err) + logger.Debug("failed to get pruned header", "hash", prunedHash, "error", err) continue } - err = bs.DeleteBlock(hash) + err = bs.DeleteBlock(prunedHash) if err != nil { - logger.Debug("failed to delete block", "hash", hash, "error", err) + logger.Debug("failed to delete block", "hash", prunedHash, "error", err) continue } - logger.Trace("pruned block", "hash", hash, "number", header.Number) + logger.Trace("pruned block", "hash", prunedHash, "number", header.Number) go func(header *types.Header) { bs.pruneKeyCh <- header }(header) @@ -179,6 +180,21 @@ func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) er return err } + header, err := bs.GetHeader(hash) + if err != nil { + return fmt.Errorf("failed to get finalised header, hash: %s, error: %s", hash, err) + } + + err = telemetry.GetInstance().SendMessage( + telemetry.NewNotifyFinalizedTM( + header.Hash(), + header.Number, + ), + ) + if err != nil { + return fmt.Errorf("could not send 'notify.finalized' telemetry message, error: %s", err) + } + return bs.setHighestRoundAndSetID(round, setID) } diff --git a/dot/state/block_notify.go b/dot/state/block_notify.go index 4237f645b2..d0f7e4181c 100644 --- a/dot/state/block_notify.go +++ b/dot/state/block_notify.go @@ -20,7 +20,6 @@ import ( "errors" "sync" - "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/runtime" @@ -112,15 +111,6 @@ func (bs *BlockState) notifyFinalized(hash common.Hash, round, setID uint64) { go func(ch chan *types.FinalisationInfo) { select { case ch <- info: - err := telemetry.GetInstance().SendMessage( - telemetry.NewNotifyFinalizedTM( - info.Header.Hash(), - info.Header.Number, - ), - ) - if err != nil { - logger.Error("could not send 'notify.finalized' telemetry message", "error", err) - } default: } }(ch) From 39cd90d0605792a4dd28d82f5366197813799850 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Fri, 15 Oct 2021 18:42:38 +0530 Subject: [PATCH 5/9] Make NotifyFinalizedTM.Height string instread of big.Int to match with the format accepted by telemetry server --- dot/state/block_finalisation.go | 2 +- dot/telemetry/notify_finalized.go | 8 +++----- dot/telemetry/telemetry_test.go | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dot/state/block_finalisation.go b/dot/state/block_finalisation.go index b23af57bc9..294fd26a95 100644 --- a/dot/state/block_finalisation.go +++ b/dot/state/block_finalisation.go @@ -188,7 +188,7 @@ func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) er err = telemetry.GetInstance().SendMessage( telemetry.NewNotifyFinalizedTM( header.Hash(), - header.Number, + header.Number.String(), ), ) if err != nil { diff --git a/dot/telemetry/notify_finalized.go b/dot/telemetry/notify_finalized.go index 581f795b7c..41b29596aa 100644 --- a/dot/telemetry/notify_finalized.go +++ b/dot/telemetry/notify_finalized.go @@ -17,8 +17,6 @@ package telemetry import ( - "math/big" - "github.com/ChainSafe/gossamer/lib/common" ) @@ -28,12 +26,12 @@ import ( type NotifyFinalizedTM struct { Best common.Hash `json:"best"` // Height is same as block.Header.Number - Height *big.Int `json:"height"` - Msg string `json:"msg"` + Height string `json:"height"` + Msg string `json:"msg"` } // NewNotifyFinalizedTM gets a new NotifyFinalizedTM struct. -func NewNotifyFinalizedTM(best common.Hash, height *big.Int) *NotifyFinalizedTM { +func NewNotifyFinalizedTM(best common.Hash, height string) *NotifyFinalizedTM { return &NotifyFinalizedTM{ Best: best, Height: height, diff --git a/dot/telemetry/telemetry_test.go b/dot/telemetry/telemetry_test.go index dc25496bdf..1869923adc 100644 --- a/dot/telemetry/telemetry_test.go +++ b/dot/telemetry/telemetry_test.go @@ -80,7 +80,7 @@ func TestHandler_SendMulti(t *testing.T) { go func() { bestHash := common.MustHexToHash("0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6") - GetInstance().SendMessage(NewNotifyFinalizedTM(bestHash, big.NewInt(32375))) + GetInstance().SendMessage(NewNotifyFinalizedTM(bestHash, "32375")) wg.Done() }() From 1bb3f5bf8cabc6de39970cecafe1764943f3e0ce Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Fri, 15 Oct 2021 19:28:11 +0530 Subject: [PATCH 6/9] Fix telemetry test --- dot/telemetry/telemetry_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dot/telemetry/telemetry_test.go b/dot/telemetry/telemetry_test.go index 1869923adc..0ab186c5fa 100644 --- a/dot/telemetry/telemetry_test.go +++ b/dot/telemetry/telemetry_test.go @@ -91,9 +91,9 @@ func TestHandler_SendMulti(t *testing.T) { expected2 := []byte(`{"best":"0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6","height":2,"msg":"block.import","origin":"NetworkInitialSync","ts":`) expected3 := []byte(`{"bandwidth_download":2,"bandwidth_upload":3,"msg":"system.interval","peers":1,"ts":`) expected4 := []byte(`{"best":"0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6","finalized_hash":"0x687197c11b4cf95374159843e7f46fbcd63558db981aaef01a8bac2a44a1d6b2","finalized_height":32256,"height":32375,"msg":"system.interval","ts":`) // nolint - expected5 := []byte(`{"best":"0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6","height":32375,"msg":"notify.finalized","ts":`) + expected5 := []byte(`{"best":"0x07b749b6e20fd5f1159153a2e790235018621dd06072a62bcd25e8576f6ff5e6","height":"32375","msg":"notify.finalized","ts":`) - expected := [][]byte{expected1, expected3, expected4, expected2, expected5} + expected := [][]byte{expected1, expected3, expected4, expected5, expected2} var actual [][]byte for data := range resultCh { From 4ad75912ba87350ddf2b87ad2533744807b72e23 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Tue, 19 Oct 2021 19:23:01 +0530 Subject: [PATCH 7/9] Addressed some reviews - consts for message types - unexport message structs - other small changes --- dot/telemetry/block_import.go | 14 ++++++-------- dot/telemetry/network_state.go | 22 +++++++++++----------- dot/telemetry/notify_finalized.go | 14 ++++++-------- dot/telemetry/system_connected.go | 14 ++++++-------- dot/telemetry/system_interval.go | 19 ++++++++----------- dot/telemetry/telemetry.go | 9 +++++++++ 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/dot/telemetry/block_import.go b/dot/telemetry/block_import.go index b8af7cda7f..5e6d9278f1 100644 --- a/dot/telemetry/block_import.go +++ b/dot/telemetry/block_import.go @@ -22,24 +22,22 @@ import ( "github.com/ChainSafe/gossamer/lib/common" ) -// BlockImportTM struct to hold block import telemetry messages -type BlockImportTM struct { +// blockImportTM struct to hold block import telemetry messages +type blockImportTM struct { BestHash *common.Hash `json:"best"` Height *big.Int `json:"height"` - Msg string `json:"msg"` Origin string `json:"origin"` } // NewBlockImportTM function to create new Block Import Telemetry Message -func NewBlockImportTM(bestHash *common.Hash, height *big.Int, origin string) *BlockImportTM { - return &BlockImportTM{ +func NewBlockImportTM(bestHash *common.Hash, height *big.Int, origin string) *blockImportTM { + return &blockImportTM{ BestHash: bestHash, Height: height, - Msg: "block.import", Origin: origin, } } -func (tm *BlockImportTM) messageType() string { - return tm.Msg +func (tm *blockImportTM) messageType() string { + return blockImportMsg } diff --git a/dot/telemetry/network_state.go b/dot/telemetry/network_state.go index 66c9298c45..21905df890 100644 --- a/dot/telemetry/network_state.go +++ b/dot/telemetry/network_state.go @@ -23,23 +23,24 @@ import ( libp2phost "github.com/libp2p/go-libp2p-core/host" ) -// NetworkStateTM struct to hold network state telemetry messages -type NetworkStateTM struct { - Msg string `json:"msg"` +// networkStateTM struct to hold network state telemetry messages +type networkStateTM struct { State map[string]interface{} `json:"state"` } // NewNetworkStateTM function to create new Network State Telemetry Message -func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) *NetworkStateTM { +func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) *networkStateTM { netState := make(map[string]interface{}) netState["peerId"] = host.ID() - hostAddrs := []string{} + hostAddrs := make([]string, 0, len(host.Addrs())) for _, v := range host.Addrs() { hostAddrs = append(hostAddrs, v.String()) } netState["externalAddressess"] = hostAddrs - listAddrs := []string{} - for _, v := range host.Network().ListenAddresses() { + + netListAddrs := host.Network().ListenAddresses() + listAddrs := make([]string, 0, len(netListAddrs)) + for _, v := range netListAddrs { listAddrs = append(listAddrs, fmt.Sprintf("%s/p2p/%s", v, host.ID())) } netState["listenedAddressess"] = listAddrs @@ -55,12 +56,11 @@ func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) *Netwo } netState["connectedPeers"] = peers - return &NetworkStateTM{ - Msg: "system.network_state", + return &networkStateTM{ State: netState, } } -func (tm *NetworkStateTM) messageType() string { - return tm.Msg +func (tm *networkStateTM) messageType() string { + return systemNetworkStateMsg } diff --git a/dot/telemetry/notify_finalized.go b/dot/telemetry/notify_finalized.go index 41b29596aa..fd08716741 100644 --- a/dot/telemetry/notify_finalized.go +++ b/dot/telemetry/notify_finalized.go @@ -21,24 +21,22 @@ import ( ) //nolint -// NotifyFinalizedTM holds `notify.finalized` telemetry message, which is +// notifyFinalizedTM holds `notify.finalized` telemetry message, which is // supposed to be send when a new block gets finalized. -type NotifyFinalizedTM struct { +type notifyFinalizedTM struct { Best common.Hash `json:"best"` // Height is same as block.Header.Number Height string `json:"height"` - Msg string `json:"msg"` } // NewNotifyFinalizedTM gets a new NotifyFinalizedTM struct. -func NewNotifyFinalizedTM(best common.Hash, height string) *NotifyFinalizedTM { - return &NotifyFinalizedTM{ +func NewNotifyFinalizedTM(best common.Hash, height string) *notifyFinalizedTM { + return ¬ifyFinalizedTM{ Best: best, Height: height, - Msg: "notify.finalized", } } -func (tm *NotifyFinalizedTM) messageType() string { - return tm.Msg +func (tm *notifyFinalizedTM) messageType() string { + return notifyFinalizedMsg } diff --git a/dot/telemetry/system_connected.go b/dot/telemetry/system_connected.go index a8a2986ce8..73ae95ee80 100644 --- a/dot/telemetry/system_connected.go +++ b/dot/telemetry/system_connected.go @@ -18,13 +18,12 @@ package telemetry import "github.com/ChainSafe/gossamer/lib/common" -// SystemConnectedTM struct to hold system connected telemetry messages -type SystemConnectedTM struct { +// systemConnectedTM struct to hold system connected telemetry messages +type systemConnectedTM struct { Authority bool `json:"authority"` Chain string `json:"chain"` GenesisHash *common.Hash `json:"genesis_hash"` Implementation string `json:"implementation"` - Msg string `json:"msg"` Name string `json:"name"` NetworkID string `json:"network_id"` StartupTime string `json:"startup_time"` @@ -33,13 +32,12 @@ type SystemConnectedTM struct { // NewSystemConnectedTM function to create new System Connected Telemetry Message func NewSystemConnectedTM(authority bool, chain string, genesisHash *common.Hash, - implementation, name, networkID, startupTime, version string) *SystemConnectedTM { - return &SystemConnectedTM{ + implementation, name, networkID, startupTime, version string) *systemConnectedTM { + return &systemConnectedTM{ Authority: authority, Chain: chain, GenesisHash: genesisHash, Implementation: implementation, - Msg: "system.connected", Name: name, NetworkID: networkID, StartupTime: startupTime, @@ -47,6 +45,6 @@ func NewSystemConnectedTM(authority bool, chain string, genesisHash *common.Hash } } -func (tm *SystemConnectedTM) messageType() string { - return tm.Msg +func (tm *systemConnectedTM) messageType() string { + return systemConnectedMsg } diff --git a/dot/telemetry/system_interval.go b/dot/telemetry/system_interval.go index a44e99ea73..a608ab5a7a 100644 --- a/dot/telemetry/system_interval.go +++ b/dot/telemetry/system_interval.go @@ -22,11 +22,10 @@ import ( "github.com/ChainSafe/gossamer/lib/common" ) -// SystemIntervalTM struct to hold system interval telemetry messages -type SystemIntervalTM struct { +// systemIntervalTM struct to hold system interval telemetry messages +type systemIntervalTM struct { BandwidthDownload float64 `json:"bandwidth_download,omitempty"` BandwidthUpload float64 `json:"bandwidth_upload,omitempty"` - Msg string `json:"msg"` Peers int `json:"peers,omitempty"` BestHash *common.Hash `json:"best,omitempty"` BestHeight *big.Int `json:"height,omitempty"` @@ -37,20 +36,18 @@ type SystemIntervalTM struct { } // NewBandwidthTM function to create new Bandwidth Telemetry Message -func NewBandwidthTM(bandwidthDownload, bandwidthUpload float64, peers int) *SystemIntervalTM { - return &SystemIntervalTM{ +func NewBandwidthTM(bandwidthDownload, bandwidthUpload float64, peers int) *systemIntervalTM { + return &systemIntervalTM{ BandwidthDownload: bandwidthDownload, BandwidthUpload: bandwidthUpload, - Msg: "system.interval", Peers: peers, } } // NewBlockIntervalTM function to create new Block Interval Telemetry Message func NewBlockIntervalTM(beshHash *common.Hash, bestHeight *big.Int, finalisedHash *common.Hash, - finalisedHeight, txCount, usedStateCacheSize *big.Int) *SystemIntervalTM { - return &SystemIntervalTM{ - Msg: "system.interval", + finalisedHeight, txCount, usedStateCacheSize *big.Int) *systemIntervalTM { + return &systemIntervalTM{ BestHash: beshHash, BestHeight: bestHeight, FinalisedHash: finalisedHash, @@ -60,6 +57,6 @@ func NewBlockIntervalTM(beshHash *common.Hash, bestHeight *big.Int, finalisedHas } } -func (tm *SystemIntervalTM) messageType() string { - return tm.Msg +func (tm *systemIntervalTM) messageType() string { + return systemIntervalMsg } diff --git a/dot/telemetry/telemetry.go b/dot/telemetry/telemetry.go index b3bfe1df62..2135925c68 100644 --- a/dot/telemetry/telemetry.go +++ b/dot/telemetry/telemetry.go @@ -27,6 +27,15 @@ import ( "github.com/gorilla/websocket" ) +// telemetry message types +const ( + notifyFinalizedMsg = "notify.finalized" + blockImportMsg = "block.import" + systemNetworkStateMsg = "system.network_state" + systemConnectedMsg = "system.connected" + systemIntervalMsg = "system.interval" +) + type telemetryConnection struct { wsconn *websocket.Conn verbosity int From 9f31279f67131e188030f9ad86fa767faefa7e97 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Tue, 19 Oct 2021 23:24:16 +0530 Subject: [PATCH 8/9] return Message interface instead of individual Message structs --- dot/telemetry/block_import.go | 2 +- dot/telemetry/network_state.go | 2 +- dot/telemetry/notify_finalized.go | 2 +- dot/telemetry/system_connected.go | 2 +- dot/telemetry/system_interval.go | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dot/telemetry/block_import.go b/dot/telemetry/block_import.go index 5e6d9278f1..8ea641f10a 100644 --- a/dot/telemetry/block_import.go +++ b/dot/telemetry/block_import.go @@ -30,7 +30,7 @@ type blockImportTM struct { } // NewBlockImportTM function to create new Block Import Telemetry Message -func NewBlockImportTM(bestHash *common.Hash, height *big.Int, origin string) *blockImportTM { +func NewBlockImportTM(bestHash *common.Hash, height *big.Int, origin string) Message { return &blockImportTM{ BestHash: bestHash, Height: height, diff --git a/dot/telemetry/network_state.go b/dot/telemetry/network_state.go index 21905df890..139e3b97ee 100644 --- a/dot/telemetry/network_state.go +++ b/dot/telemetry/network_state.go @@ -29,7 +29,7 @@ type networkStateTM struct { } // NewNetworkStateTM function to create new Network State Telemetry Message -func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) *networkStateTM { +func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) Message { netState := make(map[string]interface{}) netState["peerId"] = host.ID() hostAddrs := make([]string, 0, len(host.Addrs())) diff --git a/dot/telemetry/notify_finalized.go b/dot/telemetry/notify_finalized.go index fd08716741..aad89ffc7c 100644 --- a/dot/telemetry/notify_finalized.go +++ b/dot/telemetry/notify_finalized.go @@ -30,7 +30,7 @@ type notifyFinalizedTM struct { } // NewNotifyFinalizedTM gets a new NotifyFinalizedTM struct. -func NewNotifyFinalizedTM(best common.Hash, height string) *notifyFinalizedTM { +func NewNotifyFinalizedTM(best common.Hash, height string) Message { return ¬ifyFinalizedTM{ Best: best, Height: height, diff --git a/dot/telemetry/system_connected.go b/dot/telemetry/system_connected.go index 73ae95ee80..a299a952c2 100644 --- a/dot/telemetry/system_connected.go +++ b/dot/telemetry/system_connected.go @@ -32,7 +32,7 @@ type systemConnectedTM struct { // NewSystemConnectedTM function to create new System Connected Telemetry Message func NewSystemConnectedTM(authority bool, chain string, genesisHash *common.Hash, - implementation, name, networkID, startupTime, version string) *systemConnectedTM { + implementation, name, networkID, startupTime, version string) Message { return &systemConnectedTM{ Authority: authority, Chain: chain, diff --git a/dot/telemetry/system_interval.go b/dot/telemetry/system_interval.go index a608ab5a7a..05b96034b0 100644 --- a/dot/telemetry/system_interval.go +++ b/dot/telemetry/system_interval.go @@ -36,7 +36,7 @@ type systemIntervalTM struct { } // NewBandwidthTM function to create new Bandwidth Telemetry Message -func NewBandwidthTM(bandwidthDownload, bandwidthUpload float64, peers int) *systemIntervalTM { +func NewBandwidthTM(bandwidthDownload, bandwidthUpload float64, peers int) Message { return &systemIntervalTM{ BandwidthDownload: bandwidthDownload, BandwidthUpload: bandwidthUpload, @@ -46,7 +46,7 @@ func NewBandwidthTM(bandwidthDownload, bandwidthUpload float64, peers int) *syst // NewBlockIntervalTM function to create new Block Interval Telemetry Message func NewBlockIntervalTM(beshHash *common.Hash, bestHeight *big.Int, finalisedHash *common.Hash, - finalisedHeight, txCount, usedStateCacheSize *big.Int) *systemIntervalTM { + finalisedHeight, txCount, usedStateCacheSize *big.Int) Message { return &systemIntervalTM{ BestHash: beshHash, BestHeight: bestHeight, From cf1faccb44873a7ed35d40d7f89ea58ecb07763b Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Tue, 19 Oct 2021 23:58:45 +0530 Subject: [PATCH 9/9] Omit the reciever name when unused --- dot/telemetry/block_import.go | 2 +- dot/telemetry/network_state.go | 2 +- dot/telemetry/notify_finalized.go | 2 +- dot/telemetry/system_connected.go | 2 +- dot/telemetry/system_interval.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dot/telemetry/block_import.go b/dot/telemetry/block_import.go index 8ea641f10a..3b1a7250b8 100644 --- a/dot/telemetry/block_import.go +++ b/dot/telemetry/block_import.go @@ -38,6 +38,6 @@ func NewBlockImportTM(bestHash *common.Hash, height *big.Int, origin string) Mes } } -func (tm *blockImportTM) messageType() string { +func (blockImportTM) messageType() string { return blockImportMsg } diff --git a/dot/telemetry/network_state.go b/dot/telemetry/network_state.go index 139e3b97ee..c7be70246f 100644 --- a/dot/telemetry/network_state.go +++ b/dot/telemetry/network_state.go @@ -61,6 +61,6 @@ func NewNetworkStateTM(host libp2phost.Host, peerInfos []common.PeerInfo) Messag } } -func (tm *networkStateTM) messageType() string { +func (networkStateTM) messageType() string { return systemNetworkStateMsg } diff --git a/dot/telemetry/notify_finalized.go b/dot/telemetry/notify_finalized.go index aad89ffc7c..9f29c3df38 100644 --- a/dot/telemetry/notify_finalized.go +++ b/dot/telemetry/notify_finalized.go @@ -37,6 +37,6 @@ func NewNotifyFinalizedTM(best common.Hash, height string) Message { } } -func (tm *notifyFinalizedTM) messageType() string { +func (notifyFinalizedTM) messageType() string { return notifyFinalizedMsg } diff --git a/dot/telemetry/system_connected.go b/dot/telemetry/system_connected.go index a299a952c2..2bcff1a552 100644 --- a/dot/telemetry/system_connected.go +++ b/dot/telemetry/system_connected.go @@ -45,6 +45,6 @@ func NewSystemConnectedTM(authority bool, chain string, genesisHash *common.Hash } } -func (tm *systemConnectedTM) messageType() string { +func (systemConnectedTM) messageType() string { return systemConnectedMsg } diff --git a/dot/telemetry/system_interval.go b/dot/telemetry/system_interval.go index 05b96034b0..10149f8798 100644 --- a/dot/telemetry/system_interval.go +++ b/dot/telemetry/system_interval.go @@ -57,6 +57,6 @@ func NewBlockIntervalTM(beshHash *common.Hash, bestHeight *big.Int, finalisedHas } } -func (tm *systemIntervalTM) messageType() string { +func (systemIntervalTM) messageType() string { return systemIntervalMsg }