From 90f30ea4bf206272c1ef67037844f6d65110df05 Mon Sep 17 00:00:00 2001 From: Marketen Date: Mon, 28 Jul 2025 17:15:00 +0200 Subject: [PATCH 1/6] previous liveness --- .../services/dutieschecker_service.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/internal/application/services/dutieschecker_service.go b/internal/application/services/dutieschecker_service.go index 13e4c0d..78496c5 100644 --- a/internal/application/services/dutieschecker_service.go +++ b/internal/application/services/dutieschecker_service.go @@ -17,10 +17,13 @@ type DutiesChecker struct { PollInterval time.Duration lastJustifiedEpoch domain.Epoch - lastLivenessState *bool lastRunHadError bool SlashedNotified map[domain.ValidatorIndex]bool + + // Tracking previous states for notifications + previouslyAllLive bool + previouslyOffline bool } func (a *DutiesChecker) Run(ctx context.Context) { @@ -86,25 +89,28 @@ func (a *DutiesChecker) performChecks(ctx context.Context, justifiedEpoch domain // Debug print: show offline, online, and allLive status logger.Debug("Liveness check: offline=%v, online=%v, allLive=%v", offline, online, allLive) - if len(offline) > 0 && (a.lastLivenessState == nil || *a.lastLivenessState) { + // Check for the first condition: 1 or more validators offline when all were previously live + if len(offline) > 0 && a.previouslyAllLive { if notificationsEnabled[domain.ValidatorLiveness] { if err := a.Notifier.SendValidatorLivenessNot(offline, false); err != nil { logger.Warn("Error sending validator liveness notification: %v", err) } } - val := false - a.lastLivenessState = &val + a.previouslyOffline = true } - if allLive && (a.lastLivenessState == nil || !*a.lastLivenessState) { + + // Check for the second condition: all validators online after 1 or more were offline + if allLive && a.previouslyOffline { if notificationsEnabled[domain.ValidatorLiveness] { if err := a.Notifier.SendValidatorLivenessNot(indices, true); err != nil { logger.Warn("Error sending validator liveness notification: %v", err) } } - val := true - a.lastLivenessState = &val + a.previouslyAllLive = true + a.previouslyOffline = false } + // Check block proposals (successful or missed) proposed, missed, err := a.checkProposals(ctx, justifiedEpoch, indices) if err != nil { logger.Error("Error checking block proposals: %v", err) @@ -121,6 +127,7 @@ func (a *DutiesChecker) performChecks(ctx context.Context, justifiedEpoch domain } } + // Check for slashed validators slashed, err := a.Beacon.GetSlashedValidators(ctx, indices) if err != nil { logger.Error("Error fetching slashed validators: %v", err) From a7ed0c5c70564d4da2a1d85fd8494681a0488686 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Mon, 28 Jul 2025 17:34:24 +0200 Subject: [PATCH 2/6] fix notifier url --- internal/config/config_loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/config_loader.go b/internal/config/config_loader.go index 880f7f9..b2b083c 100644 --- a/internal/config/config_loader.go +++ b/internal/config/config_loader.go @@ -29,7 +29,7 @@ func LoadConfig() Config { beaconEndpoint := fmt.Sprintf("http://beacon-chain.%s.dncore.dappnode:3500", network) web3SignerEndpoint := fmt.Sprintf("http://web3signer.%s.dncore.dappnode:9000", network) dappmanagerEndpoint := "http://dappmanager.dappnode" - notifierEndpoint := "http://notifier.dappnode:8080" + notifierEndpoint := "http://notifier.notifications.dappnode:8080" brainEndpoint := fmt.Sprintf("http://brain.web3signer-%s.dappnode", network) // Allow override via environment variables From dcd93f137ed597bf1b766f5be08a50d36b6aff48 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Mon, 28 Jul 2025 17:43:44 +0200 Subject: [PATCH 3/6] add loggers --- internal/application/services/dutieschecker_service.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/application/services/dutieschecker_service.go b/internal/application/services/dutieschecker_service.go index 78496c5..1e4b108 100644 --- a/internal/application/services/dutieschecker_service.go +++ b/internal/application/services/dutieschecker_service.go @@ -92,6 +92,7 @@ func (a *DutiesChecker) performChecks(ctx context.Context, justifiedEpoch domain // Check for the first condition: 1 or more validators offline when all were previously live if len(offline) > 0 && a.previouslyAllLive { if notificationsEnabled[domain.ValidatorLiveness] { + logger.Debug("Sending notification for validators going offline: %v", offline) if err := a.Notifier.SendValidatorLivenessNot(offline, false); err != nil { logger.Warn("Error sending validator liveness notification: %v", err) } @@ -102,6 +103,7 @@ func (a *DutiesChecker) performChecks(ctx context.Context, justifiedEpoch domain // Check for the second condition: all validators online after 1 or more were offline if allLive && a.previouslyOffline { if notificationsEnabled[domain.ValidatorLiveness] { + logger.Debug("Sending notification for all validators back online: %v", indices) if err := a.Notifier.SendValidatorLivenessNot(indices, true); err != nil { logger.Warn("Error sending validator liveness notification: %v", err) } From 89f479a7e4919c3afe2b5c17eacee79abd5c7cdb Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Mon, 28 Jul 2025 17:48:17 +0200 Subject: [PATCH 4/6] add previous log --- internal/application/services/dutieschecker_service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/application/services/dutieschecker_service.go b/internal/application/services/dutieschecker_service.go index 1e4b108..f75646d 100644 --- a/internal/application/services/dutieschecker_service.go +++ b/internal/application/services/dutieschecker_service.go @@ -88,6 +88,7 @@ func (a *DutiesChecker) performChecks(ctx context.Context, justifiedEpoch domain // Debug print: show offline, online, and allLive status logger.Debug("Liveness check: offline=%v, online=%v, allLive=%v", offline, online, allLive) + logger.Debug("Previously all live: %v, previously offline: %v", a.previouslyAllLive, a.previouslyOffline) // Check for the first condition: 1 or more validators offline when all were previously live if len(offline) > 0 && a.previouslyAllLive { From fc11a356e154cea440fb84549469a81f1f9fcc92 Mon Sep 17 00:00:00 2001 From: Marketen Date: Tue, 29 Jul 2025 09:57:58 +0200 Subject: [PATCH 5/6] correct previous allive/ofline --- cmd/main.go | 14 ++++++++------ .../services/dutieschecker_service.go | 17 +++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 3606000..55b93c9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -55,12 +55,14 @@ func main() { // Start the duties checker service in a goroutine dutiesChecker := &services.DutiesChecker{ - Beacon: beacon, - Brain: brain, - Notifier: notifier, - Dappmanager: dappmanager, - PollInterval: 1 * time.Minute, - SlashedNotified: make(map[domain.ValidatorIndex]bool), + Beacon: beacon, + Brain: brain, + Notifier: notifier, + Dappmanager: dappmanager, + PollInterval: 1 * time.Minute, + SlashedNotified: make(map[domain.ValidatorIndex]bool), + PreviouslyAllLive: true, // assume all validators were live at start + PreviouslyOffline: false, } wg.Add(1) go func() { diff --git a/internal/application/services/dutieschecker_service.go b/internal/application/services/dutieschecker_service.go index f75646d..1970a6e 100644 --- a/internal/application/services/dutieschecker_service.go +++ b/internal/application/services/dutieschecker_service.go @@ -22,8 +22,8 @@ type DutiesChecker struct { SlashedNotified map[domain.ValidatorIndex]bool // Tracking previous states for notifications - previouslyAllLive bool - previouslyOffline bool + PreviouslyAllLive bool + PreviouslyOffline bool } func (a *DutiesChecker) Run(ctx context.Context) { @@ -88,29 +88,30 @@ func (a *DutiesChecker) performChecks(ctx context.Context, justifiedEpoch domain // Debug print: show offline, online, and allLive status logger.Debug("Liveness check: offline=%v, online=%v, allLive=%v", offline, online, allLive) - logger.Debug("Previously all live: %v, previously offline: %v", a.previouslyAllLive, a.previouslyOffline) + logger.Debug("Previously all live: %v, previously offline: %v", a.PreviouslyAllLive, a.PreviouslyOffline) // Check for the first condition: 1 or more validators offline when all were previously live - if len(offline) > 0 && a.previouslyAllLive { + if len(offline) > 0 && a.PreviouslyAllLive { if notificationsEnabled[domain.ValidatorLiveness] { logger.Debug("Sending notification for validators going offline: %v", offline) if err := a.Notifier.SendValidatorLivenessNot(offline, false); err != nil { logger.Warn("Error sending validator liveness notification: %v", err) } } - a.previouslyOffline = true + a.PreviouslyAllLive = false + a.PreviouslyOffline = true } // Check for the second condition: all validators online after 1 or more were offline - if allLive && a.previouslyOffline { + if allLive && a.PreviouslyOffline { if notificationsEnabled[domain.ValidatorLiveness] { logger.Debug("Sending notification for all validators back online: %v", indices) if err := a.Notifier.SendValidatorLivenessNot(indices, true); err != nil { logger.Warn("Error sending validator liveness notification: %v", err) } } - a.previouslyAllLive = true - a.previouslyOffline = false + a.PreviouslyAllLive = true + a.PreviouslyOffline = false } // Check block proposals (successful or missed) From c24968a3bc97784a735a469cec4b64357fa73f40 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Tue, 29 Jul 2025 10:56:58 +0200 Subject: [PATCH 6/6] improve notifier messages --- internal/adapters/notifier/notifier.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/adapters/notifier/notifier.go b/internal/adapters/notifier/notifier.go index 5b74d4d..9188327 100644 --- a/internal/adapters/notifier/notifier.go +++ b/internal/adapters/notifier/notifier.go @@ -121,8 +121,8 @@ func (n *Notifier) SendValidatorLivenessNot(validators []domain.ValidatorIndex, } } if live { - title = fmt.Sprintf("Validator(s) Online: %s", indexesToString(validators)) - body = fmt.Sprintf("✅ Validator(s) %s are back online on %s.", indexesToString(validators), n.Network) + title = fmt.Sprintf("All validators back online (%d)", len(validators)) + body = fmt.Sprintf("✅ All validators are back online on %s (%d).", n.Network, len(validators)) priority = Info status = Resolved isBanner = false @@ -215,7 +215,12 @@ func (n *Notifier) SendBlockProposalNot(validators []domain.ValidatorIndex, epoc // Helper to join validator indexes as comma-separated string func indexesToString(indexes []domain.ValidatorIndex) string { var s []string - for _, idx := range indexes { + max := 10 + for i, idx := range indexes { + if i == max { + s = append(s, "...") + break + } s = append(s, fmt.Sprintf("%d", idx)) } return strings.Join(s, ",")