From b30bcf980e82847c729e6602fffed5fa9a55e9da Mon Sep 17 00:00:00 2001 From: Drew Erny Date: Fri, 22 Jun 2018 12:03:17 -0700 Subject: [PATCH] Add metrics for store object types Adds metrics for most of the user-facing store object types. Allows the user to keep track of how many objects are in the object store at any given time. Cherry picks #2673, applies cleanly (cherry picked from commit b67abf85fcae39716863aec108144d9408ee9fbe) Signed-off-by: Drew Erny --- api/objects.pb.go | 261 +++++++++++++++++++++ api/storeobject.go | 15 ++ manager/metrics/collector.go | 211 ++++++++++++++--- protobuf/plugin/storeobject/storeobject.go | 51 ++++ 4 files changed, 510 insertions(+), 28 deletions(-) diff --git a/api/objects.pb.go b/api/objects.pb.go index e9c2438502..a02c710d97 100644 --- a/api/objects.pb.go +++ b/api/objects.pb.go @@ -1957,6 +1957,10 @@ func sozObjects(x uint64) (n int) { type NodeCheckFunc func(t1, t2 *Node) bool +type EventNode interface { + IsEventNode() bool +} + type EventCreateNode struct { Node *Node Checks []NodeCheckFunc @@ -1976,6 +1980,14 @@ func (e EventCreateNode) Matches(apiEvent github_com_docker_go_events.Event) boo return true } +func (e EventCreateNode) IsEventCreate() bool { + return true +} + +func (e EventCreateNode) IsEventNode() bool { + return true +} + type EventUpdateNode struct { Node *Node OldNode *Node @@ -1996,6 +2008,14 @@ func (e EventUpdateNode) Matches(apiEvent github_com_docker_go_events.Event) boo return true } +func (e EventUpdateNode) IsEventUpdate() bool { + return true +} + +func (e EventUpdateNode) IsEventNode() bool { + return true +} + type EventDeleteNode struct { Node *Node Checks []NodeCheckFunc @@ -2014,6 +2034,15 @@ func (e EventDeleteNode) Matches(apiEvent github_com_docker_go_events.Event) boo } return true } + +func (e EventDeleteNode) IsEventDelete() bool { + return true +} + +func (e EventDeleteNode) IsEventNode() bool { + return true +} + func (m *Node) CopyStoreObject() StoreObject { return m.Copy() } @@ -2204,6 +2233,10 @@ func (indexer NodeCustomIndexer) FromObject(obj interface{}) (bool, [][]byte, er type ServiceCheckFunc func(t1, t2 *Service) bool +type EventService interface { + IsEventService() bool +} + type EventCreateService struct { Service *Service Checks []ServiceCheckFunc @@ -2223,6 +2256,14 @@ func (e EventCreateService) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventCreateService) IsEventCreate() bool { + return true +} + +func (e EventCreateService) IsEventService() bool { + return true +} + type EventUpdateService struct { Service *Service OldService *Service @@ -2243,6 +2284,14 @@ func (e EventUpdateService) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventUpdateService) IsEventUpdate() bool { + return true +} + +func (e EventUpdateService) IsEventService() bool { + return true +} + type EventDeleteService struct { Service *Service Checks []ServiceCheckFunc @@ -2261,6 +2310,15 @@ func (e EventDeleteService) Matches(apiEvent github_com_docker_go_events.Event) } return true } + +func (e EventDeleteService) IsEventDelete() bool { + return true +} + +func (e EventDeleteService) IsEventService() bool { + return true +} + func (m *Service) CopyStoreObject() StoreObject { return m.Copy() } @@ -2421,6 +2479,10 @@ func (indexer ServiceCustomIndexer) FromObject(obj interface{}) (bool, [][]byte, type TaskCheckFunc func(t1, t2 *Task) bool +type EventTask interface { + IsEventTask() bool +} + type EventCreateTask struct { Task *Task Checks []TaskCheckFunc @@ -2440,6 +2502,14 @@ func (e EventCreateTask) Matches(apiEvent github_com_docker_go_events.Event) boo return true } +func (e EventCreateTask) IsEventCreate() bool { + return true +} + +func (e EventCreateTask) IsEventTask() bool { + return true +} + type EventUpdateTask struct { Task *Task OldTask *Task @@ -2460,6 +2530,14 @@ func (e EventUpdateTask) Matches(apiEvent github_com_docker_go_events.Event) boo return true } +func (e EventUpdateTask) IsEventUpdate() bool { + return true +} + +func (e EventUpdateTask) IsEventTask() bool { + return true +} + type EventDeleteTask struct { Task *Task Checks []TaskCheckFunc @@ -2478,6 +2556,15 @@ func (e EventDeleteTask) Matches(apiEvent github_com_docker_go_events.Event) boo } return true } + +func (e EventDeleteTask) IsEventDelete() bool { + return true +} + +func (e EventDeleteTask) IsEventTask() bool { + return true +} + func (m *Task) CopyStoreObject() StoreObject { return m.Copy() } @@ -2681,6 +2768,10 @@ func (indexer TaskCustomIndexer) FromObject(obj interface{}) (bool, [][]byte, er type NetworkCheckFunc func(t1, t2 *Network) bool +type EventNetwork interface { + IsEventNetwork() bool +} + type EventCreateNetwork struct { Network *Network Checks []NetworkCheckFunc @@ -2700,6 +2791,14 @@ func (e EventCreateNetwork) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventCreateNetwork) IsEventCreate() bool { + return true +} + +func (e EventCreateNetwork) IsEventNetwork() bool { + return true +} + type EventUpdateNetwork struct { Network *Network OldNetwork *Network @@ -2720,6 +2819,14 @@ func (e EventUpdateNetwork) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventUpdateNetwork) IsEventUpdate() bool { + return true +} + +func (e EventUpdateNetwork) IsEventNetwork() bool { + return true +} + type EventDeleteNetwork struct { Network *Network Checks []NetworkCheckFunc @@ -2738,6 +2845,15 @@ func (e EventDeleteNetwork) Matches(apiEvent github_com_docker_go_events.Event) } return true } + +func (e EventDeleteNetwork) IsEventDelete() bool { + return true +} + +func (e EventDeleteNetwork) IsEventNetwork() bool { + return true +} + func (m *Network) CopyStoreObject() StoreObject { return m.Copy() } @@ -2898,6 +3014,10 @@ func (indexer NetworkCustomIndexer) FromObject(obj interface{}) (bool, [][]byte, type ClusterCheckFunc func(t1, t2 *Cluster) bool +type EventCluster interface { + IsEventCluster() bool +} + type EventCreateCluster struct { Cluster *Cluster Checks []ClusterCheckFunc @@ -2917,6 +3037,14 @@ func (e EventCreateCluster) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventCreateCluster) IsEventCreate() bool { + return true +} + +func (e EventCreateCluster) IsEventCluster() bool { + return true +} + type EventUpdateCluster struct { Cluster *Cluster OldCluster *Cluster @@ -2937,6 +3065,14 @@ func (e EventUpdateCluster) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventUpdateCluster) IsEventUpdate() bool { + return true +} + +func (e EventUpdateCluster) IsEventCluster() bool { + return true +} + type EventDeleteCluster struct { Cluster *Cluster Checks []ClusterCheckFunc @@ -2955,6 +3091,15 @@ func (e EventDeleteCluster) Matches(apiEvent github_com_docker_go_events.Event) } return true } + +func (e EventDeleteCluster) IsEventDelete() bool { + return true +} + +func (e EventDeleteCluster) IsEventCluster() bool { + return true +} + func (m *Cluster) CopyStoreObject() StoreObject { return m.Copy() } @@ -3115,6 +3260,10 @@ func (indexer ClusterCustomIndexer) FromObject(obj interface{}) (bool, [][]byte, type SecretCheckFunc func(t1, t2 *Secret) bool +type EventSecret interface { + IsEventSecret() bool +} + type EventCreateSecret struct { Secret *Secret Checks []SecretCheckFunc @@ -3134,6 +3283,14 @@ func (e EventCreateSecret) Matches(apiEvent github_com_docker_go_events.Event) b return true } +func (e EventCreateSecret) IsEventCreate() bool { + return true +} + +func (e EventCreateSecret) IsEventSecret() bool { + return true +} + type EventUpdateSecret struct { Secret *Secret OldSecret *Secret @@ -3154,6 +3311,14 @@ func (e EventUpdateSecret) Matches(apiEvent github_com_docker_go_events.Event) b return true } +func (e EventUpdateSecret) IsEventUpdate() bool { + return true +} + +func (e EventUpdateSecret) IsEventSecret() bool { + return true +} + type EventDeleteSecret struct { Secret *Secret Checks []SecretCheckFunc @@ -3172,6 +3337,15 @@ func (e EventDeleteSecret) Matches(apiEvent github_com_docker_go_events.Event) b } return true } + +func (e EventDeleteSecret) IsEventDelete() bool { + return true +} + +func (e EventDeleteSecret) IsEventSecret() bool { + return true +} + func (m *Secret) CopyStoreObject() StoreObject { return m.Copy() } @@ -3332,6 +3506,10 @@ func (indexer SecretCustomIndexer) FromObject(obj interface{}) (bool, [][]byte, type ConfigCheckFunc func(t1, t2 *Config) bool +type EventConfig interface { + IsEventConfig() bool +} + type EventCreateConfig struct { Config *Config Checks []ConfigCheckFunc @@ -3351,6 +3529,14 @@ func (e EventCreateConfig) Matches(apiEvent github_com_docker_go_events.Event) b return true } +func (e EventCreateConfig) IsEventCreate() bool { + return true +} + +func (e EventCreateConfig) IsEventConfig() bool { + return true +} + type EventUpdateConfig struct { Config *Config OldConfig *Config @@ -3371,6 +3557,14 @@ func (e EventUpdateConfig) Matches(apiEvent github_com_docker_go_events.Event) b return true } +func (e EventUpdateConfig) IsEventUpdate() bool { + return true +} + +func (e EventUpdateConfig) IsEventConfig() bool { + return true +} + type EventDeleteConfig struct { Config *Config Checks []ConfigCheckFunc @@ -3389,6 +3583,15 @@ func (e EventDeleteConfig) Matches(apiEvent github_com_docker_go_events.Event) b } return true } + +func (e EventDeleteConfig) IsEventDelete() bool { + return true +} + +func (e EventDeleteConfig) IsEventConfig() bool { + return true +} + func (m *Config) CopyStoreObject() StoreObject { return m.Copy() } @@ -3549,6 +3752,10 @@ func (indexer ConfigCustomIndexer) FromObject(obj interface{}) (bool, [][]byte, type ResourceCheckFunc func(t1, t2 *Resource) bool +type EventResource interface { + IsEventResource() bool +} + type EventCreateResource struct { Resource *Resource Checks []ResourceCheckFunc @@ -3568,6 +3775,14 @@ func (e EventCreateResource) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventCreateResource) IsEventCreate() bool { + return true +} + +func (e EventCreateResource) IsEventResource() bool { + return true +} + type EventUpdateResource struct { Resource *Resource OldResource *Resource @@ -3588,6 +3803,14 @@ func (e EventUpdateResource) Matches(apiEvent github_com_docker_go_events.Event) return true } +func (e EventUpdateResource) IsEventUpdate() bool { + return true +} + +func (e EventUpdateResource) IsEventResource() bool { + return true +} + type EventDeleteResource struct { Resource *Resource Checks []ResourceCheckFunc @@ -3606,6 +3829,15 @@ func (e EventDeleteResource) Matches(apiEvent github_com_docker_go_events.Event) } return true } + +func (e EventDeleteResource) IsEventDelete() bool { + return true +} + +func (e EventDeleteResource) IsEventResource() bool { + return true +} + func (m *Resource) CopyStoreObject() StoreObject { return m.Copy() } @@ -3772,6 +4004,10 @@ func (indexer ResourceCustomIndexer) FromObject(obj interface{}) (bool, [][]byte type ExtensionCheckFunc func(t1, t2 *Extension) bool +type EventExtension interface { + IsEventExtension() bool +} + type EventCreateExtension struct { Extension *Extension Checks []ExtensionCheckFunc @@ -3791,6 +4027,14 @@ func (e EventCreateExtension) Matches(apiEvent github_com_docker_go_events.Event return true } +func (e EventCreateExtension) IsEventCreate() bool { + return true +} + +func (e EventCreateExtension) IsEventExtension() bool { + return true +} + type EventUpdateExtension struct { Extension *Extension OldExtension *Extension @@ -3811,6 +4055,14 @@ func (e EventUpdateExtension) Matches(apiEvent github_com_docker_go_events.Event return true } +func (e EventUpdateExtension) IsEventUpdate() bool { + return true +} + +func (e EventUpdateExtension) IsEventExtension() bool { + return true +} + type EventDeleteExtension struct { Extension *Extension Checks []ExtensionCheckFunc @@ -3829,6 +4081,15 @@ func (e EventDeleteExtension) Matches(apiEvent github_com_docker_go_events.Event } return true } + +func (e EventDeleteExtension) IsEventDelete() bool { + return true +} + +func (e EventDeleteExtension) IsEventExtension() bool { + return true +} + func (m *Extension) CopyStoreObject() StoreObject { return m.Copy() } diff --git a/api/storeobject.go b/api/storeobject.go index 48b50b72dd..d140fa3e0c 100644 --- a/api/storeobject.go +++ b/api/storeobject.go @@ -38,6 +38,21 @@ type Event interface { Matches(events.Event) bool } +// EventCreate is an interface implemented by every creation event type +type EventCreate interface { + IsEventCreate() bool +} + +// EventUpdate is an interface impelemented by every update event type +type EventUpdate interface { + IsEventUpdate() bool +} + +// EventDelete is an interface implemented by every delete event type +type EventDelete interface { + IsEventDelete() +} + func customIndexer(kind string, annotations *Annotations) (bool, [][]byte, error) { var converted [][]byte diff --git a/manager/metrics/collector.go b/manager/metrics/collector.go index d5adcb8306..384743707d 100644 --- a/manager/metrics/collector.go +++ b/manager/metrics/collector.go @@ -5,21 +5,36 @@ import ( "strings" + "github.com/docker/go-events" metrics "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state/store" ) var ( - ns = metrics.NewNamespace("swarm", "manager", nil) + ns = metrics.NewNamespace("swarm", "manager", nil) + + // counts of the various objects in swarmkit nodesMetric metrics.LabeledGauge + tasksMetric metrics.LabeledGauge + + // none of these objects have state, so they're just regular gauges + servicesMetric metrics.Gauge + networksMetric metrics.Gauge + secretsMetric metrics.Gauge + configsMetric metrics.Gauge ) func init() { nodesMetric = ns.NewLabeledGauge("nodes", "The number of nodes", "", "state") - for _, state := range api.NodeStatus_State_name { - nodesMetric.WithValues(strings.ToLower(state)).Set(0) - } + tasksMetric = ns.NewLabeledGauge("tasks", "The number of tasks in the cluster object store", metrics.Total, "state") + servicesMetric = ns.NewGauge("services", "The number of services in the cluster object store", metrics.Total) + networksMetric = ns.NewGauge("networks", "The number of networks in the cluster object store", metrics.Total) + secretsMetric = ns.NewGauge("secrets", "The number of secrets in the cluster object store", metrics.Total) + configsMetric = ns.NewGauge("configs", "The number of configs in the cluster object store", metrics.Total) + + resetMetrics() + metrics.Register(ns) } @@ -42,20 +57,6 @@ func NewCollector(store *store.MemoryStore) *Collector { } } -func (c *Collector) updateNodeState(prevNode, newNode *api.Node) { - // Skip updates if nothing changed. - if prevNode != nil && newNode != nil && prevNode.Status.State == newNode.Status.State { - return - } - - if prevNode != nil { - nodesMetric.WithValues(strings.ToLower(prevNode.Status.State.String())).Dec(1) - } - if newNode != nil { - nodesMetric.WithValues(strings.ToLower(newNode.Status.State.String())).Inc(1) - } -} - // Run contains the collector event loop func (c *Collector) Run(ctx context.Context) error { defer close(c.doneChan) @@ -65,9 +66,46 @@ func (c *Collector) Run(ctx context.Context) error { if err != nil { return err } - for _, node := range nodes { - c.updateNodeState(nil, node) + tasks, err := store.FindTasks(readTx, store.All) + if err != nil { + return err } + services, err := store.FindServices(readTx, store.All) + if err != nil { + return err + } + networks, err := store.FindNetworks(readTx, store.All) + if err != nil { + return err + } + secrets, err := store.FindSecrets(readTx, store.All) + if err != nil { + return err + } + configs, err := store.FindConfigs(readTx, store.All) + if err != nil { + return err + } + + for _, obj := range nodes { + c.handleEvent(obj.EventCreate()) + } + for _, obj := range tasks { + c.handleEvent(obj.EventCreate()) + } + for _, obj := range services { + c.handleEvent(obj.EventCreate()) + } + for _, obj := range networks { + c.handleEvent(obj.EventCreate()) + } + for _, obj := range secrets { + c.handleEvent(obj.EventCreate()) + } + for _, obj := range configs { + c.handleEvent(obj.EventCreate()) + } + return nil }) if err != nil { @@ -78,14 +116,7 @@ func (c *Collector) Run(ctx context.Context) error { for { select { case event := <-watcher: - switch v := event.(type) { - case api.EventCreateNode: - c.updateNodeState(nil, v.Node) - case api.EventUpdateNode: - c.updateNodeState(v.OldNode, v.Node) - case api.EventDeleteNode: - c.updateNodeState(v.Node, nil) - } + c.handleEvent(event) case <-c.stopChan: return nil } @@ -98,7 +129,131 @@ func (c *Collector) Stop() { <-c.doneChan // Clean the metrics on exit. + resetMetrics() +} + +// resetMetrics resets all metrics to their default (base) value +func resetMetrics() { for _, state := range api.NodeStatus_State_name { nodesMetric.WithValues(strings.ToLower(state)).Set(0) } + for _, state := range api.TaskState_name { + tasksMetric.WithValues(strings.ToLower(state)).Set(0) + } + servicesMetric.Set(0) + networksMetric.Set(0) + secretsMetric.Set(0) + configsMetric.Set(0) + +} + +// handleEvent handles a single incoming cluster event. +func (c *Collector) handleEvent(event events.Event) { + switch event.(type) { + case api.EventNode: + c.handleNodeEvent(event) + case api.EventTask: + c.handleTaskEvent(event) + case api.EventService: + c.handleServiceEvent(event) + case api.EventNetwork: + c.handleNetworkEvent(event) + case api.EventSecret: + c.handleSecretsEvent(event) + case api.EventConfig: + c.handleConfigsEvent(event) + } +} + +func (c *Collector) handleNodeEvent(event events.Event) { + var prevNode, newNode *api.Node + + switch v := event.(type) { + case api.EventCreateNode: + prevNode, newNode = nil, v.Node + case api.EventUpdateNode: + prevNode, newNode = v.OldNode, v.Node + case api.EventDeleteNode: + prevNode, newNode = v.Node, nil + } + + // Skip updates if nothing changed. + if prevNode != nil && newNode != nil && prevNode.Status.State == newNode.Status.State { + return + } + + if prevNode != nil { + nodesMetric.WithValues(strings.ToLower(prevNode.Status.State.String())).Dec(1) + } + if newNode != nil { + nodesMetric.WithValues(strings.ToLower(newNode.Status.State.String())).Inc(1) + } + return +} + +func (c *Collector) handleTaskEvent(event events.Event) { + var prevTask, newTask *api.Task + + switch v := event.(type) { + case api.EventCreateTask: + prevTask, newTask = nil, v.Task + case api.EventUpdateTask: + prevTask, newTask = v.OldTask, v.Task + case api.EventDeleteTask: + prevTask, newTask = v.Task, nil + } + + // Skip updates if nothing changed. + if prevTask != nil && newTask != nil && prevTask.Status.State == newTask.Status.State { + return + } + + if prevTask != nil { + tasksMetric.WithValues( + strings.ToLower(prevTask.Status.State.String()), + ).Dec(1) + } + if newTask != nil { + tasksMetric.WithValues( + strings.ToLower(newTask.Status.State.String()), + ).Inc(1) + } + + return +} + +func (c *Collector) handleServiceEvent(event events.Event) { + switch event.(type) { + case api.EventCreateService: + servicesMetric.Inc(1) + case api.EventDeleteService: + servicesMetric.Dec(1) + } +} + +func (c *Collector) handleNetworkEvent(event events.Event) { + switch event.(type) { + case api.EventCreateNetwork: + networksMetric.Inc(1) + case api.EventDeleteNetwork: + networksMetric.Dec(1) + } +} + +func (c *Collector) handleSecretsEvent(event events.Event) { + switch event.(type) { + case api.EventCreateSecret: + secretsMetric.Inc(1) + case api.EventDeleteSecret: + secretsMetric.Dec(1) + } +} + +func (c *Collector) handleConfigsEvent(event events.Event) { + switch event.(type) { + case api.EventCreateConfig: + configsMetric.Inc(1) + case api.EventDeleteConfig: + configsMetric.Dec(1) + } } diff --git a/protobuf/plugin/storeobject/storeobject.go b/protobuf/plugin/storeobject/storeobject.go index 225802afb0..dcfe600fef 100644 --- a/protobuf/plugin/storeobject/storeobject.go +++ b/protobuf/plugin/storeobject/storeobject.go @@ -43,6 +43,40 @@ func (d *storeObjectGen) genMsgStoreObject(m *generator.Descriptor, storeObject d.P("type ", ccTypeName, "CheckFunc func(t1, t2 *", ccTypeName, ") bool") d.P() + // generate the event object type interface for this type + // event types implement some empty interfaces, for ease of use, like such: + // + // type EventCreate interface { + // IsEventCreatet() bool + // } + // + // type EventNode interface { + // IsEventNode() bool + // } + // + // then, each event has the corresponding interfaces implemented for its + // type. for example: + // + // func (e EventCreateNode) IsEventCreate() bool { + // return true + // } + // + // func (e EventCreateNode) IsEventNode() bool { + // return true + // } + // + // this lets the user filter events based on their interface type. + // note that the event type for each object type needs to be generated for + // each object. the event change type (Create/Update/Delete) is + // hand-written in the storeobject.go file because they are only needed + // once. + d.P("type Event", ccTypeName, " interface {") + d.In() + d.P("IsEvent", ccTypeName, "() bool") + d.Out() + d.P("}") + d.P() + for _, event := range []string{"Create", "Update", "Delete"} { d.P("type Event", event, ccTypeName, " struct {") d.In() @@ -75,6 +109,23 @@ func (d *storeObjectGen) genMsgStoreObject(m *generator.Descriptor, storeObject d.P("return true") d.Out() d.P("}") + d.P() + + // implement event change type interface (IsEventCreate) + d.P("func (e Event", event, ccTypeName, ") IsEvent", event, "() bool {") + d.In() + d.P("return true") + d.Out() + d.P("}") + d.P() + + // implement event object type interface (IsEventNode) + d.P("func (e Event", event, ccTypeName, ") IsEvent", ccTypeName, "() bool {") + d.In() + d.P("return true") + d.Out() + d.P("}") + d.P() } // Generate methods for this type