From 0f4cdbda1c8ff5c40f73e216c8dfea2cc3ba1d04 Mon Sep 17 00:00:00 2001 From: Juan Bran Date: Mon, 4 Jan 2021 16:56:21 -0800 Subject: [PATCH 1/4] Adding protocols collector Signed-off-by: Juan Bran --- README.md | 1 + collector/fixtures/e2e-output.txt | 37 ++++++ collector/fixtures/proc/net/protocols | 14 +++ collector/protocols_linux.go | 170 ++++++++++++++++++++++++++ 4 files changed, 222 insertions(+) create mode 100644 collector/fixtures/proc/net/protocols create mode 100644 collector/protocols_linux.go diff --git a/README.md b/README.md index 9405fa3e2a..dcd44fcf19 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ nfs | Exposes NFS client statistics from `/proc/net/rpc/nfs`. This is the same i nfsd | Exposes NFS kernel server statistics from `/proc/net/rpc/nfsd`. This is the same information as `nfsstat -s`. | Linux powersupplyclass | Exposes Power Supply statistics from `/sys/class/power_supply` | Linux pressure | Exposes pressure stall statistics from `/proc/pressure/`. | Linux (kernel 4.20+ and/or [CONFIG\_PSI](https://www.kernel.org/doc/html/latest/accounting/psi.html)) +protocols | Exposes protocol statistics from `/proc/net/protocols`. | Linux rapl | Exposes various statistics from `/sys/class/powercap`. | Linux schedstat | Exposes task scheduler statistics from `/proc/schedstat`. | Linux sockstat | Exposes various statistics from `/proc/net/sockstat`. | Linux diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 1f57032a9d..1c352372a6 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -2620,6 +2620,42 @@ node_procs_blocked 0 # HELP node_procs_running Number of processes in runnable state. # TYPE node_procs_running gauge node_procs_running 2 +# HELP node_protocols_tcp_maxhdr Max header size for tcp +# TYPE node_protocols_tcp_maxhdr gauge +node_protocols_tcp_maxhdr 320 +# HELP node_protocols_tcp_memory Total number of 4KB pages allocated by tcp +# TYPE node_protocols_tcp_memory gauge +node_protocols_tcp_memory 1.225378e+06 +# HELP node_protocols_tcp_pressure Indicates whether tcp is experiencing memory pressure; 1 = true, 0 = false, -1 = not implemented +# TYPE node_protocols_tcp_pressure gauge +node_protocols_tcp_pressure 1 +# HELP node_protocols_tcp_size The size, in bytes, of tcp +# TYPE node_protocols_tcp_size gauge +node_protocols_tcp_size 1984 +# HELP node_protocols_tcp_slab Bool indicating whether tcp is allocated from SLAB +# TYPE node_protocols_tcp_slab gauge +node_protocols_tcp_slab 1 +# HELP node_protocols_tcp_sockets Number of sockets in use by tcp +# TYPE node_protocols_tcp_sockets gauge +node_protocols_tcp_sockets 93064 +# HELP node_protocols_tcpv6_maxhdr Max header size for tcpv6 +# TYPE node_protocols_tcpv6_maxhdr gauge +node_protocols_tcpv6_maxhdr 320 +# HELP node_protocols_tcpv6_memory Total number of 4KB pages allocated by tcpv6 +# TYPE node_protocols_tcpv6_memory gauge +node_protocols_tcpv6_memory 1.225378e+06 +# HELP node_protocols_tcpv6_pressure Indicates whether tcpv6 is experiencing memory pressure; 1 = true, 0 = false, -1 = not implemented +# TYPE node_protocols_tcpv6_pressure gauge +node_protocols_tcpv6_pressure 0 +# HELP node_protocols_tcpv6_size The size, in bytes, of tcpv6 +# TYPE node_protocols_tcpv6_size gauge +node_protocols_tcpv6_size 2144 +# HELP node_protocols_tcpv6_slab Bool indicating whether tcpv6 is allocated from SLAB +# TYPE node_protocols_tcpv6_slab gauge +node_protocols_tcpv6_slab 1 +# HELP node_protocols_tcpv6_sockets Number of sockets in use by tcpv6 +# TYPE node_protocols_tcpv6_sockets gauge +node_protocols_tcpv6_sockets 1937 # HELP node_qdisc_backlog Number of bytes currently in queue to be sent. # TYPE node_qdisc_backlog gauge node_qdisc_backlog{device="eth0",kind="pfifo_fast"} 0 @@ -2701,6 +2737,7 @@ node_scrape_collector_success{collector="nfsd"} 1 node_scrape_collector_success{collector="powersupplyclass"} 1 node_scrape_collector_success{collector="pressure"} 1 node_scrape_collector_success{collector="processes"} 1 +node_scrape_collector_success{collector="protocols"} 1 node_scrape_collector_success{collector="qdisc"} 1 node_scrape_collector_success{collector="rapl"} 1 node_scrape_collector_success{collector="schedstat"} 1 diff --git a/collector/fixtures/proc/net/protocols b/collector/fixtures/proc/net/protocols new file mode 100644 index 0000000000..bd09331412 --- /dev/null +++ b/collector/fixtures/proc/net/protocols @@ -0,0 +1,14 @@ +protocol size sockets memory press maxhdr slab module cl co di ac io in de sh ss gs se re sp bi br ha uh gp em +PACKET 1344 2 -1 NI 0 no kernel n n n n n n n n n n n n n n n n n n n +PINGv6 1112 0 -1 NI 0 yes kernel y y y n n y n n y y y y n y y y y y n +RAWv6 1112 1 -1 NI 0 yes kernel y y y n y y y n y y y y n y y y y n n +UDPLITEv6 1216 0 57 NI 0 yes kernel y y y n y y y n y y y y n n n y y y n +UDPv6 1216 10 57 NI 0 yes kernel y y y n y y y n y y y y n n n y y y n +TCPv6 2144 1937 1225378 no 320 yes kernel y y y y y y y y y y y y y n y y y y y +UNIX 1024 120 -1 NI 0 yes kernel n n n n n n n n n n n n n n n n n n n +UDP-Lite 1024 0 57 NI 0 yes kernel y y y n y y y n y y y y y n n y y y n +PING 904 0 -1 NI 0 yes kernel y y y n n y n n y y y y n y y y y y n +RAW 912 0 -1 NI 0 yes kernel y y y n y y y n y y y y n y y y y n n +UDP 1024 73 57 NI 0 yes kernel y y y n y y y n y y y y y n n y y y n +TCP 1984 93064 1225378 yes 320 yes kernel y y y y y y y y y y y y y n y y y y y +NETLINK 1040 16 -1 NI 0 no kernel n n n n n n n n n n n n n n n n n n n diff --git a/collector/protocols_linux.go b/collector/protocols_linux.go new file mode 100644 index 0000000000..1a7b3baaa1 --- /dev/null +++ b/collector/protocols_linux.go @@ -0,0 +1,170 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "fmt" + "regexp" + "strings" + + "github.com/go-kit/kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" + "gopkg.in/alecthomas/kingpin.v2" +) + +const ( + subsystem = "protocols" +) + +var ( + netProtocolFilter = kingpin.Flag("collector.protocols.filter", "Regex of protocols to return for the collector.").Default("^tcp.*").String() +) + +type protocolsCollector struct { + fs procfs.FS + logger log.Logger +} + +func init() { + registerCollector(subsystem, defaultEnabled, NewProtocolsCollector) +} + +// NewProtocolsCollector returns a Collector exposing net/protocols stats +func NewProtocolsCollector(logger log.Logger) (Collector, error) { + fs, err := procfs.NewFS(*procPath) + if err != nil { + return nil, fmt.Errorf("failed to open procfs: %w", err) + } + return &protocolsCollector{fs, logger}, nil +} + +// Update implements Collector and exposes /proc/net/protocols metrics +func (c *protocolsCollector) Update(ch chan<- prometheus.Metric) error { + + protocolStats, err := c.fs.NetProtocols() + if err != nil { + return fmt.Errorf("couldn't get protocols: %w", err) + } + + type protoPair struct { + name string + v *float64 + } + + // In the interest of reudcing cardinality we are only interested in the + // first 8 fields as subsequent fields are not numerical or likely to change + // over time. + for _, p := range protocolStats { + p.Name = strings.Replace(strings.ToLower(p.Name), "-", "", -1) + re := regexp.MustCompile(*netProtocolFilter) + if !re.MatchString(p.Name) { + continue + } + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + subsystem, + fmt.Sprintf("%s_size", p.Name), + ), + fmt.Sprintf("The size, in bytes, of %s", p.Name), + nil, + nil, + ), + prometheus.GaugeValue, + float64(p.Size), + ) + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + subsystem, + fmt.Sprintf("%s_sockets", p.Name), + ), + fmt.Sprintf("Number of sockets in use by %s", p.Name), + nil, + nil, + ), + prometheus.GaugeValue, + float64(p.Sockets), + ) + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + subsystem, + fmt.Sprintf("%s_memory", p.Name), + ), + fmt.Sprintf("Total number of 4KB pages allocated by %s", p.Name), + nil, + nil, + ), + prometheus.GaugeValue, + float64(p.Memory), + ) + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + subsystem, + fmt.Sprintf("%s_pressure", p.Name), + ), + fmt.Sprintf("Indicates whether %s is experiencing memory pressure; 1 = true, 0 = false, -1 = not implemented", p.Name), + nil, + nil, + ), + prometheus.GaugeValue, + float64(p.Pressure), + ) + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + subsystem, + fmt.Sprintf("%s_maxhdr", p.Name), + ), + fmt.Sprintf("Max header size for %s", p.Name), + nil, + nil, + ), + prometheus.GaugeValue, + float64(p.MaxHeader), + ) + slabVal := float64(0.0) + if p.Slab { + slabVal = 1.0 + } + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + subsystem, + fmt.Sprintf("%s_slab", p.Name), + ), + fmt.Sprintf("Bool indicating whether %s is allocated from SLAB", p.Name), + nil, + nil, + ), + prometheus.GaugeValue, + slabVal, + ) + } + return nil +} From 580b14f215899e004fe0d3d267cd39f3d58dbab6 Mon Sep 17 00:00:00 2001 From: Juan Bran Date: Mon, 11 Jan 2021 13:56:34 -0800 Subject: [PATCH 2/4] upadting procfs to include dependency Signed-off-by: Juan Bran --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 623193cd1a..0985d2a24b 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.15.0 github.com/prometheus/exporter-toolkit v0.4.0 - github.com/prometheus/procfs v0.2.0 + github.com/prometheus/procfs v0.3.0 github.com/siebenmann/go-kstat v0.0.0-20200303194639-4e8294f9e9d5 github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a go.uber.org/multierr v1.5.0 // indirect diff --git a/go.sum b/go.sum index 548bc52396..97d45d3a7c 100644 --- a/go.sum +++ b/go.sum @@ -295,6 +295,8 @@ github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFB github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= From 0088e5996d1a1fce3f1591a4e5e0d024f0e837a5 Mon Sep 17 00:00:00 2001 From: Juan Bran Date: Mon, 11 Jan 2021 15:39:21 -0800 Subject: [PATCH 3/4] Cleaning up Signed-off-by: Juan Bran --- collector/protocols_linux.go | 5 ----- go.sum | 2 -- 2 files changed, 7 deletions(-) diff --git a/collector/protocols_linux.go b/collector/protocols_linux.go index 1a7b3baaa1..509e555685 100644 --- a/collector/protocols_linux.go +++ b/collector/protocols_linux.go @@ -58,11 +58,6 @@ func (c *protocolsCollector) Update(ch chan<- prometheus.Metric) error { return fmt.Errorf("couldn't get protocols: %w", err) } - type protoPair struct { - name string - v *float64 - } - // In the interest of reudcing cardinality we are only interested in the // first 8 fields as subsequent fields are not numerical or likely to change // over time. diff --git a/go.sum b/go.sum index 97d45d3a7c..977b09ecef 100644 --- a/go.sum +++ b/go.sum @@ -293,8 +293,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= From 69d23056f26e1e8154a5ffe7480fc3fd723de08c Mon Sep 17 00:00:00 2001 From: Juan Bran Date: Tue, 12 Jan 2021 22:30:20 -0800 Subject: [PATCH 4/4] Updating e2e to match bug fix in procfs (see: procfs/pull/343) Signed-off-by: Juan Bran --- collector/fixtures/e2e-output.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 1c352372a6..f33e6092ad 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3001,7 +3001,7 @@ node_xfs_inode_operation_reclaims_total{device="sda1"} 0 node_xfs_inode_operation_recycled_total{device="sda1"} 0 # HELP node_xfs_read_calls_total Number of read(2) system calls made to files in a filesystem. # TYPE node_xfs_read_calls_total counter -node_xfs_read_calls_total{device="sda1"} 28 +node_xfs_read_calls_total{device="sda1"} 0 # HELP node_xfs_vnode_active_total Number of vnodes not on free lists for a filesystem. # TYPE node_xfs_vnode_active_total counter node_xfs_vnode_active_total{device="sda1"} 4 @@ -3025,7 +3025,7 @@ node_xfs_vnode_release_total{device="sda1"} 1 node_xfs_vnode_remove_total{device="sda1"} 1 # HELP node_xfs_write_calls_total Number of write(2) system calls made to files in a filesystem. # TYPE node_xfs_write_calls_total counter -node_xfs_write_calls_total{device="sda1"} 0 +node_xfs_write_calls_total{device="sda1"} 28 # HELP node_zfs_abd_linear_cnt kstat.zfs.misc.abdstats.linear_cnt # TYPE node_zfs_abd_linear_cnt untyped node_zfs_abd_linear_cnt 62