Skip to content

Use procfs/blockdevice for diskstats #2072

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
luizluca wants to merge 1 commit into from
Closed

Use procfs/blockdevice for diskstats #2072

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 88 additions & 47 deletions collector/diskstats_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,27 +16,24 @@
package collector

import (
"bufio"
"fmt"
"io"
"os"
"regexp"
"strconv"
"strings"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"gopkg.in/alecthomas/kingpin.v2"

"github.com/prometheus/procfs/blockdevice"
)

const (
diskSectorSize = 512
diskstatsFilename = "diskstats"
diskSectorSize = 512
)

var (
ignoredDevices = kingpin.Flag("collector.diskstats.ignored-devices", "Regexp of devices to ignore for diskstats.").Default("^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d+$").String()
ignoredDevices = kingpin.Flag("collector.diskstats.ignored-devices", "Regexp of devices to ignore for diskstats.").Default("^(ram|loop|fd|(h|s|v|xv)d[a-z]|(mmcblk|nvme\\d+n)\\d+p)\\d+$").String()
preferSysFS = kingpin.Flag("collector.diskstats.prefer-sysfs", "Using /sys automatically skips partition metrics.").Default("true").Bool()
)

type typedFactorDesc struct {
Expand All @@ -53,7 +50,9 @@ func (d *typedFactorDesc) mustNewConstMetric(value float64, labels ...string) pr
}

type diskstatsCollector struct {
fs blockdevice.FS
ignoredDevicesPattern *regexp.Regexp
preferSysFS bool
descs []typedFactorDesc
logger log.Logger
}
Expand All @@ -67,8 +66,15 @@ func init() {
func NewDiskstatsCollector(logger log.Logger) (Collector, error) {
var diskLabelNames = []string{"device"}

fs, err := blockdevice.NewFS(*procPath, *sysPath)
if err != nil {
return nil, fmt.Errorf("failed to open procfs: %w", err)
}

return &diskstatsCollector{
fs: fs,
ignoredDevicesPattern: regexp.MustCompile(*ignoredDevices),
preferSysFS: *preferSysFS,
descs: []typedFactorDesc{
{
desc: readsCompletedDesc, valueType: prometheus.CounterValue,
Expand Down Expand Up @@ -185,56 +191,91 @@ func NewDiskstatsCollector(logger log.Logger) (Collector, error) {
}

func (c *diskstatsCollector) Update(ch chan<- prometheus.Metric) error {
diskStats, err := getDiskStats()
if err != nil {
return fmt.Errorf("couldn't get diskstats: %w", err)
}
var (
stats = map[string]blockdevice.IOStats{}
counts = map[string]int{}
useSysFS = c.preferSysFS
)

for dev, stats := range diskStats {
if c.ignoredDevicesPattern.MatchString(dev) {
level.Debug(c.logger).Log("msg", "Ignoring device", "device", dev)
continue
}
RETRY:
for {
if useSysFS {
devices, err := c.fs.SysBlockDevices()
if err != nil {
level.Warn(c.logger).Log("msg", "couldn't list devices from /sys. Retry from /proc", "err", err)
useSysFS = false
continue RETRY
}

for i, value := range stats {
// ignore unrecognized additional stats
if i >= len(c.descs) {
break
for _, dev := range devices {
if c.ignoredDevicesPattern.MatchString(dev) {
level.Debug(c.logger).Log("msg", "Ignoring device", "device", dev)
continue
}

stat, count, err := c.fs.SysBlockDeviceStat(dev)
if err != nil {
level.Warn(c.logger).Log("msg", "couldn't get diskstats. Retry from /proc", "device", dev, "err", err)
useSysFS = false
continue RETRY
}

stats[dev] = stat
counts[dev] = count
}
v, err := strconv.ParseFloat(value, 64)
} else {
diskstats, err := c.fs.ProcDiskstats()
if err != nil {
return fmt.Errorf("invalid value %s in diskstats: %w", value, err)
return fmt.Errorf("couldn't get diskstats: %w", err)
}

for _, diskstat := range diskstats {
dev := diskstat.Info.DeviceName

if c.ignoredDevicesPattern.MatchString(dev) {
level.Debug(c.logger).Log("msg", "Ignoring device", "device", dev)
continue
}

stats[dev] = diskstat.IOStats
// Do not count major, minor and device name
counts[dev] = diskstat.IoStatsCount - 3
}
ch <- c.descs[i].mustNewConstMetric(v, dev)
}
break
}
return nil
}

func getDiskStats() (map[string][]string, error) {
file, err := os.Open(procFilePath(diskstatsFilename))
if err != nil {
return nil, err
}
defer file.Close()
for dev, stat := range stats {

return parseDiskStats(file)
}
count := counts[dev]

func parseDiskStats(r io.Reader) (map[string][]string, error) {
var (
diskStats = map[string][]string{}
scanner = bufio.NewScanner(r)
)
statValues := []uint64{
stat.ReadIOs,
stat.ReadMerges,
stat.ReadSectors,
stat.ReadTicks,
stat.WriteIOs,
stat.WriteMerges,
stat.WriteSectors,
stat.WriteTicks,
stat.IOsInProgress,
stat.IOsTotalTicks,
stat.WeightedIOTicks,
stat.DiscardIOs,
stat.DiscardMerges,
stat.DiscardSectors,
stat.DiscardTicks,
stat.FlushRequestsCompleted,
stat.TimeSpentFlushing,
}

for scanner.Scan() {
parts := strings.Fields(scanner.Text())
if len(parts) < 4 { // we strip major, minor and dev
return nil, fmt.Errorf("invalid line in %s: %s", procFilePath(diskstatsFilename), scanner.Text())
for i := 0; i < count; i++ {
// ignore unrecognized additional stats
if i >= len(c.descs) {
break
}
ch <- c.descs[i].mustNewConstMetric(float64(statValues[i]), dev)
}
dev := parts[2]
diskStats[dev] = parts[3:]
}

return diskStats, scanner.Err()
return nil
}
52 changes: 0 additions & 52 deletions collector/diskstats_linux_test.go
View file
Open in desktop

This file was deleted.

22 changes: 0 additions & 22 deletions collector/fixtures/e2e-output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,8 +412,6 @@ node_disk_io_now{device="dm-3"} 0
node_disk_io_now{device="dm-4"} 0
node_disk_io_now{device="dm-5"} 0
node_disk_io_now{device="mmcblk0"} 0
node_disk_io_now{device="mmcblk0p1"} 0
node_disk_io_now{device="mmcblk0p2"} 0
node_disk_io_now{device="nvme0n1"} 0
node_disk_io_now{device="sda"} 0
node_disk_io_now{device="sdb"} 0
Expand All @@ -429,8 +427,6 @@ node_disk_io_time_seconds_total{device="dm-3"} 0.016
node_disk_io_time_seconds_total{device="dm-4"} 0.024
node_disk_io_time_seconds_total{device="dm-5"} 58.848
node_disk_io_time_seconds_total{device="mmcblk0"} 0.136
node_disk_io_time_seconds_total{device="mmcblk0p1"} 0.024
node_disk_io_time_seconds_total{device="mmcblk0p2"} 0.068
node_disk_io_time_seconds_total{device="nvme0n1"} 222.766
node_disk_io_time_seconds_total{device="sda"} 9653.880000000001
node_disk_io_time_seconds_total{device="sdb"} 60.730000000000004
Expand All @@ -446,8 +442,6 @@ node_disk_io_time_weighted_seconds_total{device="dm-3"} 0.10400000000000001
node_disk_io_time_weighted_seconds_total{device="dm-4"} 0.044
node_disk_io_time_weighted_seconds_total{device="dm-5"} 105.632
node_disk_io_time_weighted_seconds_total{device="mmcblk0"} 0.156
node_disk_io_time_weighted_seconds_total{device="mmcblk0p1"} 0.024
node_disk_io_time_weighted_seconds_total{device="mmcblk0p2"} 0.068
node_disk_io_time_weighted_seconds_total{device="nvme0n1"} 1032.546
node_disk_io_time_weighted_seconds_total{device="sda"} 82621.804
node_disk_io_time_weighted_seconds_total{device="sdb"} 67.07000000000001
Expand All @@ -463,8 +457,6 @@ node_disk_read_bytes_total{device="dm-3"} 1.98144e+06
node_disk_read_bytes_total{device="dm-4"} 529408
node_disk_read_bytes_total{device="dm-5"} 4.3150848e+07
node_disk_read_bytes_total{device="mmcblk0"} 798720
node_disk_read_bytes_total{device="mmcblk0p1"} 81920
node_disk_read_bytes_total{device="mmcblk0p2"} 389120
node_disk_read_bytes_total{device="nvme0n1"} 2.377714176e+09
node_disk_read_bytes_total{device="sda"} 5.13713216512e+11
node_disk_read_bytes_total{device="sdb"} 4.944782848e+09
Expand All @@ -480,8 +472,6 @@ node_disk_read_time_seconds_total{device="dm-3"} 0.10400000000000001
node_disk_read_time_seconds_total{device="dm-4"} 0.028
node_disk_read_time_seconds_total{device="dm-5"} 0.924
node_disk_read_time_seconds_total{device="mmcblk0"} 0.156
node_disk_read_time_seconds_total{device="mmcblk0p1"} 0.024
node_disk_read_time_seconds_total{device="mmcblk0p2"} 0.068
node_disk_read_time_seconds_total{device="nvme0n1"} 21.650000000000002
node_disk_read_time_seconds_total{device="sda"} 18492.372
node_disk_read_time_seconds_total{device="sdb"} 0.084
Expand All @@ -497,8 +487,6 @@ node_disk_reads_completed_total{device="dm-3"} 3870
node_disk_reads_completed_total{device="dm-4"} 392
node_disk_reads_completed_total{device="dm-5"} 3729
node_disk_reads_completed_total{device="mmcblk0"} 192
node_disk_reads_completed_total{device="mmcblk0p1"} 17
node_disk_reads_completed_total{device="mmcblk0p2"} 95
node_disk_reads_completed_total{device="nvme0n1"} 47114
node_disk_reads_completed_total{device="sda"} 2.5354637e+07
node_disk_reads_completed_total{device="sdb"} 326552
Expand All @@ -514,8 +502,6 @@ node_disk_reads_merged_total{device="dm-3"} 0
node_disk_reads_merged_total{device="dm-4"} 0
node_disk_reads_merged_total{device="dm-5"} 0
node_disk_reads_merged_total{device="mmcblk0"} 3
node_disk_reads_merged_total{device="mmcblk0p1"} 3
node_disk_reads_merged_total{device="mmcblk0p2"} 0
node_disk_reads_merged_total{device="nvme0n1"} 4
node_disk_reads_merged_total{device="sda"} 3.4367663e+07
node_disk_reads_merged_total{device="sdb"} 841
Expand All @@ -531,8 +517,6 @@ node_disk_write_time_seconds_total{device="dm-3"} 0
node_disk_write_time_seconds_total{device="dm-4"} 0.016
node_disk_write_time_seconds_total{device="dm-5"} 104.684
node_disk_write_time_seconds_total{device="mmcblk0"} 0
node_disk_write_time_seconds_total{device="mmcblk0p1"} 0
node_disk_write_time_seconds_total{device="mmcblk0p2"} 0
node_disk_write_time_seconds_total{device="nvme0n1"} 1011.053
node_disk_write_time_seconds_total{device="sda"} 63877.96
node_disk_write_time_seconds_total{device="sdb"} 5.007
Expand All @@ -548,8 +532,6 @@ node_disk_writes_completed_total{device="dm-3"} 0
node_disk_writes_completed_total{device="dm-4"} 38
node_disk_writes_completed_total{device="dm-5"} 98918
node_disk_writes_completed_total{device="mmcblk0"} 0
node_disk_writes_completed_total{device="mmcblk0p1"} 0
node_disk_writes_completed_total{device="mmcblk0p2"} 0
node_disk_writes_completed_total{device="nvme0n1"} 1.07832e+06
node_disk_writes_completed_total{device="sda"} 2.8444756e+07
node_disk_writes_completed_total{device="sdb"} 41822
Expand All @@ -565,8 +547,6 @@ node_disk_writes_merged_total{device="dm-3"} 0
node_disk_writes_merged_total{device="dm-4"} 0
node_disk_writes_merged_total{device="dm-5"} 0
node_disk_writes_merged_total{device="mmcblk0"} 0
node_disk_writes_merged_total{device="mmcblk0p1"} 0
node_disk_writes_merged_total{device="mmcblk0p2"} 0
node_disk_writes_merged_total{device="nvme0n1"} 43950
node_disk_writes_merged_total{device="sda"} 1.1134226e+07
node_disk_writes_merged_total{device="sdb"} 2895
Expand All @@ -582,8 +562,6 @@ node_disk_written_bytes_total{device="dm-3"} 0
node_disk_written_bytes_total{device="dm-4"} 70144
node_disk_written_bytes_total{device="dm-5"} 5.89664256e+08
node_disk_written_bytes_total{device="mmcblk0"} 0
node_disk_written_bytes_total{device="mmcblk0p1"} 0
node_disk_written_bytes_total{device="mmcblk0p2"} 0
node_disk_written_bytes_total{device="nvme0n1"} 2.0199236096e+10
node_disk_written_bytes_total{device="sda"} 2.58916880384e+11
node_disk_written_bytes_total{device="sdb"} 1.01012736e+09
Expand Down
Loading