From fccca5f38618ad478516ab81bc9f2f7d9f63a747 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Thu, 14 Nov 2019 11:48:55 +0300 Subject: [PATCH 01/39] unbound module wip --- cmd/godplugin/main.go | 1 + modules/unbound/README.md | 1 + modules/unbound/charts.go | 1 + modules/unbound/client.go | 134 ++++++++++++++++++++++++++++++++ modules/unbound/client_test.go | 98 +++++++++++++++++++++++ modules/unbound/collect.go | 22 ++++++ modules/unbound/unbound.go | 60 ++++++++++++++ modules/unbound/unbound_test.go | 1 + 8 files changed, 318 insertions(+) create mode 100644 modules/unbound/README.md create mode 100644 modules/unbound/charts.go create mode 100644 modules/unbound/client.go create mode 100644 modules/unbound/client_test.go create mode 100644 modules/unbound/collect.go create mode 100644 modules/unbound/unbound.go create mode 100644 modules/unbound/unbound_test.go diff --git a/cmd/godplugin/main.go b/cmd/godplugin/main.go index bdb1c3458..506f1d2ac 100644 --- a/cmd/godplugin/main.go +++ b/cmd/godplugin/main.go @@ -42,6 +42,7 @@ import ( _ "github.com/netdata/go.d.plugin/modules/solr" _ "github.com/netdata/go.d.plugin/modules/springboot2" _ "github.com/netdata/go.d.plugin/modules/tengine" + _ "github.com/netdata/go.d.plugin/modules/unbound" _ "github.com/netdata/go.d.plugin/modules/vcsa" _ "github.com/netdata/go.d.plugin/modules/vsphere" _ "github.com/netdata/go.d.plugin/modules/weblog" diff --git a/modules/unbound/README.md b/modules/unbound/README.md new file mode 100644 index 000000000..2fff69107 --- /dev/null +++ b/modules/unbound/README.md @@ -0,0 +1 @@ +#unbound \ No newline at end of file diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go new file mode 100644 index 000000000..8a156d14a --- /dev/null +++ b/modules/unbound/charts.go @@ -0,0 +1 @@ +package unbound diff --git a/modules/unbound/client.go b/modules/unbound/client.go new file mode 100644 index 000000000..170dda235 --- /dev/null +++ b/modules/unbound/client.go @@ -0,0 +1,134 @@ +package unbound + +import ( + "bufio" + "crypto/tls" + "fmt" + "io" + "net" + "strings" + "time" +) + +type clientConfig struct { + address string + timeout time.Duration + useTLS bool + tlsConf *tls.Config +} + +func newClient(config clientConfig) *client { + network := "tcp" + if strings.HasPrefix("/", config.address) { + network = "unix" + } + return &client{ + network: network, + address: config.address, + timeout: config.timeout, + useTLS: config.useTLS, + tlsConf: config.tlsConf, + reuseRecord: true, + record: nil, + conn: nil, + } +} + +type client struct { + network string + address string + timeout time.Duration + useTLS bool + tlsConf *tls.Config + reuseRecord bool + record []string + conn net.Conn +} + +func (c client) dial() (net.Conn, error) { + if !c.useTLS { + return net.DialTimeout(c.network, c.address, c.timeout) + } + var d net.Dialer + d.Timeout = c.timeout + return tls.DialWithDialer(&d, c.network, c.address, c.tlsConf) +} + +func (c *client) connect() (err error) { + c.conn, err = c.dial() + return err +} + +func (c *client) disconnect() error { + err := c.conn.Close() + c.conn = nil + return err +} + +func (c *client) isConnected() bool { + return c.conn != nil +} + +func (c *client) write(command string) error { + err := c.conn.SetWriteDeadline(time.Now().Add(c.timeout)) + if err != nil { + return err + } + _, err = c.conn.Write([]byte(command)) + return err +} + +func (c *client) read() (record []string, err error) { + if err = c.conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil { + return nil, err + } + + if c.reuseRecord { + record, err = read(c.record, c.conn) + c.record = record + } else { + record, err = read(nil, c.conn) + } + + return record, err +} + +func (c *client) send(command string) (lines []string, err error) { + if c.isConnected() { + _ = c.disconnect() + } + + err = c.connect() + if err != nil { + return nil, err + } + defer func() { + _ = c.disconnect() + }() + + err = c.write(command) + if err != nil { + return nil, err + } + + return c.read() +} + +const maxLinesToRead = 500 + +// https://github.com/NLnetLabs/unbound/blob/master/doc/control_proto_spec.txt +// Server executes command. And sends reply in ascii text over channel, closes the channel when done. +func read(dst []string, reader io.Reader) ([]string, error) { + dst = dst[:0] + var num int + s := bufio.NewScanner(reader) + + for s.Scan() { + dst = append(dst, s.Text()) + num++ + if num > maxLinesToRead { + return nil, fmt.Errorf("read line limit exceeded (%d)", maxLinesToRead) + } + } + return dst, s.Err() +} diff --git a/modules/unbound/client_test.go b/modules/unbound/client_test.go new file mode 100644 index 000000000..6769f0234 --- /dev/null +++ b/modules/unbound/client_test.go @@ -0,0 +1,98 @@ +package unbound + +import ( + "bufio" + "errors" + "net" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const ( + srvAddress = "127.0.0.1:38001" +) + +func Test_clientSend(t *testing.T) { + srv := &tcpServer{addr: srvAddress, respNumLines: 10} + go srv.Run() + defer srv.Close() + time.Sleep(time.Second) + + c := newClient(clientConfig{ + address: srvAddress, + timeout: time.Second, + }) + + lines, err := c.send("whatever\n") + assert.NoError(t, err) + assert.Len(t, lines, 10) + + lines, err = c.send("whatever\n") + assert.NoError(t, err) + assert.Len(t, lines, 10) +} + +func Test_clientSend_ReadLineLimitExceeded(t *testing.T) { + srv := &tcpServer{addr: srvAddress, respNumLines: maxLinesToRead + 1} + go srv.Run() + defer srv.Close() + time.Sleep(time.Second) + + c := newClient(clientConfig{ + address: srvAddress, + timeout: time.Second, + }) + + lines, err := c.send("whatever\n") + assert.Error(t, err) + assert.Len(t, lines, 0) +} + +type tcpServer struct { + addr string + server net.Listener + respNumLines int +} + +func (t *tcpServer) Run() (err error) { + t.server, err = net.Listen("tcp", t.addr) + if err != nil { + return + } + return t.handleConnections() +} + +func (t *tcpServer) Close() (err error) { + return t.server.Close() +} + +func (t *tcpServer) handleConnections() error { + for { + conn, err := t.server.Accept() + if err != nil || conn == nil { + return errors.New("could not accept connection") + } + go t.handleConnection(conn) + } +} + +func (t *tcpServer) handleConnection(conn net.Conn) { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(time.Second * 2)) + + rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) + + req, err := rw.ReadString('\n') + if err != nil { + _, _ = rw.WriteString("error failed to read input") + _ = rw.Flush() + return + } + + resp := strings.Repeat(req, t.respNumLines) + _, _ = rw.WriteString(resp) + _ = rw.Flush() +} diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go new file mode 100644 index 000000000..3e37ce39a --- /dev/null +++ b/modules/unbound/collect.go @@ -0,0 +1,22 @@ +package unbound + +import ( + "sort" + "strings" +) + +func (u *Unbound) collect() (map[string]int64, error) { + return nil, nil +} + +func findByPrefix(ms []string, prefix string) []string { + from := sort.Search(len(ms), func(i int) bool { return ms[i] >= prefix }) + if from == len(ms) || !strings.HasPrefix(ms[from], prefix) { + return nil + } + until := from + 1 + for until < len(ms) && strings.HasPrefix(ms[until], prefix) { + until++ + } + return ms[from:until] +} diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go new file mode 100644 index 000000000..0e8dc69e3 --- /dev/null +++ b/modules/unbound/unbound.go @@ -0,0 +1,60 @@ +package unbound + +import ( + "github.com/netdata/go-orchestrator/module" + "github.com/netdata/go.d.plugin/pkg/web" +) + +func init() { + creator := module.Creator{ + Create: func() module.Module { return New() }, + } + + module.Register("unbound", creator) +} + +func New() *Unbound { + config := Config{} + + return &Unbound{ + Config: config, + charts: nil, + } +} + +type ( + Config struct { + Address string `yaml:"address"` + ConfPath string `yaml:"conf_path"` + Timeout web.Duration `yaml:"timeout"` + web.ClientTLSConfig `yaml:",inline"` + } + Unbound struct { + module.Base + Config `yaml:",inline"` + + charts *module.Charts + } +) + +func (Unbound) Cleanup() {} + +func (u *Unbound) Init() bool { return true } + +func (u Unbound) Check() bool { return false } + +func (u Unbound) Charts() *module.Charts { return u.charts } + +func (u *Unbound) Collect() map[string]int64 { + mx, err := u.collect() + + if err != nil { + u.Error(err) + } + + if len(mx) == 0 { + return nil + } + + return mx +} diff --git a/modules/unbound/unbound_test.go b/modules/unbound/unbound_test.go new file mode 100644 index 000000000..8a156d14a --- /dev/null +++ b/modules/unbound/unbound_test.go @@ -0,0 +1 @@ +package unbound From e7065af6b75ec134f7e0bd5903666aa3a9d378ce Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Thu, 14 Nov 2019 15:29:18 +0300 Subject: [PATCH 02/39] add testdata --- modules/unbound/testdata/extended_stats.txt | 161 ++++++++++++++++++++ modules/unbound/testdata/stats.txt | 83 ++++++++++ 2 files changed, 244 insertions(+) create mode 100644 modules/unbound/testdata/extended_stats.txt create mode 100644 modules/unbound/testdata/stats.txt diff --git a/modules/unbound/testdata/extended_stats.txt b/modules/unbound/testdata/extended_stats.txt new file mode 100644 index 000000000..8fa6258a5 --- /dev/null +++ b/modules/unbound/testdata/extended_stats.txt @@ -0,0 +1,161 @@ +thread0.num.queries=1 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=0 +thread0.num.cachemiss=1 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=1 +thread0.requestlist.avg=0 +thread0.requestlist.max=0 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.003121 +thread0.recursion.time.median=0 +thread0.tcpusage=0 +thread1.num.queries=0 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=0 +thread1.num.cachemiss=0 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=0 +thread1.requestlist.avg=0 +thread1.requestlist.max=0 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.000000 +thread1.recursion.time.median=0 +thread1.tcpusage=0 +thread2.num.queries=0 +thread2.num.queries_ip_ratelimited=0 +thread2.num.cachehits=0 +thread2.num.cachemiss=0 +thread2.num.prefetch=0 +thread2.num.zero_ttl=0 +thread2.num.recursivereplies=0 +thread2.requestlist.avg=0 +thread2.requestlist.max=0 +thread2.requestlist.overwritten=0 +thread2.requestlist.exceeded=0 +thread2.requestlist.current.all=0 +thread2.requestlist.current.user=0 +thread2.recursion.time.avg=0.000000 +thread2.recursion.time.median=0 +thread2.tcpusage=0 +thread3.num.queries=0 +thread3.num.queries_ip_ratelimited=0 +thread3.num.cachehits=0 +thread3.num.cachemiss=0 +thread3.num.prefetch=0 +thread3.num.zero_ttl=0 +thread3.num.recursivereplies=0 +thread3.requestlist.avg=0 +thread3.requestlist.max=0 +thread3.requestlist.overwritten=0 +thread3.requestlist.exceeded=0 +thread3.requestlist.current.all=0 +thread3.requestlist.current.user=0 +thread3.recursion.time.avg=0.000000 +thread3.recursion.time.median=0 +thread3.tcpusage=0 +total.num.queries=1 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=0 +total.num.cachemiss=1 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=1 +total.requestlist.avg=0 +total.requestlist.max=0 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.003121 +total.recursion.time.median=0 +total.tcpusage=0 +time.now=1571406665.059510 +time.up=11.768185 +time.elapsed=0.801449 +mem.cache.rrset=75653 +mem.cache.message=72418 +mem.mod.iterator=16588 +mem.mod.validator=0 +mem.mod.respip=0 +mem.mod.subnet=0 +histogram.000000.000000.to.000000.000001=0 +histogram.000000.000001.to.000000.000002=0 +histogram.000000.000002.to.000000.000004=0 +histogram.000000.000004.to.000000.000008=0 +histogram.000000.000008.to.000000.000016=0 +histogram.000000.000016.to.000000.000032=0 +histogram.000000.000032.to.000000.000064=0 +histogram.000000.000064.to.000000.000128=0 +histogram.000000.000128.to.000000.000256=0 +histogram.000000.000256.to.000000.000512=0 +histogram.000000.000512.to.000000.001024=0 +histogram.000000.001024.to.000000.002048=0 +histogram.000000.002048.to.000000.004096=1 +histogram.000000.004096.to.000000.008192=0 +histogram.000000.008192.to.000000.016384=0 +histogram.000000.016384.to.000000.032768=0 +histogram.000000.032768.to.000000.065536=0 +histogram.000000.065536.to.000000.131072=0 +histogram.000000.131072.to.000000.262144=0 +histogram.000000.262144.to.000000.524288=0 +histogram.000000.524288.to.000001.000000=0 +histogram.000001.000000.to.000002.000000=0 +histogram.000002.000000.to.000004.000000=0 +histogram.000004.000000.to.000008.000000=0 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=0 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=1 +num.query.class.IN=1 +num.query.opcode.QUERY=1 +num.query.tcp=0 +num.query.tcpout=0 +num.query.ipv6=0 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=1 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=0 +num.query.flags.CD=0 +num.query.edns.present=0 +num.query.edns.DO=0 +num.answer.rcode.NOERROR=1 +num.answer.rcode.FORMERR=0 +num.answer.rcode.SERVFAIL=0 +num.answer.rcode.NXDOMAIN=0 +num.answer.rcode.NOTIMPL=0 +num.answer.rcode.REFUSED=0 +num.query.ratelimited=0 +num.answer.secure=0 +num.answer.bogus=0 +num.rrset.bogus=0 +unwanted.queries=0 +unwanted.replies=0 +msg.cache.count=23 +rrset.cache.count=32 +infra.cache.count=4 +key.cache.count=0 \ No newline at end of file diff --git a/modules/unbound/testdata/stats.txt b/modules/unbound/testdata/stats.txt new file mode 100644 index 000000000..02ad3d0e0 --- /dev/null +++ b/modules/unbound/testdata/stats.txt @@ -0,0 +1,83 @@ +thread0.num.queries=1 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=0 +thread0.num.cachemiss=1 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=0 +thread0.requestlist.avg=1 +thread0.requestlist.max=1 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=1 +thread0.requestlist.current.user=1 +thread0.recursion.time.avg=0.000000 +thread0.recursion.time.median=0 +thread0.tcpusage=0 +thread1.num.queries=2 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=2 +thread1.num.cachemiss=0 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=0 +thread1.requestlist.avg=0 +thread1.requestlist.max=0 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.000000 +thread1.recursion.time.median=0 +thread1.tcpusage=0 +thread2.num.queries=1 +thread2.num.queries_ip_ratelimited=0 +thread2.num.cachehits=0 +thread2.num.cachemiss=1 +thread2.num.prefetch=0 +thread2.num.zero_ttl=0 +thread2.num.recursivereplies=1 +thread2.requestlist.avg=0 +thread2.requestlist.max=0 +thread2.requestlist.overwritten=0 +thread2.requestlist.exceeded=0 +thread2.requestlist.current.all=0 +thread2.requestlist.current.user=0 +thread2.recursion.time.avg=0.236823 +thread2.recursion.time.median=0 +thread2.tcpusage=0 +thread3.num.queries=0 +thread3.num.queries_ip_ratelimited=0 +thread3.num.cachehits=0 +thread3.num.cachemiss=0 +thread3.num.prefetch=0 +thread3.num.zero_ttl=0 +thread3.num.recursivereplies=0 +thread3.requestlist.avg=0 +thread3.requestlist.max=0 +thread3.requestlist.overwritten=0 +thread3.requestlist.exceeded=0 +thread3.requestlist.current.all=0 +thread3.requestlist.current.user=0 +thread3.recursion.time.avg=0.000000 +thread3.recursion.time.median=0 +thread3.tcpusage=0 +total.num.queries=4 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=2 +total.num.cachemiss=2 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=1 +total.requestlist.avg=0.5 +total.requestlist.max=1 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=1 +total.requestlist.current.user=1 +total.recursion.time.avg=0.236823 +total.recursion.time.median=0 +total.tcpusage=0 +time.now=1571405114.158591 +time.up=31.232563 +time.elapsed=0.904431 \ No newline at end of file From ef90ad9e37ed9aacfdb2393266e52b322fecd297 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Thu, 14 Nov 2019 15:29:28 +0300 Subject: [PATCH 03/39] collect wip --- modules/unbound/collect.go | 133 ++++++++++++++++++++++++++++++-- modules/unbound/metrics.go | 62 +++++++++++++++ modules/unbound/unbound.go | 14 +++- modules/unbound/unbound_test.go | 56 ++++++++++++++ 4 files changed, 257 insertions(+), 8 deletions(-) create mode 100644 modules/unbound/metrics.go diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 3e37ce39a..c76de160a 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -1,22 +1,143 @@ package unbound import ( + "errors" + "fmt" "sort" + "strconv" "strings" + + "github.com/netdata/go.d.plugin/pkg/stm" ) func (u *Unbound) collect() (map[string]int64, error) { - return nil, nil + if err := u.collectStats(); err != nil { + return nil, err + } + return stm.ToMap(u.mx), nil } -func findByPrefix(ms []string, prefix string) []string { - from := sort.Search(len(ms), func(i int) bool { return ms[i] >= prefix }) - if from == len(ms) || !strings.HasPrefix(ms[from], prefix) { +func (u *Unbound) collectStats() error { + resp, err := u.client.send("UBCT1 stats_noreset\n") + if err != nil { + return err + } + switch len(resp) { + case 0: + return errors.New("empty response") + case 1: + // In case of error the first line of the response is: error \n + // For many commands the response is 'ok\n', but it is not the case for 'stats'. + return errors.New(resp[0]) + } + + stats, err := convertToUnboundStats(resp) + if err != nil { + return err + } + + u.collectTotal(stats) + return nil +} + +func (u *Unbound) collectTotal(ss stats) { + for _, s := range ss.find("total.") { + v := s.value + switch s.name { + case "total.num.queries": + u.mx.Total.Queries.Set(v) + case "total.num.queries_ip_ratelimited": + u.mx.Total.QueriesIPRL.Set(v) + case "total.num.cachehits": + u.mx.Total.Cache.Hits.Set(v) + case "total.num.cachemiss": + u.mx.Total.Cache.Miss.Set(v) + case "total.num.prefetch": + u.mx.Total.Prefetch.Set(v) + case "total.num.zero_ttl": + u.mx.Total.ZeroTTL.Set(v) + case "total.num.recursivereplies": + u.mx.Total.RecursiveReplies.Set(v) + case "total.num.dnscrypt.crypted": + u.mx.Total.DNSCrypt.Crypted.Set(v) + case "total.num.dnscrypt.cert": + u.mx.Total.DNSCrypt.Cert.Set(v) + case "total.num.dnscrypt.cleartext": + u.mx.Total.DNSCrypt.ClearText.Set(v) + case "total.num.dnscrypt.malformed": + u.mx.Total.DNSCrypt.Malformed.Set(v) + case "total.requestlist.avg": + u.mx.Total.RequestList.Avg.Set(v) + case "total.requestlist.max": + u.mx.Total.RequestList.Max.Set(v) + case "total.requestlist.overwritten": + u.mx.Total.RequestList.Overwritten.Set(v) + case "total.requestlist.exceeded": + u.mx.Total.RequestList.Exceeded.Set(v) + case "total.requestlist.current.all": + u.mx.Total.RequestList.CurrentAll.Set(v) + case "total.requestlist.current.user": + u.mx.Total.RequestList.CurrentUser.Set(v) + case "total.recursion.time.avg": + u.mx.Total.RecursionTime.Avg.Set(v) + case "total.recursion.time.median": + u.mx.Total.RecursionTime.Median.Set(v) + case "total.tcpusage": + u.mx.Total.TCPUsage.Set(v) + } + } +} + +func (u *Unbound) collectTime(ss stats) { + for _, s := range ss.find("time.") { + switch s.name { + case "time.up": + u.mx.Uptime.Set(s.value) + } + } +} + +type ( + stats []stat + stat struct { + name string + value float64 + } +) + +func (ss stats) empty() bool { + return len(ss) == 0 +} + +func (ss stats) find(prefix string) stats { + from := sort.Search(len(ss), func(i int) bool { return ss[i].name >= prefix }) + if from == len(ss) || !strings.HasPrefix(ss[from].name, prefix) { return nil } until := from + 1 - for until < len(ms) && strings.HasPrefix(ms[until], prefix) { + for until < len(ss) && strings.HasPrefix(ss[until].name, prefix) { until++ } - return ms[from:until] + return ss[from:until] +} + +func convertToUnboundStats(lines []string) (stats, error) { + sort.Strings(lines) + var ubs stats + for _, line := range lines { + // 'stats' output is a list of [name]=[value] lines. + parts := strings.Split(line, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("bad stats syntax: %s", line) + } + + name, value := parts[0], parts[1] + v, err := strconv.ParseFloat(value, 10) + if err != nil { + return nil, err + } + + ubs = append(ubs, stat{name, v}) + } + return ubs, nil } diff --git a/modules/unbound/metrics.go b/modules/unbound/metrics.go new file mode 100644 index 000000000..322fe0294 --- /dev/null +++ b/modules/unbound/metrics.go @@ -0,0 +1,62 @@ +package unbound + +import ( + "github.com/netdata/go.d.plugin/pkg/metrics" +) + +/* +total.num.queries=0 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=0 +total.num.cachemiss=0 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=0 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0 +total.requestlist.max=0 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.000000 +total.recursion.time.median=0 +total.tcpusage=0 +*/ + +type metricsData struct { + Total struct { + Queries metrics.Gauge `stm:"queries"` + QueriesIPRL metrics.Gauge `stm:"queries_ip_ratelimited"` + Cache struct { + Hits metrics.Gauge `stm:"hits"` + Miss metrics.Gauge `stm:"miss"` + } `stm:"cache"` + Prefetch metrics.Gauge `stm:"prefetch"` + ZeroTTL metrics.Gauge `stm:"zero_ttl"` + RecursiveReplies metrics.Gauge `stm:"recursive_replies"` + DNSCrypt struct { + Crypted metrics.Gauge `stm:"crypted"` + Cert metrics.Gauge `stm:"cert"` + ClearText metrics.Gauge `stm:"clear_text"` + Malformed metrics.Gauge `stm:"_malformed"` + } `stm:"dns_crypt"` + RequestList struct { + Avg metrics.Gauge `stm:"avg"` + Max metrics.Gauge `stm:"max"` + Overwritten metrics.Gauge `stm:"overwritten"` + Exceeded metrics.Gauge `stm:"exceeded"` + CurrentAll metrics.Gauge `stm:"current_all"` + CurrentUser metrics.Gauge `stm:"current_user"` + } `stm:"request_list"` + RecursionTime struct { + Avg metrics.Gauge `stm:"avg"` + Median metrics.Gauge `stm:"median"` + } `stm:"recursion_time"` + TCPUsage metrics.Gauge `stm:"tcp_usage"` + } + Uptime metrics.Gauge `stm:"uptime"` +} diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 0e8dc69e3..70a8f6055 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -1,8 +1,9 @@ package unbound import ( - "github.com/netdata/go-orchestrator/module" "github.com/netdata/go.d.plugin/pkg/web" + + "github.com/netdata/go-orchestrator/module" ) func init() { @@ -22,6 +23,10 @@ func New() *Unbound { } } +type unboundClient interface { + send(command string) ([]string, error) +} + type ( Config struct { Address string `yaml:"address"` @@ -33,13 +38,18 @@ type ( module.Base Config `yaml:",inline"` + client unboundClient charts *module.Charts + mx *metricsData } ) func (Unbound) Cleanup() {} -func (u *Unbound) Init() bool { return true } +func (u *Unbound) Init() bool { + u.mx = &metricsData{} + return true +} func (u Unbound) Check() bool { return false } diff --git a/modules/unbound/unbound_test.go b/modules/unbound/unbound_test.go index 8a156d14a..ee774a3da 100644 --- a/modules/unbound/unbound_test.go +++ b/modules/unbound/unbound_test.go @@ -1 +1,57 @@ package unbound + +import ( + "fmt" + "io/ioutil" + "testing" + "time" + + "github.com/netdata/go-orchestrator/module" + "github.com/stretchr/testify/assert" +) + +var ( + statsData, _ = ioutil.ReadFile("testdata/stats.txt") + extStatsData, _ = ioutil.ReadFile("testdata/extended_stats.txt") +) + +func Test_readTestData(t *testing.T) { + assert.NotNil(t, statsData) + assert.NotNil(t, extStatsData) +} + +func TestNew(t *testing.T) { + assert.Implements(t, (*module.Module)(nil), New()) +} + +func TestUnbound_Init(t *testing.T) { + +} + +func TestUnbound_Check(t *testing.T) { + +} + +func TestUnbound_Cleanup(t *testing.T) { + +} + +func TestUnbound_Charts(t *testing.T) { + +} + +func TestUnbound_Collect(t *testing.T) { + v := New() + v.Init() + v.client = newClient(clientConfig{ + address: "192.168.88.223:8953", + timeout: time.Second * 2, + useTLS: false, + tlsConf: nil, + }) + m := v.Collect() + + for k, v := range m { + fmt.Println(k, v) + } +} From 2db77212d9d5e1eb6235a84daa5ce08964eca0e2 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Thu, 14 Nov 2019 19:00:35 +0300 Subject: [PATCH 04/39] collect wip --- modules/unbound/collect.go | 204 ++++++++++++++++++++++++------------- modules/unbound/metrics.go | 113 ++++++++++---------- 2 files changed, 194 insertions(+), 123 deletions(-) diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index c76de160a..ff2916a6b 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -10,6 +10,14 @@ import ( "github.com/netdata/go.d.plugin/pkg/stm" ) +type ( + stats []stat + stat struct { + name string + value float64 + } +) + func (u *Unbound) collect() (map[string]int64, error) { if err := u.collectStats(); err != nil { return nil, err @@ -36,89 +44,149 @@ func (u *Unbound) collectStats() error { return err } - u.collectTotal(stats) + for _, s := range stats { + u.collectStat(s) + } return nil } -func (u *Unbound) collectTotal(ss stats) { - for _, s := range ss.find("total.") { - v := s.value - switch s.name { - case "total.num.queries": - u.mx.Total.Queries.Set(v) - case "total.num.queries_ip_ratelimited": - u.mx.Total.QueriesIPRL.Set(v) - case "total.num.cachehits": - u.mx.Total.Cache.Hits.Set(v) - case "total.num.cachemiss": - u.mx.Total.Cache.Miss.Set(v) - case "total.num.prefetch": - u.mx.Total.Prefetch.Set(v) - case "total.num.zero_ttl": - u.mx.Total.ZeroTTL.Set(v) - case "total.num.recursivereplies": - u.mx.Total.RecursiveReplies.Set(v) - case "total.num.dnscrypt.crypted": - u.mx.Total.DNSCrypt.Crypted.Set(v) - case "total.num.dnscrypt.cert": - u.mx.Total.DNSCrypt.Cert.Set(v) - case "total.num.dnscrypt.cleartext": - u.mx.Total.DNSCrypt.ClearText.Set(v) - case "total.num.dnscrypt.malformed": - u.mx.Total.DNSCrypt.Malformed.Set(v) - case "total.requestlist.avg": - u.mx.Total.RequestList.Avg.Set(v) - case "total.requestlist.max": - u.mx.Total.RequestList.Max.Set(v) - case "total.requestlist.overwritten": - u.mx.Total.RequestList.Overwritten.Set(v) - case "total.requestlist.exceeded": - u.mx.Total.RequestList.Exceeded.Set(v) - case "total.requestlist.current.all": - u.mx.Total.RequestList.CurrentAll.Set(v) - case "total.requestlist.current.user": - u.mx.Total.RequestList.CurrentUser.Set(v) - case "total.recursion.time.avg": - u.mx.Total.RecursionTime.Avg.Set(v) - case "total.recursion.time.median": - u.mx.Total.RecursionTime.Median.Set(v) - case "total.tcpusage": - u.mx.Total.TCPUsage.Set(v) - } +func (u *Unbound) collectStat(s stat) { + switch { + case strings.HasPrefix(s.name, "thread"): + u.collectThread(s) + case strings.HasPrefix(s.name, "total."): + u.collectTotal(s) + case strings.HasPrefix(s.name, "time."): + u.collectTime(s) + case strings.HasPrefix(s.name, "num.query.type."): + u.collectQueryType(s) + case strings.HasPrefix(s.name, "num.query.class."): + u.collectQueryClass(s) + case strings.HasPrefix(s.name, "num.query.opcode."): + u.collectQueryOpCode(s) + case strings.HasPrefix(s.name, "num.answer.rcode."): + u.collectAnswerRCode(s) + case strings.HasPrefix(s.name, "mem."): + u.collectMem(s) } } -func (u *Unbound) collectTime(ss stats) { - for _, s := range ss.find("time.") { - switch s.name { - case "time.up": - u.mx.Uptime.Set(s.value) - } +func (u *Unbound) collectMem(s stat) { + +} + +func (u *Unbound) collectThread(s stat) { + //thread0.* + i := strings.IndexByte(s.name, '.') + if i == -1 || i < 6 { + return + } + threadID := s.name[6:i] + c, ok := u.mx.Thread[threadID] + if !ok { + c = &common{} + u.mx.Thread[threadID] = c } + s.name = s.name[i+1:] + collectCommon(c, s) } -type ( - stats []stat - stat struct { - name string - value float64 +func (u *Unbound) collectTotal(s stat) { + i := strings.IndexByte(s.name, '.') + s.name = s.name[i+1:] + collectCommon(&u.mx.common, s) +} + +func (u *Unbound) collectTime(s stat) { + switch s.name { + case "time.up": + u.mx.Uptime = s.value } -) +} + +func (u *Unbound) collectQueryType(s stat) { + i := len("num.query.type.") + typ := s.name[i:] + v, ok := u.mx.QueryType[typ] + if !ok { + //TODO: + } + u.mx.QueryType[typ] += v +} + +func (u *Unbound) collectQueryClass(s stat) { + i := len("num.query.class.") + class := s.name[i:] + v, ok := u.mx.QueryClass[class] + if !ok { + //TODO: + } + u.mx.QueryClass[class] += v +} -func (ss stats) empty() bool { - return len(ss) == 0 +func (u *Unbound) collectQueryOpCode(s stat) { + i := len("num.query.opcode.") + opcode := s.name[i:] + v, ok := u.mx.QueryOpCode[opcode] + if !ok { + //TODO: + } + u.mx.QueryOpCode[opcode] += v } -func (ss stats) find(prefix string) stats { - from := sort.Search(len(ss), func(i int) bool { return ss[i].name >= prefix }) - if from == len(ss) || !strings.HasPrefix(ss[from].name, prefix) { - return nil +func (u *Unbound) collectAnswerRCode(s stat) { + i := len("num.answer.rcode.") + rcode := s.name[i:] + v, ok := u.mx.AnswerRCode[rcode] + if !ok { + //TODO: } - until := from + 1 - for until < len(ss) && strings.HasPrefix(ss[until].name, prefix) { - until++ + u.mx.AnswerRCode[rcode] += v +} + +func collectCommon(c *common, s stat) { + switch s.name { + case "num.queries": + c.Queries = s.value + case "num.queries_ip_ratelimited": + c.QueriesIPRL = s.value + case "num.cachehits": + c.Cache.Hits = s.value + case "num.cachemiss": + c.Cache.Miss = s.value + case "num.prefetch": + c.Prefetch = s.value + case "num.zero_ttl": + c.ZeroTTL = s.value + case "num.recursivereplies": + c.RecursiveReplies = s.value + case "num.dnscrypt.crypted": + c.DNSCrypt.Crypted = s.value + case "num.dnscrypt.cert": + c.DNSCrypt.Cert = s.value + case "num.dnscrypt.cleartext": + c.DNSCrypt.ClearText = s.value + case "num.dnscrypt.malformed": + c.DNSCrypt.Malformed = s.value + case "requestlist.avg": + c.RequestList.Avg = s.value + case "requestlist.max": + c.RequestList.Max = s.value + case "requestlist.overwritten": + c.RequestList.Overwritten = s.value + case "requestlist.exceeded": + c.RequestList.Exceeded = s.value + case "requestlist.current.all": + c.RequestList.CurrentAll = s.value + case "requestlist.current.user": + c.RequestList.CurrentUser = s.value + case "recursion.time.avg": + c.RecursionTime.Avg = s.value + case "recursion.time.median": + c.RecursionTime.Median = s.value + case "tcpusage": + c.TCPUsage = s.value } - return ss[from:until] } func convertToUnboundStats(lines []string) (stats, error) { diff --git a/modules/unbound/metrics.go b/modules/unbound/metrics.go index 322fe0294..3911a2c67 100644 --- a/modules/unbound/metrics.go +++ b/modules/unbound/metrics.go @@ -1,62 +1,65 @@ package unbound -import ( - "github.com/netdata/go.d.plugin/pkg/metrics" -) +// https://github.com/NLnetLabs/unbound/blob/master/smallapp/unbound-control.c -/* -total.num.queries=0 -total.num.queries_ip_ratelimited=0 -total.num.cachehits=0 -total.num.cachemiss=0 -total.num.prefetch=0 -total.num.zero_ttl=0 -total.num.recursivereplies=0 -total.num.dnscrypt.crypted=0 -total.num.dnscrypt.cert=0 -total.num.dnscrypt.cleartext=0 -total.num.dnscrypt.malformed=0 -total.requestlist.avg=0 -total.requestlist.max=0 -total.requestlist.overwritten=0 -total.requestlist.exceeded=0 -total.requestlist.current.all=0 -total.requestlist.current.user=0 -total.recursion.time.avg=0.000000 -total.recursion.time.median=0 -total.tcpusage=0 -*/ +type ( + cache struct { + Hits float64 `stm:"hits"` + Miss float64 `stm:"miss"` + } + dnsCrypt struct { + Crypted float64 `stm:"crypted"` + Cert float64 `stm:"cert"` + ClearText float64 `stm:"clear_text"` + Malformed float64 `stm:"malformed"` + } + requestList struct { + Avg float64 `stm:"avg"` + Max float64 `stm:"max"` + Overwritten float64 `stm:"overwritten"` + Exceeded float64 `stm:"exceeded"` + CurrentAll float64 `stm:"current_all"` + CurrentUser float64 `stm:"current_user"` + } + recursionTime struct { + Avg float64 `stm:"avg"` + Median float64 `stm:"median"` + } + common struct { + Queries float64 `stm:"queries"` + QueriesIPRL float64 `stm:"queries_ip_ratelimited"` + Cache cache `stm:"cache"` + Prefetch float64 `stm:"prefetch"` + ZeroTTL float64 `stm:"zero_ttl"` + RecursiveReplies float64 `stm:"recursive_replies"` + DNSCrypt dnsCrypt `stm:"dns_crypt"` + RequestList requestList `stm:"request_list"` + RecursionTime recursionTime `stm:"recursion_time"` + TCPUsage float64 `stm:"tcp_usage"` + } + extended struct { + QueryType map[string]float64 `stm:"query_type"` + QueryClass map[string]float64 `stm:"query_class"` + QueryOpCode map[string]float64 `stm:"query_opcode"` + AnswerRCode map[string]float64 `stm:"answer_rcode"` + } +) type metricsData struct { - Total struct { - Queries metrics.Gauge `stm:"queries"` - QueriesIPRL metrics.Gauge `stm:"queries_ip_ratelimited"` - Cache struct { - Hits metrics.Gauge `stm:"hits"` - Miss metrics.Gauge `stm:"miss"` - } `stm:"cache"` - Prefetch metrics.Gauge `stm:"prefetch"` - ZeroTTL metrics.Gauge `stm:"zero_ttl"` - RecursiveReplies metrics.Gauge `stm:"recursive_replies"` - DNSCrypt struct { - Crypted metrics.Gauge `stm:"crypted"` - Cert metrics.Gauge `stm:"cert"` - ClearText metrics.Gauge `stm:"clear_text"` - Malformed metrics.Gauge `stm:"_malformed"` - } `stm:"dns_crypt"` - RequestList struct { - Avg metrics.Gauge `stm:"avg"` - Max metrics.Gauge `stm:"max"` - Overwritten metrics.Gauge `stm:"overwritten"` - Exceeded metrics.Gauge `stm:"exceeded"` - CurrentAll metrics.Gauge `stm:"current_all"` - CurrentUser metrics.Gauge `stm:"current_user"` - } `stm:"request_list"` - RecursionTime struct { - Avg metrics.Gauge `stm:"avg"` - Median metrics.Gauge `stm:"median"` - } `stm:"recursion_time"` - TCPUsage metrics.Gauge `stm:"tcp_usage"` + common `stm:"total"` + extended `stm:""` + Uptime float64 `stm:"uptime"` + Thread map[string]*common `stm:"thread"` +} + +func newMetricsData() *metricsData { + return &metricsData{ + extended: extended{ + QueryType: make(map[string]float64), + QueryClass: make(map[string]float64), + QueryOpCode: make(map[string]float64), + AnswerRCode: make(map[string]float64), + }, + Thread: make(map[string]*common), } - Uptime metrics.Gauge `stm:"uptime"` } From 222c564f5be439f762a47f168746bb925a83bd5e Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Sat, 16 Nov 2019 16:49:26 +0300 Subject: [PATCH 05/39] collect wip --- modules/unbound/charts.go | 399 ++++++++++++++++++++ modules/unbound/client.go | 28 +- modules/unbound/collect.go | 265 +++++-------- modules/unbound/metrics.go | 65 ---- modules/unbound/testdata/extended_stats.txt | 158 ++++---- modules/unbound/testdata/stats.txt | 106 ++---- modules/unbound/unbound.go | 33 +- 7 files changed, 631 insertions(+), 423 deletions(-) delete mode 100644 modules/unbound/metrics.go diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index 8a156d14a..b465d1eb8 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -1 +1,400 @@ package unbound + +import "github.com/netdata/go-orchestrator/module" + +type ( + // Charts is an alias for module.Charts + Charts = module.Charts + // Dims is an alias for module.Dims + Dims = module.Dims + // Dim is an alias for module.Dim + Dim = module.Dim +) + +var charts = Charts{ + { + ID: "queries", + Title: "Total Queries", + Units: "queries", + Fam: "queries", + Ctx: "unbound.queries", + Dims: Dims{ + {ID: "total.num.queries", Name: "queries"}, + }, + }, + { + ID: "queries_ip_ratelimited", + Title: "Queries IP Rate Limited", + Units: "queries", + Fam: "queries", + Ctx: "unbound.queries_ip_ratelimited", + Dims: Dims{ + {ID: "total.num.queries_ip_ratelimited", Name: "queries"}, + }, + }, + { + ID: "cache", + Title: "Cache", + Units: "events", + Fam: "cache", + Ctx: "unbound.cache", + Type: module.Stacked, + Dims: Dims{ + {ID: "total.num.cachehits", Name: "hits"}, + {ID: "total.num.cachemiss", Name: "miss"}, + }, + }, + { + ID: "cache_percentage", + Title: "Cache Percentage", + Units: "percentage", + Fam: "cache", + Ctx: "unbound.cache_percantage", + Type: module.Stacked, + Dims: Dims{ + {ID: "total.num.cachehits", Name: "hits", Algo: module.PercentOfAbsolute}, + {ID: "total.num.cachemiss", Name: "miss", Algo: module.PercentOfAbsolute}, + }, + }, + { + ID: "queries_prefetch", + Title: "Prefetch Queries", + Units: "queries", + Fam: "cache", + Ctx: "unbound.queries_prefetch", + Dims: Dims{ + {ID: "total.num.prefetch", Name: "queries"}, + }, + }, + { + ID: "zero_ttl_responses", + Title: "Answers Served From Expired Cache", + Units: "responses", + Fam: "cache", + Ctx: "unbound.zero_ttl_responses", + Dims: Dims{ + {ID: "total.num.zero_ttl", Name: "responses"}, + }, + }, + { + ID: "dnscrypt_queries", + Title: "DNSCrypt Queries", + Units: "queries", + Fam: "dnscrypt", + Ctx: "unbound.dnscrypt_queries", + Dims: Dims{ + {ID: "total.num.dnscrypt.crypted", Name: "crypted"}, + {ID: "total.num.dnscrypt.cert", Name: "cert"}, + {ID: "total.num.dnscrypt.cleartext", Name: "cleartext"}, + {ID: "total.num.dnscrypt.malformed", Name: "malformed"}, + }, + }, + { + ID: "recursive_replies", + Title: "The number of replies sent to queries that needed recursive processing", + Units: "responses", + Fam: "responses", + Ctx: "unbound.recursive_replies", + Dims: Dims{ + {ID: "total.num.recursivereplies", Name: "recursive"}, + }, + }, + { + ID: "recursion_time", + Title: "Time t took to answer queries that needed recursive processing", + Units: "milliseconds", + Fam: "responses", + Ctx: "unbound.recursion_time", + Dims: Dims{ + {ID: "total.recursion.time.avg", Name: "avg"}, + {ID: "total.recursion.time.median", Name: "median"}, + }, + }, + { + ID: "request_list_utilization", + Title: "Request List Utilization", + Units: "queries", + Fam: "request list", + Ctx: "unbound.request_list_utilization", + Dims: Dims{ + {ID: "total.requestlist.avg", Name: "avg"}, + {ID: "total.requestlist.max", Name: "max"}, + }, + }, + { + ID: "current_request_list_utilization", + Title: "Current Request List Utilization", + Units: "queries", + Fam: "request list", + Ctx: "unbound.current_request_list_utilization", + Type: module.Area, + Dims: Dims{ + {ID: "total.requestlist.current.all", Name: "all"}, + {ID: "total.requestlist.current.user", Name: "user"}, + }, + }, + { + ID: "request_list_jostle_list", + Title: "Request List Jostle List Events", + Units: "events", + Fam: "request list", + Ctx: "unbound.request_list_jostle_list", + Dims: Dims{ + {ID: "total.requestlist.overwritten", Name: "overwritten"}, + {ID: "total.requestlist.exceeded", Name: "dropped"}, + }, + }, + { + ID: "tcpusage", + Title: "TCP Accept List Usage", + Units: "events", + Fam: "tcpusage", + Ctx: "unbound.tcpusage", + Dims: Dims{ + {ID: "total.tcpusage", Name: "tcpusage"}, + }, + }, + { + ID: "uptime", + Title: "Uptime", + Units: "seconds", + Fam: "uptime", + Ctx: "unbound.uptime", + Dims: Dims{ + {ID: "time.up", Name: "time"}, + }, + }, +} + +var ( + extendedCharts = Charts{ + { + ID: "cache_memory", + Title: "Cache Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.cache_memory", + Dims: Dims{ + {ID: "mem_cache_message", Name: "message"}, + {ID: "mem_cache_rrset", Name: "rrset"}, + {ID: "mem_cache_dnscrypt_nonce", Name: "dnscrypt_nonce"}, + {ID: "mem_cache_dnscrypt_shared_secret", Name: "dnscrypt_shared_secret"}, + }, + }, + { + ID: "mod_memory", + Title: "Module Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.mod_memory", + Dims: Dims{ + {ID: "mem_mod_ipsecmod", Name: "ipsec"}, + {ID: "mem_mod_iterator", Name: "iterator"}, + {ID: "mem_mod_respip", Name: "respip"}, + {ID: "mem_mod_subnet", Name: "subnet"}, + {ID: "mem_mod_validator", Name: "validator"}, + }, + }, + { + ID: "mem_stream_wait", + Title: "TCP and TLS Stream Waif Buffer Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.mem_stream_wait", + Dims: Dims{ + {ID: "mem_stream_wait", Name: "stream_wait"}, + }, + }, + { + ID: "cache_count", + Title: "Cache Count", + Units: "items", + Fam: "cache count", + Ctx: "unbound.cache_count", + Dims: Dims{ + {ID: "cache_count_infra", Name: "infra"}, + {ID: "cache_count_key", Name: "key"}, + {ID: "cache_count_msg", Name: "msg"}, + {ID: "cache_count_rrset", Name: "rrset"}, + {ID: "cache_count_dnscrypt_nonce", Name: "dnscrypt_nonce"}, + {ID: "cache_count_dnscrypt_shared_secret", Name: "shared_secret"}, + }, + }, + { + ID: "query_type", + Title: "Queries By Type", + Units: "queries", + Fam: "query type", + Ctx: "unbound.type_queries", + }, + { + ID: "query_class", + Title: "Queries By Class", + Units: "queries", + Fam: "query class", + Ctx: "unbound.class_queries", + }, + { + ID: "query_opcode", + Title: "Queries By OpCode", + Units: "queries", + Fam: "query opcode", + Ctx: "unbound.opcode_queries", + }, + { + ID: "query_flag", + Title: "Queries By Flag", + Units: "queries", + Fam: "query flag", + Ctx: "unbound.flag_queries", + }, + { + ID: "answer_rcode", + Title: "Answers By Rcode", + Units: "answers", + Fam: "answer rcode", + Ctx: "unbound.rcode_answers", + }, + } +) + +func (u *Unbound) updateCharts() { + if len(u.curCache.threads) > 1 { + for v := range u.curCache.threads { + if !u.cache.threads[v] { + u.cache.threads[v] = true + u.addThreadCharts(v) + } + } + } + // Extended stats contains query flags + if len(u.curCache.queryFlags) == 0 { + return + } + + if !u.hasExtCharts { + charts := extendedCharts.Copy() + if err := u.Charts().Add(*charts...); err != nil { + u.Warning(err) + } + u.hasExtCharts = true + } + + for v := range u.curCache.queryType { + if !u.cache.queryType[v] { + u.cache.queryType[v] = true + u.addDimToQueryTypeChart(v) + } + } + for v := range u.curCache.queryClass { + if !u.cache.queryClass[v] { + u.cache.queryClass[v] = true + u.addDimToQueryClassChart(v) + } + } + for v := range u.curCache.queryOpCode { + if !u.cache.queryOpCode[v] { + u.cache.queryOpCode[v] = true + u.addDimToQueryOpCodeChart(v) + } + } + for v := range u.curCache.queryFlags { + if !u.cache.queryFlags[v] { + u.cache.queryFlags[v] = true + u.addDimToQueryFlagsChart(v) + } + } + for v := range u.curCache.answerRCode { + if !u.cache.answerRCode[v] { + u.cache.answerRCode[v] = true + u.addDimToAnswerRcodeChart(v) + } + } +} + +func newThreadCharts(id string) Charts { + return nil +} + +func (u *Unbound) addThreadCharts(id string) { + charts := newThreadCharts(id) + return + if err := u.Charts().Add(charts...); err != nil { + return + } +} + +func (u *Unbound) addDimToQueryTypeChart(typ string) { + chart := u.Charts().Get("query_type") + if chart == nil { + return + } + dim := &Dim{ + ID: "query_type_" + typ, + Name: typ, + } + if err := chart.AddDim(dim); err != nil { + return + } + chart.MarkNotCreated() +} + +func (u *Unbound) addDimToQueryClassChart(class string) { + chart := u.Charts().Get("query_class") + if chart == nil { + return + } + dim := &Dim{ + ID: "query_class_" + class, + Name: class, + } + if err := chart.AddDim(dim); err != nil { + return + } + chart.MarkNotCreated() +} + +func (u *Unbound) addDimToQueryOpCodeChart(opcode string) { + chart := u.Charts().Get("query_opcode") + if chart == nil { + return + } + dim := &Dim{ + ID: "query_opcode_" + opcode, + Name: opcode, + } + if err := chart.AddDim(dim); err != nil { + return + } + chart.MarkNotCreated() +} + +func (u *Unbound) addDimToQueryFlagsChart(flag string) { + chart := u.Charts().Get("query_flag") + if chart == nil { + return + } + dim := &Dim{ + ID: "query_flag_" + flag, + Name: flag, + } + if err := chart.AddDim(dim); err != nil { + return + } + chart.MarkNotCreated() +} + +func (u *Unbound) addDimToAnswerRcodeChart(rcode string) { + chart := u.Charts().Get("answer_rcode") + if chart == nil { + return + } + dim := &Dim{ + ID: "answer_rcode_" + rcode, + Name: rcode, + } + if err := chart.AddDim(dim); err != nil { + return + } + chart.MarkNotCreated() +} diff --git a/modules/unbound/client.go b/modules/unbound/client.go index 170dda235..e0645b341 100644 --- a/modules/unbound/client.go +++ b/modules/unbound/client.go @@ -65,56 +65,42 @@ func (c *client) disconnect() error { return err } -func (c *client) isConnected() bool { - return c.conn != nil -} - -func (c *client) write(command string) error { +func (c *client) write(command string) (int, error) { err := c.conn.SetWriteDeadline(time.Now().Add(c.timeout)) if err != nil { - return err + return 0, err } - _, err = c.conn.Write([]byte(command)) - return err + return c.conn.Write([]byte(command)) } func (c *client) read() (record []string, err error) { if err = c.conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil { return nil, err } - if c.reuseRecord { record, err = read(c.record, c.conn) c.record = record } else { record, err = read(nil, c.conn) } - return record, err } func (c *client) send(command string) (lines []string, err error) { - if c.isConnected() { - _ = c.disconnect() - } - - err = c.connect() - if err != nil { + if err = c.connect(); err != nil { return nil, err } defer func() { _ = c.disconnect() }() - - err = c.write(command) - if err != nil { + if _, err = c.write(command); err != nil { return nil, err } - return c.read() } -const maxLinesToRead = 500 +// just a high value, number of lines depends on number of threads, every thread adds 20 lines +const maxLinesToRead = 2000 // https://github.com/NLnetLabs/unbound/blob/master/doc/control_proto_spec.txt // Server executes command. And sends reply in ascii text over channel, closes the channel when done. diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index ff2916a6b..b0fd30c2d 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -3,209 +3,134 @@ package unbound import ( "errors" "fmt" - "sort" "strconv" "strings" - - "github.com/netdata/go.d.plugin/pkg/stm" ) -type ( - stats []stat - stat struct { - name string - value float64 - } -) +// https://github.com/NLnetLabs/unbound/blob/master/smallapp/unbound-control.c +// https://github.com/NLnetLabs/unbound/blob/master/libunbound/unbound.h (ub_server_stats, ub_shm_stat_info) +// https://docs.menandmice.com/display/MM/Unbound+request-list+demystified +// https://docs.datadoghq.com/integrations/unbound/#metrics func (u *Unbound) collect() (map[string]int64, error) { - if err := u.collectStats(); err != nil { + stats, err := u.scrapeUnboundStats() + if err != nil { return nil, err } - return stm.ToMap(u.mx), nil + + u.curCache.clear() + mx := u.collectStats(stats) + u.updateCharts() + return mx, nil } -func (u *Unbound) collectStats() error { - resp, err := u.client.send("UBCT1 stats_noreset\n") +func (u *Unbound) scrapeUnboundStats() ([]entry, error) { + output, err := u.client.send("UBCT1 stats\n") if err != nil { - return err + return nil, err } - switch len(resp) { + switch len(output) { case 0: - return errors.New("empty response") + return nil, errors.New("empty response") case 1: - // In case of error the first line of the response is: error \n - // For many commands the response is 'ok\n', but it is not the case for 'stats'. - return errors.New(resp[0]) - } - - stats, err := convertToUnboundStats(resp) - if err != nil { - return err - } - - for _, s := range stats { - u.collectStat(s) - } - return nil -} - -func (u *Unbound) collectStat(s stat) { - switch { - case strings.HasPrefix(s.name, "thread"): - u.collectThread(s) - case strings.HasPrefix(s.name, "total."): - u.collectTotal(s) - case strings.HasPrefix(s.name, "time."): - u.collectTime(s) - case strings.HasPrefix(s.name, "num.query.type."): - u.collectQueryType(s) - case strings.HasPrefix(s.name, "num.query.class."): - u.collectQueryClass(s) - case strings.HasPrefix(s.name, "num.query.opcode."): - u.collectQueryOpCode(s) - case strings.HasPrefix(s.name, "num.answer.rcode."): - u.collectAnswerRCode(s) - case strings.HasPrefix(s.name, "mem."): - u.collectMem(s) + // in case of error the first line of the response is: error \n + return nil, errors.New(output[0]) } + return parseStatsOutput(output) } -func (u *Unbound) collectMem(s stat) { - -} - -func (u *Unbound) collectThread(s stat) { - //thread0.* - i := strings.IndexByte(s.name, '.') - if i == -1 || i < 6 { - return - } - threadID := s.name[6:i] - c, ok := u.mx.Thread[threadID] - if !ok { - c = &common{} - u.mx.Thread[threadID] = c +func (u *Unbound) collectStats(stats []entry) map[string]int64 { + mx := make(map[string]int64, len(stats)) + for _, e := range stats { + if e.hasPrefix("histogram") { + continue + } + // *.requestlist.avg, *.recursion.time.avg, *recursion.time.median + if e.hasSuffix("avg") || e.hasSuffix("median") { + e.value *= 1000 + } + switch { + case e.hasPrefix("thread"): + v := extractThreadID(e.key) + u.curCache.threads[v] = true + case e.hasPrefix("num.query.type"): + v := extractQueryType(e.key) + u.curCache.queryType[v] = true + case e.hasPrefix("num.query.class"): + v := extractQueryClass(e.key) + u.curCache.queryClass[v] = true + case e.hasPrefix("num.query.opcode"): + v := extractQueryOpCode(e.key) + u.curCache.queryOpCode[v] = true + case e.hasPrefix("num.query.flags"): + v := extractQueryFlag(e.key) + u.curCache.queryFlags[v] = true + case e.hasPrefix("num.answer.rcode"): + v := extractAnswerRCode(e.key) + u.curCache.answerRCode[v] = true + } + mx[e.key] = int64(e.value) } - s.name = s.name[i+1:] - collectCommon(c, s) + return mx } -func (u *Unbound) collectTotal(s stat) { - i := strings.IndexByte(s.name, '.') - s.name = s.name[i+1:] - collectCommon(&u.mx.common, s) -} +func extractThreadID(key string) string { idx := strings.IndexByte(key, '.'); return key[6:idx] } +func extractQueryType(key string) string { i := len("num.query.type."); return key[i:] } +func extractQueryClass(key string) string { i := len("num.query.class."); return key[i:] } +func extractQueryOpCode(key string) string { i := len("num.query.opcode."); return key[i:] } +func extractQueryFlag(key string) string { i := len("num.query.flags."); return key[i:] } +func extractAnswerRCode(key string) string { i := len("num.answer.rcode."); return key[i:] } -func (u *Unbound) collectTime(s stat) { - switch s.name { - case "time.up": - u.mx.Uptime = s.value - } +type entry struct { + key string + value float64 } -func (u *Unbound) collectQueryType(s stat) { - i := len("num.query.type.") - typ := s.name[i:] - v, ok := u.mx.QueryType[typ] - if !ok { - //TODO: - } - u.mx.QueryType[typ] += v -} +func (e entry) hasPrefix(prefix string) bool { return strings.HasPrefix(e.key, prefix) } +func (e entry) hasSuffix(suffix string) bool { return strings.HasSuffix(e.key, suffix) } -func (u *Unbound) collectQueryClass(s stat) { - i := len("num.query.class.") - class := s.name[i:] - v, ok := u.mx.QueryClass[class] - if !ok { - //TODO: +func parseStatsOutput(output []string) ([]entry, error) { + var es []entry + for _, v := range output { + e, err := parseStatsLine(v) + if err != nil { + return nil, err + } + es = append(es, e) } - u.mx.QueryClass[class] += v + return es, nil } -func (u *Unbound) collectQueryOpCode(s stat) { - i := len("num.query.opcode.") - opcode := s.name[i:] - v, ok := u.mx.QueryOpCode[opcode] - if !ok { - //TODO: +func parseStatsLine(line string) (entry, error) { + // 'stats' output is a list of [key]=[value] lines. + parts := strings.Split(line, "=") + if len(parts) != 2 { + return entry{}, fmt.Errorf("bad line syntax: %s", line) } - u.mx.QueryOpCode[opcode] += v + f, err := strconv.ParseFloat(parts[1], 10) + return entry{key: parts[0], value: f}, err } -func (u *Unbound) collectAnswerRCode(s stat) { - i := len("num.answer.rcode.") - rcode := s.name[i:] - v, ok := u.mx.AnswerRCode[rcode] - if !ok { - //TODO: +func newCollectCache() collectCache { + return collectCache{ + threads: make(map[string]bool), + queryType: make(map[string]bool), + queryClass: make(map[string]bool), + queryOpCode: make(map[string]bool), + queryFlags: make(map[string]bool), + answerRCode: make(map[string]bool), } - u.mx.AnswerRCode[rcode] += v } -func collectCommon(c *common, s stat) { - switch s.name { - case "num.queries": - c.Queries = s.value - case "num.queries_ip_ratelimited": - c.QueriesIPRL = s.value - case "num.cachehits": - c.Cache.Hits = s.value - case "num.cachemiss": - c.Cache.Miss = s.value - case "num.prefetch": - c.Prefetch = s.value - case "num.zero_ttl": - c.ZeroTTL = s.value - case "num.recursivereplies": - c.RecursiveReplies = s.value - case "num.dnscrypt.crypted": - c.DNSCrypt.Crypted = s.value - case "num.dnscrypt.cert": - c.DNSCrypt.Cert = s.value - case "num.dnscrypt.cleartext": - c.DNSCrypt.ClearText = s.value - case "num.dnscrypt.malformed": - c.DNSCrypt.Malformed = s.value - case "requestlist.avg": - c.RequestList.Avg = s.value - case "requestlist.max": - c.RequestList.Max = s.value - case "requestlist.overwritten": - c.RequestList.Overwritten = s.value - case "requestlist.exceeded": - c.RequestList.Exceeded = s.value - case "requestlist.current.all": - c.RequestList.CurrentAll = s.value - case "requestlist.current.user": - c.RequestList.CurrentUser = s.value - case "recursion.time.avg": - c.RecursionTime.Avg = s.value - case "recursion.time.median": - c.RecursionTime.Median = s.value - case "tcpusage": - c.TCPUsage = s.value - } +type collectCache struct { + threads map[string]bool + queryType map[string]bool + queryClass map[string]bool + queryOpCode map[string]bool + queryFlags map[string]bool + answerRCode map[string]bool } -func convertToUnboundStats(lines []string) (stats, error) { - sort.Strings(lines) - var ubs stats - for _, line := range lines { - // 'stats' output is a list of [name]=[value] lines. - parts := strings.Split(line, "=") - if len(parts) != 2 { - return nil, fmt.Errorf("bad stats syntax: %s", line) - } - - name, value := parts[0], parts[1] - v, err := strconv.ParseFloat(value, 10) - if err != nil { - return nil, err - } - - ubs = append(ubs, stat{name, v}) - } - return ubs, nil +func (c *collectCache) clear() { + *c = newCollectCache() } diff --git a/modules/unbound/metrics.go b/modules/unbound/metrics.go deleted file mode 100644 index 3911a2c67..000000000 --- a/modules/unbound/metrics.go +++ /dev/null @@ -1,65 +0,0 @@ -package unbound - -// https://github.com/NLnetLabs/unbound/blob/master/smallapp/unbound-control.c - -type ( - cache struct { - Hits float64 `stm:"hits"` - Miss float64 `stm:"miss"` - } - dnsCrypt struct { - Crypted float64 `stm:"crypted"` - Cert float64 `stm:"cert"` - ClearText float64 `stm:"clear_text"` - Malformed float64 `stm:"malformed"` - } - requestList struct { - Avg float64 `stm:"avg"` - Max float64 `stm:"max"` - Overwritten float64 `stm:"overwritten"` - Exceeded float64 `stm:"exceeded"` - CurrentAll float64 `stm:"current_all"` - CurrentUser float64 `stm:"current_user"` - } - recursionTime struct { - Avg float64 `stm:"avg"` - Median float64 `stm:"median"` - } - common struct { - Queries float64 `stm:"queries"` - QueriesIPRL float64 `stm:"queries_ip_ratelimited"` - Cache cache `stm:"cache"` - Prefetch float64 `stm:"prefetch"` - ZeroTTL float64 `stm:"zero_ttl"` - RecursiveReplies float64 `stm:"recursive_replies"` - DNSCrypt dnsCrypt `stm:"dns_crypt"` - RequestList requestList `stm:"request_list"` - RecursionTime recursionTime `stm:"recursion_time"` - TCPUsage float64 `stm:"tcp_usage"` - } - extended struct { - QueryType map[string]float64 `stm:"query_type"` - QueryClass map[string]float64 `stm:"query_class"` - QueryOpCode map[string]float64 `stm:"query_opcode"` - AnswerRCode map[string]float64 `stm:"answer_rcode"` - } -) - -type metricsData struct { - common `stm:"total"` - extended `stm:""` - Uptime float64 `stm:"uptime"` - Thread map[string]*common `stm:"thread"` -} - -func newMetricsData() *metricsData { - return &metricsData{ - extended: extended{ - QueryType: make(map[string]float64), - QueryClass: make(map[string]float64), - QueryOpCode: make(map[string]float64), - AnswerRCode: make(map[string]float64), - }, - Thread: make(map[string]*common), - } -} diff --git a/modules/unbound/testdata/extended_stats.txt b/modules/unbound/testdata/extended_stats.txt index 8fa6258a5..4431315b7 100644 --- a/modules/unbound/testdata/extended_stats.txt +++ b/modules/unbound/testdata/extended_stats.txt @@ -1,93 +1,56 @@ -thread0.num.queries=1 +thread0.num.queries=23 thread0.num.queries_ip_ratelimited=0 -thread0.num.cachehits=0 -thread0.num.cachemiss=1 +thread0.num.cachehits=10 +thread0.num.cachemiss=13 thread0.num.prefetch=0 thread0.num.zero_ttl=0 -thread0.num.recursivereplies=1 -thread0.requestlist.avg=0 -thread0.requestlist.max=0 +thread0.num.recursivereplies=13 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=1 +thread0.requestlist.max=5 thread0.requestlist.overwritten=0 thread0.requestlist.exceeded=0 thread0.requestlist.current.all=0 thread0.requestlist.current.user=0 -thread0.recursion.time.avg=0.003121 -thread0.recursion.time.median=0 +thread0.recursion.time.avg=0.078938 +thread0.recursion.time.median=0.049152 thread0.tcpusage=0 -thread1.num.queries=0 -thread1.num.queries_ip_ratelimited=0 -thread1.num.cachehits=0 -thread1.num.cachemiss=0 -thread1.num.prefetch=0 -thread1.num.zero_ttl=0 -thread1.num.recursivereplies=0 -thread1.requestlist.avg=0 -thread1.requestlist.max=0 -thread1.requestlist.overwritten=0 -thread1.requestlist.exceeded=0 -thread1.requestlist.current.all=0 -thread1.requestlist.current.user=0 -thread1.recursion.time.avg=0.000000 -thread1.recursion.time.median=0 -thread1.tcpusage=0 -thread2.num.queries=0 -thread2.num.queries_ip_ratelimited=0 -thread2.num.cachehits=0 -thread2.num.cachemiss=0 -thread2.num.prefetch=0 -thread2.num.zero_ttl=0 -thread2.num.recursivereplies=0 -thread2.requestlist.avg=0 -thread2.requestlist.max=0 -thread2.requestlist.overwritten=0 -thread2.requestlist.exceeded=0 -thread2.requestlist.current.all=0 -thread2.requestlist.current.user=0 -thread2.recursion.time.avg=0.000000 -thread2.recursion.time.median=0 -thread2.tcpusage=0 -thread3.num.queries=0 -thread3.num.queries_ip_ratelimited=0 -thread3.num.cachehits=0 -thread3.num.cachemiss=0 -thread3.num.prefetch=0 -thread3.num.zero_ttl=0 -thread3.num.recursivereplies=0 -thread3.requestlist.avg=0 -thread3.requestlist.max=0 -thread3.requestlist.overwritten=0 -thread3.requestlist.exceeded=0 -thread3.requestlist.current.all=0 -thread3.requestlist.current.user=0 -thread3.recursion.time.avg=0.000000 -thread3.recursion.time.median=0 -thread3.tcpusage=0 -total.num.queries=1 +total.num.queries=23 total.num.queries_ip_ratelimited=0 -total.num.cachehits=0 -total.num.cachemiss=1 +total.num.cachehits=10 +total.num.cachemiss=13 total.num.prefetch=0 total.num.zero_ttl=0 -total.num.recursivereplies=1 -total.requestlist.avg=0 -total.requestlist.max=0 +total.num.recursivereplies=13 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=1 +total.requestlist.max=5 total.requestlist.overwritten=0 total.requestlist.exceeded=0 total.requestlist.current.all=0 total.requestlist.current.user=0 -total.recursion.time.avg=0.003121 -total.recursion.time.median=0 +total.recursion.time.avg=0.078938 +total.recursion.time.median=0.049152 total.tcpusage=0 -time.now=1571406665.059510 -time.up=11.768185 -time.elapsed=0.801449 -mem.cache.rrset=75653 -mem.cache.message=72418 +time.now=1573817686.220375 +time.up=1206.824144 +time.elapsed=1128.946812 +mem.cache.rrset=104566 +mem.cache.message=76765 mem.mod.iterator=16588 -mem.mod.validator=0 +mem.mod.validator=72332 mem.mod.respip=0 -mem.mod.subnet=0 -histogram.000000.000000.to.000000.000001=0 +mem.mod.subnet=74504 +mem.cache.dnscrypt_shared_secret=0 +mem.cache.dnscrypt_nonce=0 +mem.streamwait=0 +histogram.000000.000000.to.000000.000001=1 histogram.000000.000001.to.000000.000002=0 histogram.000000.000002.to.000000.000004=0 histogram.000000.000004.to.000000.000008=0 @@ -99,14 +62,14 @@ histogram.000000.000128.to.000000.000256=0 histogram.000000.000256.to.000000.000512=0 histogram.000000.000512.to.000000.001024=0 histogram.000000.001024.to.000000.002048=0 -histogram.000000.002048.to.000000.004096=1 +histogram.000000.002048.to.000000.004096=0 histogram.000000.004096.to.000000.008192=0 -histogram.000000.008192.to.000000.016384=0 -histogram.000000.016384.to.000000.032768=0 -histogram.000000.032768.to.000000.065536=0 -histogram.000000.065536.to.000000.131072=0 -histogram.000000.131072.to.000000.262144=0 -histogram.000000.262144.to.000000.524288=0 +histogram.000000.008192.to.000000.016384=3 +histogram.000000.016384.to.000000.032768=1 +histogram.000000.032768.to.000000.065536=3 +histogram.000000.065536.to.000000.131072=2 +histogram.000000.131072.to.000000.262144=2 +histogram.000000.262144.to.000000.524288=1 histogram.000000.524288.to.000001.000000=0 histogram.000001.000000.to.000002.000000=0 histogram.000002.000000.to.000004.000000=0 @@ -127,35 +90,50 @@ histogram.032768.000000.to.065536.000000=0 histogram.065536.000000.to.131072.000000=0 histogram.131072.000000.to.262144.000000=0 histogram.262144.000000.to.524288.000000=0 -num.query.type.A=1 -num.query.class.IN=1 -num.query.opcode.QUERY=1 +num.query.type.A=12 +num.query.type.PTR=1 +num.query.type.MX=5 +num.query.type.AAAA=5 +num.query.class.IN=23 +num.query.opcode.QUERY=23 num.query.tcp=0 num.query.tcpout=0 +num.query.tls=0 +num.query.tls.resume=0 num.query.ipv6=0 num.query.flags.QR=0 num.query.flags.AA=0 num.query.flags.TC=0 -num.query.flags.RD=1 +num.query.flags.RD=23 num.query.flags.RA=0 num.query.flags.Z=0 num.query.flags.AD=0 num.query.flags.CD=0 num.query.edns.present=0 num.query.edns.DO=0 -num.answer.rcode.NOERROR=1 +num.answer.rcode.NOERROR=15 num.answer.rcode.FORMERR=0 num.answer.rcode.SERVFAIL=0 -num.answer.rcode.NXDOMAIN=0 +num.answer.rcode.NXDOMAIN=8 num.answer.rcode.NOTIMPL=0 num.answer.rcode.REFUSED=0 num.query.ratelimited=0 -num.answer.secure=0 +num.answer.secure=7 num.answer.bogus=0 num.rrset.bogus=0 +num.query.aggressive.NOERROR=0 +num.query.aggressive.NXDOMAIN=0 unwanted.queries=0 unwanted.replies=0 -msg.cache.count=23 -rrset.cache.count=32 -infra.cache.count=4 -key.cache.count=0 \ No newline at end of file +msg.cache.count=33 +rrset.cache.count=124 +infra.cache.count=58 +key.cache.count=7 +dnscrypt_shared_secret.cache.count=0 +dnscrypt_nonce.cache.count=0 +num.query.dnscrypt.shared_secret.cachemiss=0 +num.query.dnscrypt.replay=0 +num.query.authzone.up=0 +num.query.authzone.down=0 +num.query.subnet=0 +num.query.subnet_cache=0 \ No newline at end of file diff --git a/modules/unbound/testdata/stats.txt b/modules/unbound/testdata/stats.txt index 02ad3d0e0..51cdef163 100644 --- a/modules/unbound/testdata/stats.txt +++ b/modules/unbound/testdata/stats.txt @@ -1,83 +1,49 @@ -thread0.num.queries=1 +thread0.num.queries=23 thread0.num.queries_ip_ratelimited=0 -thread0.num.cachehits=0 -thread0.num.cachemiss=1 +thread0.num.cachehits=10 +thread0.num.cachemiss=13 thread0.num.prefetch=0 thread0.num.zero_ttl=0 -thread0.num.recursivereplies=0 +thread0.num.recursivereplies=13 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 thread0.requestlist.avg=1 -thread0.requestlist.max=1 +thread0.requestlist.max=5 thread0.requestlist.overwritten=0 thread0.requestlist.exceeded=0 -thread0.requestlist.current.all=1 -thread0.requestlist.current.user=1 -thread0.recursion.time.avg=0.000000 -thread0.recursion.time.median=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.078938 +thread0.recursion.time.median=0.049152 thread0.tcpusage=0 -thread1.num.queries=2 -thread1.num.queries_ip_ratelimited=0 -thread1.num.cachehits=2 -thread1.num.cachemiss=0 -thread1.num.prefetch=0 -thread1.num.zero_ttl=0 -thread1.num.recursivereplies=0 -thread1.requestlist.avg=0 -thread1.requestlist.max=0 -thread1.requestlist.overwritten=0 -thread1.requestlist.exceeded=0 -thread1.requestlist.current.all=0 -thread1.requestlist.current.user=0 -thread1.recursion.time.avg=0.000000 -thread1.recursion.time.median=0 -thread1.tcpusage=0 -thread2.num.queries=1 -thread2.num.queries_ip_ratelimited=0 -thread2.num.cachehits=0 -thread2.num.cachemiss=1 -thread2.num.prefetch=0 -thread2.num.zero_ttl=0 -thread2.num.recursivereplies=1 -thread2.requestlist.avg=0 -thread2.requestlist.max=0 -thread2.requestlist.overwritten=0 -thread2.requestlist.exceeded=0 -thread2.requestlist.current.all=0 -thread2.requestlist.current.user=0 -thread2.recursion.time.avg=0.236823 -thread2.recursion.time.median=0 -thread2.tcpusage=0 -thread3.num.queries=0 -thread3.num.queries_ip_ratelimited=0 -thread3.num.cachehits=0 -thread3.num.cachemiss=0 -thread3.num.prefetch=0 -thread3.num.zero_ttl=0 -thread3.num.recursivereplies=0 -thread3.requestlist.avg=0 -thread3.requestlist.max=0 -thread3.requestlist.overwritten=0 -thread3.requestlist.exceeded=0 -thread3.requestlist.current.all=0 -thread3.requestlist.current.user=0 -thread3.recursion.time.avg=0.000000 -thread3.recursion.time.median=0 -thread3.tcpusage=0 -total.num.queries=4 + +total.num.queries=23 total.num.queries_ip_ratelimited=0 -total.num.cachehits=2 -total.num.cachemiss=2 + +total.num.cachehits=10 +total.num.cachemiss=13 total.num.prefetch=0 total.num.zero_ttl=0 -total.num.recursivereplies=1 -total.requestlist.avg=0.5 -total.requestlist.max=1 + +total.num.recursivereplies=13 + +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 + +total.requestlist.avg=1 +total.requestlist.max=5 total.requestlist.overwritten=0 total.requestlist.exceeded=0 -total.requestlist.current.all=1 -total.requestlist.current.user=1 -total.recursion.time.avg=0.236823 -total.recursion.time.median=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 + +total.recursion.time.avg=0.078938 +total.recursion.time.median=0.049152 total.tcpusage=0 -time.now=1571405114.158591 -time.up=31.232563 -time.elapsed=0.904431 \ No newline at end of file +time.now=1573817686.220375 +time.up=1206.824144 +time.elapsed=1128.946812 \ No newline at end of file diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 70a8f6055..dcbc87765 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -1,6 +1,8 @@ package unbound import ( + "time" + "github.com/netdata/go.d.plugin/pkg/web" "github.com/netdata/go-orchestrator/module" @@ -15,11 +17,16 @@ func init() { } func New() *Unbound { - config := Config{} + config := Config{ + Address: "192.168.88.223:8953", + Timeout: web.Duration{Duration: time.Second * 2}, + } return &Unbound{ - Config: config, - charts: nil, + Config: config, + charts: charts.Copy(), + curCache: newCollectCache(), + cache: newCollectCache(), } } @@ -38,20 +45,32 @@ type ( module.Base Config `yaml:",inline"` - client unboundClient + client unboundClient + cache collectCache + curCache collectCache + + cumulative bool + hasExtCharts bool + charts *module.Charts - mx *metricsData } ) func (Unbound) Cleanup() {} func (u *Unbound) Init() bool { - u.mx = &metricsData{} + u.client = newClient(clientConfig{ + address: u.Address, + timeout: u.Timeout.Duration, + useTLS: false, + tlsConf: nil, + }) return true } -func (u Unbound) Check() bool { return false } +func (u Unbound) Check() bool { + return true +} func (u Unbound) Charts() *module.Charts { return u.charts } From 30d0bdcd8665d450dbc32bca11867a6331265b66 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Sat, 16 Nov 2019 17:46:12 +0300 Subject: [PATCH 06/39] collect wip --- modules/unbound/collect.go | 37 +++++++++--- modules/unbound/testdata/unbound.conf | 85 +++++++++++++++++++++++++++ modules/unbound/unbound.go | 5 +- 3 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 modules/unbound/testdata/unbound.conf diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index b0fd30c2d..33a8bdfeb 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -1,7 +1,6 @@ package unbound import ( - "errors" "fmt" "strconv" "strings" @@ -25,29 +24,39 @@ func (u *Unbound) collect() (map[string]int64, error) { } func (u *Unbound) scrapeUnboundStats() ([]entry, error) { - output, err := u.client.send("UBCT1 stats\n") + const command = "UBCT1 stats" + output, err := u.client.send(command + "\n") if err != nil { return nil, err } switch len(output) { case 0: - return nil, errors.New("empty response") + return nil, fmt.Errorf("command '%s': empty resopnse", command) case 1: // in case of error the first line of the response is: error \n - return nil, errors.New(output[0]) + return nil, fmt.Errorf("command '%s': '%s'", command, output[0]) } return parseStatsOutput(output) } func (u *Unbound) collectStats(stats []entry) map[string]int64 { + var resetTimings bool + if u.cumulative { + n := findTotalQueries(stats) + resetTimings = n > 0 && n == u.prevTotQueries + u.prevTotQueries = n + } + mx := make(map[string]int64, len(stats)) + for _, e := range stats { - if e.hasPrefix("histogram") { - continue - } // *.requestlist.avg, *.recursion.time.avg, *recursion.time.median if e.hasSuffix("avg") || e.hasSuffix("median") { - e.value *= 1000 + if resetTimings { + e.value = 0 + } else { + e.value *= 1000 + } } switch { case e.hasPrefix("thread"): @@ -74,6 +83,15 @@ func (u *Unbound) collectStats(stats []entry) map[string]int64 { return mx } +func findTotalQueries(stats []entry) float64 { + for _, e := range stats { + if e.key == "total.num.queries" { + return e.value + } + } + return -1 +} + func extractThreadID(key string) string { idx := strings.IndexByte(key, '.'); return key[6:idx] } func extractQueryType(key string) string { i := len("num.query.type."); return key[i:] } func extractQueryClass(key string) string { i := len("num.query.class."); return key[i:] } @@ -96,6 +114,9 @@ func parseStatsOutput(output []string) ([]entry, error) { if err != nil { return nil, err } + if e.hasPrefix("histogram") { + continue + } es = append(es, e) } return es, nil diff --git a/modules/unbound/testdata/unbound.conf b/modules/unbound/testdata/unbound.conf new file mode 100644 index 000000000..bcef643e8 --- /dev/null +++ b/modules/unbound/testdata/unbound.conf @@ -0,0 +1,85 @@ +# +# Example configuration file. +# +# See unbound.conf(5) man page, version 1.9.4. +# +# this is a comment. + +#Use this to include other text into the file. +#include: "otherfile.conf" + +# The server clause sets the main parameters. +server: + # whitespace is not necessary, but looks cleaner. + + # verbosity number, 0 is least verbose. 1 is default. + # verbosity: 1 + + # print statistics to the log (for every thread) every N seconds. + # Set to "" or 0 to disable. Default is disabled. + # statistics-interval: 0 + + # enable shm for stats, default no. if you enable also enable + # statistics-interval, every time it also writes stats to the + # shared memory segment keyed with shm-key. + # shm-enable: no + + # shm for stats uses this key, and key+1 for the shared mem segment. + # shm-key: 11777 + + # enable cumulative statistics, without clearing them after printing. + # statistics-cumulative: no + statistics-cumulative: yes + + # enable extended statistics (query types, answer codes, status) + # printed from unbound-control. default off, because of speed. + # extended-statistics: no + extended-statistics: yes + + # number of threads to create. 1 disables threading. + num-threads: 2 + +# Python config section. To enable: +# o use --with-pythonmodule to configure before compiling. +# o list python in the module-config string (above) to enable. +# It can be at the start, it gets validated results, or just before +# the iterator and process before DNSSEC validation. +# o and give a python-script to run. +python: + # Script file to load + # python-script: "/etc/unbound/ubmodule-tst.py" + +# Remote control config section. +remote-control: + # Enable remote control with unbound-control(8) here. + # set up the keys and certificates with unbound-control-setup. + control-enable: yes + + # what interfaces are listened to for remote control. + # give 0.0.0.0 and ::0 to listen to all interfaces. + # set to an absolute path to use a unix local name pipe, certificates + # are not used for that, so key and cert files need not be present. + # control-interface: 127.0.0.1 + control-interface: 0.0.0.0 + # control-interface: ::1 + #control-interface: /var/run/test.sock + + # port number for remote control operations. + control-port: 8953 + + # for localhost, you can disable use of TLS by setting this to "no" + # For local sockets this option is ignored, and TLS is not used. + # control-use-cert: "yes" + control-use-cert: "yes" + + # unbound server key file. + server-key-file: "/etc/unbound/unbound_server.key" + + # unbound server certificate file. + server-cert-file: "/etc/unbound/unbound_server.pem" + + # unbound-control key file. + control-key-file: "/etc/unbound/unbound_control.key" + + # unbound-control certificate file. + control-cert-file: "/etc/unbound/unbound_control.pem" diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index dcbc87765..151adabb7 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -49,8 +49,9 @@ type ( cache collectCache curCache collectCache - cumulative bool - hasExtCharts bool + prevTotQueries float64 + cumulative bool + hasExtCharts bool charts *module.Charts } From ff33329f3caf41a7bc0ffca1ca8b1d79876a0e13 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Sat, 16 Nov 2019 20:05:20 +0300 Subject: [PATCH 07/39] collect wip --- modules/unbound/collect.go | 2 +- modules/unbound/config.go | 64 +++++++++++++++++++++++++++ modules/unbound/testdata/unbound.conf | 6 +-- modules/unbound/unbound.go | 57 ++++++++++++++++++++---- 4 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 modules/unbound/config.go diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 33a8bdfeb..d144d23e8 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -41,7 +41,7 @@ func (u *Unbound) scrapeUnboundStats() ([]entry, error) { func (u *Unbound) collectStats(stats []entry) map[string]int64 { var resetTimings bool - if u.cumulative { + if u.Cumulative { n := findTotalQueries(stats) resetTimings = n > 0 && n == u.prevTotQueries u.prevTotQueries = n diff --git a/modules/unbound/config.go b/modules/unbound/config.go new file mode 100644 index 000000000..3e5f7d971 --- /dev/null +++ b/modules/unbound/config.go @@ -0,0 +1,64 @@ +package unbound + +import ( + "bytes" + "fmt" + "io/ioutil" + + "gopkg.in/yaml.v2" +) + +type ( + str string + strBool string +) + +func (s str) isSet() bool { return s != "" } +func (s strBool) isSet() bool { return s != "" } +func (s strBool) bool() bool { return s == "yes" } + +type unboundConfig struct { + Server *struct { + StatisticsCumulative strBool `yaml:"statistics-cumulative"` + } `yaml:"server"` + RemoteControl *struct { + ControlEnable strBool `yaml:"control-enable"` + ControlInterface str `yaml:"control-interface"` + ControlPort str `yaml:"control-port"` + ControlUseCert strBool `yaml:"control-use-cert"` + ControlKeyFile str `yaml:"control-key-file"` + ControlCertFile str `yaml:"control-cert-file"` + } `yaml:"remote-control"` +} + +func (c unboundConfig) hasServerSection() bool { + return c.Server != nil +} +func (c unboundConfig) hasRemoteControlSection() bool { + return c.RemoteControl != nil +} +func (c unboundConfig) isRemoteControlDisabled() bool { + return c.hasRemoteControlSection() && c.RemoteControl.ControlEnable.isSet() && c.RemoteControl.ControlEnable.bool() +} + +func adjustUnboundConfig(cfg []byte) []byte { + return bytes.ReplaceAll(cfg, []byte("\t"), []byte(" ")) +} + +func readConfig(config string) (*unboundConfig, error) { + if config == "" { + return nil, nil + } + b, err := ioutil.ReadFile(config) + if err != nil { + return nil, fmt.Errorf("error on reading config: %v", err) + } + + b = adjustUnboundConfig(b) + + var cfg unboundConfig + if err := yaml.Unmarshal(b, &cfg); err != nil { + return nil, fmt.Errorf("error on parsing config: %v", err) + } + return &cfg, nil +} diff --git a/modules/unbound/testdata/unbound.conf b/modules/unbound/testdata/unbound.conf index bcef643e8..31a7a59c9 100644 --- a/modules/unbound/testdata/unbound.conf +++ b/modules/unbound/testdata/unbound.conf @@ -62,7 +62,7 @@ remote-control: # control-interface: 127.0.0.1 control-interface: 0.0.0.0 # control-interface: ::1 - #control-interface: /var/run/test.sock + # control-interface: /var/run/test.sock # port number for remote control operations. control-port: 8953 @@ -73,10 +73,10 @@ remote-control: control-use-cert: "yes" # unbound server key file. - server-key-file: "/etc/unbound/unbound_server.key" + # server-key-file: "/etc/unbound/unbound_server.key" # unbound server certificate file. - server-cert-file: "/etc/unbound/unbound_server.pem" + # server-cert-file: "/etc/unbound/unbound_server.pem" # unbound-control key file. control-key-file: "/etc/unbound/unbound_control.key" diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 151adabb7..9b9d61c8c 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -18,8 +18,10 @@ func init() { func New() *Unbound { config := Config{ - Address: "192.168.88.223:8953", - Timeout: web.Duration{Duration: time.Second * 2}, + // "/etc/unbound/unbound.conf" + Address: "192.168.88.223:8953", + ConfPath: "/Users/ilyam/Projects/goland/go.d.plugin/modules/unbound/testdata/unbound.conf", + Timeout: web.Duration{Duration: time.Second * 2}, } return &Unbound{ @@ -39,6 +41,8 @@ type ( Address string `yaml:"address"` ConfPath string `yaml:"conf_path"` Timeout web.Duration `yaml:"timeout"` + DisableTLS bool `yaml:"disable_tls"` + Cumulative bool `yaml:"cumulative_stats"` web.ClientTLSConfig `yaml:",inline"` } Unbound struct { @@ -50,7 +54,6 @@ type ( curCache collectCache prevTotQueries float64 - cumulative bool hasExtCharts bool charts *module.Charts @@ -60,12 +63,19 @@ type ( func (Unbound) Cleanup() {} func (u *Unbound) Init() bool { - u.client = newClient(clientConfig{ - address: u.Address, - timeout: u.Timeout.Duration, - useTLS: false, - tlsConf: nil, - }) + cfg, err := readConfig(u.ConfPath) + if err != nil { + u.Warning(err) + } + if cfg != nil { + if cfg.isRemoteControlDisabled() { + u.Info("") + return false + } + u.applyConfig(cfg) + } + + u.client = u.createClient() return true } @@ -88,3 +98,32 @@ func (u *Unbound) Collect() map[string]int64 { return mx } + +func (u *Unbound) applyConfig(cfg *unboundConfig) { + if cfg.hasServerSection() { + if cfg.Server.StatisticsCumulative.isSet() { + u.Cumulative = cfg.Server.StatisticsCumulative.bool() + } + } + if !cfg.hasRemoteControlSection() { + return + } + if cfg.RemoteControl.ControlUseCert.isSet() { + u.DisableTLS = cfg.RemoteControl.ControlUseCert.bool() + } + if cfg.RemoteControl.ControlKeyFile.isSet() { + u.TLSKey = string(cfg.RemoteControl.ControlKeyFile) + } + if cfg.RemoteControl.ControlCertFile.isSet() { + u.TLSCert = string(cfg.RemoteControl.ControlCertFile) + } +} + +func (u *Unbound) createClient() unboundClient { + return newClient(clientConfig{ + address: u.Address, + timeout: u.Timeout.Duration, + useTLS: false, + tlsConf: nil, + }) +} From 7aa5769e6847742793b4067c009adc5dabea6a36 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Sat, 16 Nov 2019 20:57:44 +0300 Subject: [PATCH 08/39] collect wip --- modules/unbound/config.go | 2 +- modules/unbound/unbound.go | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/modules/unbound/config.go b/modules/unbound/config.go index 3e5f7d971..722bbdc87 100644 --- a/modules/unbound/config.go +++ b/modules/unbound/config.go @@ -38,7 +38,7 @@ func (c unboundConfig) hasRemoteControlSection() bool { return c.RemoteControl != nil } func (c unboundConfig) isRemoteControlDisabled() bool { - return c.hasRemoteControlSection() && c.RemoteControl.ControlEnable.isSet() && c.RemoteControl.ControlEnable.bool() + return c.hasRemoteControlSection() && c.RemoteControl.ControlEnable.isSet() && !c.RemoteControl.ControlEnable.bool() } func adjustUnboundConfig(cfg []byte) []byte { diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 9b9d61c8c..485db79cf 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -1,6 +1,8 @@ package unbound import ( + "crypto/tls" + "strings" "time" "github.com/netdata/go.d.plugin/pkg/web" @@ -69,25 +71,31 @@ func (u *Unbound) Init() bool { } if cfg != nil { if cfg.isRemoteControlDisabled() { - u.Info("") + u.Info("remote control is disabled in the configuration file") return false } u.applyConfig(cfg) } - u.client = u.createClient() + cl, err := u.createClient() + if err != nil { + u.Errorf("creating client: %v", err) + return false + } + u.client = cl return true } -func (u Unbound) Check() bool { - return true +func (u *Unbound) Check() bool { + return len(u.Collect()) > 0 } -func (u Unbound) Charts() *module.Charts { return u.charts } +func (u Unbound) Charts() *module.Charts { + return u.charts +} func (u *Unbound) Collect() map[string]int64 { mx, err := u.collect() - if err != nil { u.Error(err) } @@ -119,11 +127,21 @@ func (u *Unbound) applyConfig(cfg *unboundConfig) { } } -func (u *Unbound) createClient() unboundClient { - return newClient(clientConfig{ +func (u *Unbound) createClient() (uClient unboundClient, err error) { + useTLS := !isUnixSocket(u.Address) && !u.DisableTLS + var tlsCfg *tls.Config + if useTLS { + if tlsCfg, err = web.NewTLSConfig(u.ClientTLSConfig); err != nil { + return nil, err + } + } + uClient = newClient(clientConfig{ address: u.Address, timeout: u.Timeout.Duration, - useTLS: false, - tlsConf: nil, + useTLS: useTLS, + tlsConf: tlsCfg, }) + return uClient, err } + +func isUnixSocket(address string) bool { return strings.HasPrefix(address, "/") } From 4e71e42b06d9a8c5008390a51f225409f31b85cc Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Sat, 16 Nov 2019 21:32:27 +0300 Subject: [PATCH 09/39] move init stuff to init.go --- modules/unbound/config.go | 64 ------------------ modules/unbound/init.go | 135 +++++++++++++++++++++++++++++++++++++ modules/unbound/unbound.go | 57 +--------------- 3 files changed, 138 insertions(+), 118 deletions(-) delete mode 100644 modules/unbound/config.go create mode 100644 modules/unbound/init.go diff --git a/modules/unbound/config.go b/modules/unbound/config.go deleted file mode 100644 index 722bbdc87..000000000 --- a/modules/unbound/config.go +++ /dev/null @@ -1,64 +0,0 @@ -package unbound - -import ( - "bytes" - "fmt" - "io/ioutil" - - "gopkg.in/yaml.v2" -) - -type ( - str string - strBool string -) - -func (s str) isSet() bool { return s != "" } -func (s strBool) isSet() bool { return s != "" } -func (s strBool) bool() bool { return s == "yes" } - -type unboundConfig struct { - Server *struct { - StatisticsCumulative strBool `yaml:"statistics-cumulative"` - } `yaml:"server"` - RemoteControl *struct { - ControlEnable strBool `yaml:"control-enable"` - ControlInterface str `yaml:"control-interface"` - ControlPort str `yaml:"control-port"` - ControlUseCert strBool `yaml:"control-use-cert"` - ControlKeyFile str `yaml:"control-key-file"` - ControlCertFile str `yaml:"control-cert-file"` - } `yaml:"remote-control"` -} - -func (c unboundConfig) hasServerSection() bool { - return c.Server != nil -} -func (c unboundConfig) hasRemoteControlSection() bool { - return c.RemoteControl != nil -} -func (c unboundConfig) isRemoteControlDisabled() bool { - return c.hasRemoteControlSection() && c.RemoteControl.ControlEnable.isSet() && !c.RemoteControl.ControlEnable.bool() -} - -func adjustUnboundConfig(cfg []byte) []byte { - return bytes.ReplaceAll(cfg, []byte("\t"), []byte(" ")) -} - -func readConfig(config string) (*unboundConfig, error) { - if config == "" { - return nil, nil - } - b, err := ioutil.ReadFile(config) - if err != nil { - return nil, fmt.Errorf("error on reading config: %v", err) - } - - b = adjustUnboundConfig(b) - - var cfg unboundConfig - if err := yaml.Unmarshal(b, &cfg); err != nil { - return nil, fmt.Errorf("error on parsing config: %v", err) - } - return &cfg, nil -} diff --git a/modules/unbound/init.go b/modules/unbound/init.go new file mode 100644 index 000000000..51e5e4c3e --- /dev/null +++ b/modules/unbound/init.go @@ -0,0 +1,135 @@ +package unbound + +import ( + "bytes" + "crypto/tls" + "fmt" + "io/ioutil" + "net" + "strings" + + "github.com/netdata/go.d.plugin/pkg/web" + + "gopkg.in/yaml.v2" +) + +type ( + str string + strBool string +) + +func (s str) isSet() bool { return s != "" } +func (s strBool) isSet() bool { return s != "" } +func (s strBool) bool() bool { return s == "yes" } + +type unboundConfig struct { + Server *struct { + StatisticsCumulative strBool `yaml:"statistics-cumulative"` + } `yaml:"server"` + RemoteControl *struct { + ControlEnable strBool `yaml:"control-enable"` + ControlInterface str `yaml:"control-interface"` + ControlPort str `yaml:"control-port"` + ControlUseCert strBool `yaml:"control-use-cert"` + ControlKeyFile str `yaml:"control-key-file"` + ControlCertFile str `yaml:"control-cert-file"` + } `yaml:"remote-control"` +} + +func (c unboundConfig) hasServerSection() bool { + return c.Server != nil +} +func (c unboundConfig) hasRemoteControlSection() bool { + return c.RemoteControl != nil +} +func (c unboundConfig) isRemoteControlDisabled() bool { + return c.hasRemoteControlSection() && c.RemoteControl.ControlEnable.isSet() && !c.RemoteControl.ControlEnable.bool() +} + +func (u *Unbound) initConfig() bool { + if u.ConfPath == "" { + return true + } + cfg, err := readConfig(u.ConfPath) + if err != nil { + u.Warning(err) + } + if cfg == nil { + return true + } + if cfg.isRemoteControlDisabled() { + u.Info("remote control is disabled in the configuration file") + return false + } + u.applyConfig(cfg) + return true +} + +func (u *Unbound) applyConfig(cfg *unboundConfig) { + if cfg.hasServerSection() { + if cfg.Server.StatisticsCumulative.isSet() { + u.Cumulative = cfg.Server.StatisticsCumulative.bool() + } + } + if !cfg.hasRemoteControlSection() { + return + } + if cfg.RemoteControl.ControlUseCert.isSet() { + u.DisableTLS = cfg.RemoteControl.ControlUseCert.bool() + } + if cfg.RemoteControl.ControlKeyFile.isSet() { + u.TLSKey = string(cfg.RemoteControl.ControlKeyFile) + } + if cfg.RemoteControl.ControlCertFile.isSet() { + u.TLSCert = string(cfg.RemoteControl.ControlCertFile) + } + if cfg.RemoteControl.ControlInterface.isSet() { + u.Address = string(cfg.RemoteControl.ControlInterface) + } + if cfg.RemoteControl.ControlPort.isSet() && !isUnixSocket(u.Address) { + host, _, _ := net.SplitHostPort(u.Address) + u.Address = net.JoinHostPort(host, string(cfg.RemoteControl.ControlPort)) + } +} + +func (u *Unbound) initClient() (err error) { + var tlsCfg *tls.Config + + useTLS := !isUnixSocket(u.Address) && !u.DisableTLS + if useTLS { + if tlsCfg, err = web.NewTLSConfig(u.ClientTLSConfig); err != nil { + return err + } + } + + u.client = newClient(clientConfig{ + address: u.Address, + timeout: u.Timeout.Duration, + useTLS: useTLS, + tlsConf: tlsCfg, + }) + return nil +} + +func isUnixSocket(address string) bool { + return strings.HasPrefix(address, "/") +} + +func readConfig(config string) (*unboundConfig, error) { + b, err := ioutil.ReadFile(config) + if err != nil { + return nil, fmt.Errorf("error on reading config: %v", err) + } + + b = adjustUnboundConfig(b) + + var cfg unboundConfig + if err := yaml.Unmarshal(b, &cfg); err != nil { + return nil, fmt.Errorf("error on parsing config: %v", err) + } + return &cfg, nil +} + +func adjustUnboundConfig(cfg []byte) []byte { + return bytes.ReplaceAll(cfg, []byte("\t"), []byte(" ")) +} diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 485db79cf..c8b31e5d8 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -1,8 +1,6 @@ package unbound import ( - "crypto/tls" - "strings" "time" "github.com/netdata/go.d.plugin/pkg/web" @@ -65,24 +63,14 @@ type ( func (Unbound) Cleanup() {} func (u *Unbound) Init() bool { - cfg, err := readConfig(u.ConfPath) - if err != nil { - u.Warning(err) - } - if cfg != nil { - if cfg.isRemoteControlDisabled() { - u.Info("remote control is disabled in the configuration file") - return false - } - u.applyConfig(cfg) + if !u.initConfig() { + return false } - cl, err := u.createClient() - if err != nil { + if err := u.initClient(); err != nil { u.Errorf("creating client: %v", err) return false } - u.client = cl return true } @@ -106,42 +94,3 @@ func (u *Unbound) Collect() map[string]int64 { return mx } - -func (u *Unbound) applyConfig(cfg *unboundConfig) { - if cfg.hasServerSection() { - if cfg.Server.StatisticsCumulative.isSet() { - u.Cumulative = cfg.Server.StatisticsCumulative.bool() - } - } - if !cfg.hasRemoteControlSection() { - return - } - if cfg.RemoteControl.ControlUseCert.isSet() { - u.DisableTLS = cfg.RemoteControl.ControlUseCert.bool() - } - if cfg.RemoteControl.ControlKeyFile.isSet() { - u.TLSKey = string(cfg.RemoteControl.ControlKeyFile) - } - if cfg.RemoteControl.ControlCertFile.isSet() { - u.TLSCert = string(cfg.RemoteControl.ControlCertFile) - } -} - -func (u *Unbound) createClient() (uClient unboundClient, err error) { - useTLS := !isUnixSocket(u.Address) && !u.DisableTLS - var tlsCfg *tls.Config - if useTLS { - if tlsCfg, err = web.NewTLSConfig(u.ClientTLSConfig); err != nil { - return nil, err - } - } - uClient = newClient(clientConfig{ - address: u.Address, - timeout: u.Timeout.Duration, - useTLS: useTLS, - tlsConf: tlsCfg, - }) - return uClient, err -} - -func isUnixSocket(address string) bool { return strings.HasPrefix(address, "/") } From 8d743098a4068941c01b18c451983af38097cd33 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Sat, 16 Nov 2019 22:24:33 +0300 Subject: [PATCH 10/39] init minor --- modules/unbound/init.go | 60 +++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/modules/unbound/init.go b/modules/unbound/init.go index 51e5e4c3e..0d334695c 100644 --- a/modules/unbound/init.go +++ b/modules/unbound/init.go @@ -23,27 +23,27 @@ func (s strBool) isSet() bool { return s != "" } func (s strBool) bool() bool { return s == "yes" } type unboundConfig struct { - Server *struct { - StatisticsCumulative strBool `yaml:"statistics-cumulative"` + Srv *struct { + Cumulative strBool `yaml:"statistics-cumulative"` } `yaml:"server"` - RemoteControl *struct { - ControlEnable strBool `yaml:"control-enable"` - ControlInterface str `yaml:"control-interface"` - ControlPort str `yaml:"control-port"` - ControlUseCert strBool `yaml:"control-use-cert"` - ControlKeyFile str `yaml:"control-key-file"` - ControlCertFile str `yaml:"control-cert-file"` + RC *struct { + Enable strBool `yaml:"control-enable"` + Interface str `yaml:"control-interface"` + Port str `yaml:"control-port"` + UseCert strBool `yaml:"control-use-cert"` + KeyFile str `yaml:"control-key-file"` + CertFile str `yaml:"control-cert-file"` } `yaml:"remote-control"` } -func (c unboundConfig) hasServerSection() bool { - return c.Server != nil +func (c unboundConfig) hasServer() bool { + return c.Srv != nil } -func (c unboundConfig) hasRemoteControlSection() bool { - return c.RemoteControl != nil +func (c unboundConfig) hasRemoteControl() bool { + return c.RC != nil } func (c unboundConfig) isRemoteControlDisabled() bool { - return c.hasRemoteControlSection() && c.RemoteControl.ControlEnable.isSet() && !c.RemoteControl.ControlEnable.bool() + return c.hasRemoteControl() && c.RC.Enable.isSet() && !c.RC.Enable.bool() } func (u *Unbound) initConfig() bool { @@ -66,29 +66,28 @@ func (u *Unbound) initConfig() bool { } func (u *Unbound) applyConfig(cfg *unboundConfig) { - if cfg.hasServerSection() { - if cfg.Server.StatisticsCumulative.isSet() { - u.Cumulative = cfg.Server.StatisticsCumulative.bool() - } + if cfg.hasServer() && cfg.Srv.Cumulative.isSet() { + u.Cumulative = cfg.Srv.Cumulative.bool() } - if !cfg.hasRemoteControlSection() { + if !cfg.hasRemoteControl() { return } - if cfg.RemoteControl.ControlUseCert.isSet() { - u.DisableTLS = cfg.RemoteControl.ControlUseCert.bool() + if cfg.RC.UseCert.isSet() { + u.DisableTLS = cfg.RC.UseCert.bool() } - if cfg.RemoteControl.ControlKeyFile.isSet() { - u.TLSKey = string(cfg.RemoteControl.ControlKeyFile) + if cfg.RC.KeyFile.isSet() { + u.TLSKey = string(cfg.RC.KeyFile) } - if cfg.RemoteControl.ControlCertFile.isSet() { - u.TLSCert = string(cfg.RemoteControl.ControlCertFile) + if cfg.RC.CertFile.isSet() { + u.TLSCert = string(cfg.RC.CertFile) } - if cfg.RemoteControl.ControlInterface.isSet() { - u.Address = string(cfg.RemoteControl.ControlInterface) + if cfg.RC.Interface.isSet() { + u.Address = string(cfg.RC.Interface) } - if cfg.RemoteControl.ControlPort.isSet() && !isUnixSocket(u.Address) { + if cfg.RC.Port.isSet() && !isUnixSocket(u.Address) { host, _, _ := net.SplitHostPort(u.Address) - u.Address = net.JoinHostPort(host, string(cfg.RemoteControl.ControlPort)) + port := string(cfg.RC.Port) + u.Address = net.JoinHostPort(host, port) } } @@ -96,6 +95,9 @@ func (u *Unbound) initClient() (err error) { var tlsCfg *tls.Config useTLS := !isUnixSocket(u.Address) && !u.DisableTLS + //if useTLS && (u.TLSCert == "" || u.TLSKey == "") { + // return errors.New("") + //} if useTLS { if tlsCfg, err = web.NewTLSConfig(u.ClientTLSConfig); err != nil { return err From 353df42b7f6e26906dd81a20766b989703788a4f Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Sun, 17 Nov 2019 22:52:37 +0300 Subject: [PATCH 11/39] request list avg util, recur time avg/median calc fix in cumulative mode --- modules/unbound/collect.go | 53 ++++++++++++++++++++++++-------------- modules/unbound/unbound.go | 12 +++++---- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index d144d23e8..228c894d7 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -40,26 +40,16 @@ func (u *Unbound) scrapeUnboundStats() ([]entry, error) { } func (u *Unbound) collectStats(stats []entry) map[string]int64 { - var resetTimings bool - if u.Cumulative { - n := findTotalQueries(stats) - resetTimings = n > 0 && n == u.prevTotQueries - u.prevTotQueries = n - } - + reqListMul, recurTimeMul := u.findMultipliers(stats) mx := make(map[string]int64, len(stats)) for _, e := range stats { - // *.requestlist.avg, *.recursion.time.avg, *recursion.time.median - if e.hasSuffix("avg") || e.hasSuffix("median") { - if resetTimings { - e.value = 0 - } else { - e.value *= 1000 - } - } switch { - case e.hasPrefix("thread"): + case e.hasSuffix("requestlist.avg"): + e.value *= reqListMul + case e.hasSuffix("recursion.time.avg"), e.hasSuffix("recursion.time.median"): + e.value *= recurTimeMul + case e.hasPrefix("thread") && e.hasSuffix("num.queries"): v := extractThreadID(e.key) u.curCache.threads[v] = true case e.hasPrefix("num.query.type"): @@ -83,13 +73,36 @@ func (u *Unbound) collectStats(stats []entry) map[string]int64 { return mx } -func findTotalQueries(stats []entry) float64 { +func (u *Unbound) findMultipliers(stats []entry) (float64, float64) { + reqListMul, recurTimeMul := float64(1000), float64(1000) + if !u.Cumulative { + return reqListMul, recurTimeMul + } + cacheMisses, recurReplies := findCacheMissAndRecurReplies(stats) + if u.prevCacheMiss == cacheMisses { + reqListMul = 0 + } + if u.prevRecReplies == recurReplies { + reqListMul = 0 + } + u.prevCacheMiss, u.prevRecReplies = cacheMisses, recurReplies + return reqListMul, recurTimeMul +} + +func findCacheMissAndRecurReplies(stats []entry) (float64, float64) { + cacheMisses, recurReplies := float64(-1), float64(-1) for _, e := range stats { - if e.key == "total.num.queries" { - return e.value + switch e.key { + case "total.num.cachemiss": + cacheMisses = e.value + case "total.num.recursivereplies": + recurReplies = e.value + } + if cacheMisses != -1 && recurReplies != -1 { + break } } - return -1 + return cacheMisses, recurReplies } func extractThreadID(key string) string { idx := strings.IndexByte(key, '.'); return key[6:idx] } diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index c8b31e5d8..4cbc051f9 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -26,7 +26,6 @@ func New() *Unbound { return &Unbound{ Config: config, - charts: charts.Copy(), curCache: newCollectCache(), cache: newCollectCache(), } @@ -53,8 +52,11 @@ type ( cache collectCache curCache collectCache - prevTotQueries float64 - hasExtCharts bool + // used in cumulative mode + prevCacheMiss float64 + prevRecReplies float64 + + hasExtCharts bool charts *module.Charts } @@ -71,6 +73,7 @@ func (u *Unbound) Init() bool { u.Errorf("creating client: %v", err) return false } + u.charts = charts(u.Cumulative) return true } @@ -78,7 +81,7 @@ func (u *Unbound) Check() bool { return len(u.Collect()) > 0 } -func (u Unbound) Charts() *module.Charts { +func (u Unbound) Charts() *Charts { return u.charts } @@ -91,6 +94,5 @@ func (u *Unbound) Collect() map[string]int64 { if len(mx) == 0 { return nil } - return mx } From 1672a8d63ccbd6fbe673e24df3975420ba365445 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 00:26:35 +0300 Subject: [PATCH 12/39] charts wip --- modules/unbound/charts.go | 361 +++++++++++++++++++------------------ modules/unbound/collect.go | 40 ++-- modules/unbound/unbound.go | 6 +- 3 files changed, 207 insertions(+), 200 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index b465d1eb8..e9b016ba8 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -1,18 +1,55 @@ package unbound -import "github.com/netdata/go-orchestrator/module" +import ( + "github.com/netdata/go-orchestrator/module" +) type ( // Charts is an alias for module.Charts Charts = module.Charts + // Charts is an alias for module.Charts + Chart = module.Chart // Dims is an alias for module.Dims Dims = module.Dims // Dim is an alias for module.Dim Dim = module.Dim ) -var charts = Charts{ - { +func charts(cumulative bool) *Charts { + return &Charts{ + makeIncrIf(queriesChart.Copy(), cumulative), + makeIncrIf(queriesIPRLChart.Copy(), cumulative), + makeIncrIf(cacheChart.Copy(), cumulative), + makePercOfIncrIf(cachePercentageChart.Copy(), cumulative), + makeIncrIf(prefetchChart.Copy(), cumulative), + makeIncrIf(zeroTTLChart.Copy(), cumulative), + makeIncrIf(dnsCryptChart.Copy(), cumulative), + makeIncrIf(recurRepliesChart.Copy(), cumulative), + recurTimeChart.Copy(), + reqListUtilChart.Copy(), + reqListCurUtilChart.Copy(), + makeIncrIf(reqListJostleChart.Copy(), cumulative), + tcpUsageChart.Copy(), + uptimeChart.Copy(), + } +} + +func extendedCharts(cumulative bool) *Charts { + return &Charts{ + memCacheChart.Copy(), + memModChart.Copy(), + memStreamWaitChart.Copy(), + cacheCountChart.Copy(), + makeIncrIf(queryTypeChart.Copy(), cumulative), + makeIncrIf(queryClassChart.Copy(), cumulative), + makeIncrIf(queryOpCodeChart.Copy(), cumulative), + makeIncrIf(queryFlagChart.Copy(), cumulative), + makeIncrIf(answerRCodeChart.Copy(), cumulative), + } +} + +var ( + queriesChart = Chart{ ID: "queries", Title: "Total Queries", Units: "queries", @@ -21,8 +58,8 @@ var charts = Charts{ Dims: Dims{ {ID: "total.num.queries", Name: "queries"}, }, - }, - { + } + queriesIPRLChart = Chart{ ID: "queries_ip_ratelimited", Title: "Queries IP Rate Limited", Units: "queries", @@ -31,8 +68,8 @@ var charts = Charts{ Dims: Dims{ {ID: "total.num.queries_ip_ratelimited", Name: "queries"}, }, - }, - { + } + cacheChart = Chart{ ID: "cache", Title: "Cache", Units: "events", @@ -43,8 +80,8 @@ var charts = Charts{ {ID: "total.num.cachehits", Name: "hits"}, {ID: "total.num.cachemiss", Name: "miss"}, }, - }, - { + } + cachePercentageChart = Chart{ ID: "cache_percentage", Title: "Cache Percentage", Units: "percentage", @@ -55,8 +92,8 @@ var charts = Charts{ {ID: "total.num.cachehits", Name: "hits", Algo: module.PercentOfAbsolute}, {ID: "total.num.cachemiss", Name: "miss", Algo: module.PercentOfAbsolute}, }, - }, - { + } + prefetchChart = Chart{ ID: "queries_prefetch", Title: "Prefetch Queries", Units: "queries", @@ -65,8 +102,8 @@ var charts = Charts{ Dims: Dims{ {ID: "total.num.prefetch", Name: "queries"}, }, - }, - { + } + zeroTTLChart = Chart{ ID: "zero_ttl_responses", Title: "Answers Served From Expired Cache", Units: "responses", @@ -75,8 +112,8 @@ var charts = Charts{ Dims: Dims{ {ID: "total.num.zero_ttl", Name: "responses"}, }, - }, - { + } + dnsCryptChart = Chart{ ID: "dnscrypt_queries", Title: "DNSCrypt Queries", Units: "queries", @@ -88,8 +125,8 @@ var charts = Charts{ {ID: "total.num.dnscrypt.cleartext", Name: "cleartext"}, {ID: "total.num.dnscrypt.malformed", Name: "malformed"}, }, - }, - { + } + recurRepliesChart = Chart{ ID: "recursive_replies", Title: "The number of replies sent to queries that needed recursive processing", Units: "responses", @@ -98,8 +135,8 @@ var charts = Charts{ Dims: Dims{ {ID: "total.num.recursivereplies", Name: "recursive"}, }, - }, - { + } + recurTimeChart = Chart{ ID: "recursion_time", Title: "Time t took to answer queries that needed recursive processing", Units: "milliseconds", @@ -109,8 +146,8 @@ var charts = Charts{ {ID: "total.recursion.time.avg", Name: "avg"}, {ID: "total.recursion.time.median", Name: "median"}, }, - }, - { + } + reqListUtilChart = Chart{ ID: "request_list_utilization", Title: "Request List Utilization", Units: "queries", @@ -118,10 +155,10 @@ var charts = Charts{ Ctx: "unbound.request_list_utilization", Dims: Dims{ {ID: "total.requestlist.avg", Name: "avg"}, - {ID: "total.requestlist.max", Name: "max"}, + //{ID: "total.requestlist.max", Name: "max"}, // }, - }, - { + } + reqListCurUtilChart = Chart{ ID: "current_request_list_utilization", Title: "Current Request List Utilization", Units: "queries", @@ -132,8 +169,8 @@ var charts = Charts{ {ID: "total.requestlist.current.all", Name: "all"}, {ID: "total.requestlist.current.user", Name: "user"}, }, - }, - { + } + reqListJostleChart = Chart{ ID: "request_list_jostle_list", Title: "Request List Jostle List Events", Units: "events", @@ -143,8 +180,8 @@ var charts = Charts{ {ID: "total.requestlist.overwritten", Name: "overwritten"}, {ID: "total.requestlist.exceeded", Name: "dropped"}, }, - }, - { + } + tcpUsageChart = Chart{ ID: "tcpusage", Title: "TCP Accept List Usage", Units: "events", @@ -153,8 +190,8 @@ var charts = Charts{ Dims: Dims{ {ID: "total.tcpusage", Name: "tcpusage"}, }, - }, - { + } + uptimeChart = Chart{ ID: "uptime", Title: "Uptime", Units: "seconds", @@ -163,99 +200,97 @@ var charts = Charts{ Dims: Dims{ {ID: "time.up", Name: "time"}, }, - }, -} + } +) var ( - extendedCharts = Charts{ - { - ID: "cache_memory", - Title: "Cache Memory", - Units: "KB", - Fam: "mem", - Ctx: "unbound.cache_memory", - Dims: Dims{ - {ID: "mem_cache_message", Name: "message"}, - {ID: "mem_cache_rrset", Name: "rrset"}, - {ID: "mem_cache_dnscrypt_nonce", Name: "dnscrypt_nonce"}, - {ID: "mem_cache_dnscrypt_shared_secret", Name: "dnscrypt_shared_secret"}, - }, - }, - { - ID: "mod_memory", - Title: "Module Memory", - Units: "KB", - Fam: "mem", - Ctx: "unbound.mod_memory", - Dims: Dims{ - {ID: "mem_mod_ipsecmod", Name: "ipsec"}, - {ID: "mem_mod_iterator", Name: "iterator"}, - {ID: "mem_mod_respip", Name: "respip"}, - {ID: "mem_mod_subnet", Name: "subnet"}, - {ID: "mem_mod_validator", Name: "validator"}, - }, - }, - { - ID: "mem_stream_wait", - Title: "TCP and TLS Stream Waif Buffer Memory", - Units: "KB", - Fam: "mem", - Ctx: "unbound.mem_stream_wait", - Dims: Dims{ - {ID: "mem_stream_wait", Name: "stream_wait"}, - }, - }, - { - ID: "cache_count", - Title: "Cache Count", - Units: "items", - Fam: "cache count", - Ctx: "unbound.cache_count", - Dims: Dims{ - {ID: "cache_count_infra", Name: "infra"}, - {ID: "cache_count_key", Name: "key"}, - {ID: "cache_count_msg", Name: "msg"}, - {ID: "cache_count_rrset", Name: "rrset"}, - {ID: "cache_count_dnscrypt_nonce", Name: "dnscrypt_nonce"}, - {ID: "cache_count_dnscrypt_shared_secret", Name: "shared_secret"}, - }, - }, - { - ID: "query_type", - Title: "Queries By Type", - Units: "queries", - Fam: "query type", - Ctx: "unbound.type_queries", - }, - { - ID: "query_class", - Title: "Queries By Class", - Units: "queries", - Fam: "query class", - Ctx: "unbound.class_queries", + memCacheChart = Chart{ + ID: "cache_memory", + Title: "Cache Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.cache_memory", + Dims: Dims{ + {ID: "mem_cache_message", Name: "message"}, + {ID: "mem_cache_rrset", Name: "rrset"}, + {ID: "mem_cache_dnscrypt_nonce", Name: "dnscrypt_nonce"}, + {ID: "mem_cache_dnscrypt_shared_secret", Name: "dnscrypt_shared_secret"}, }, - { - ID: "query_opcode", - Title: "Queries By OpCode", - Units: "queries", - Fam: "query opcode", - Ctx: "unbound.opcode_queries", + } + memModChart = Chart{ + ID: "mod_memory", + Title: "Module Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.mod_memory", + Dims: Dims{ + {ID: "mem_mod_ipsecmod", Name: "ipsec"}, + {ID: "mem_mod_iterator", Name: "iterator"}, + {ID: "mem_mod_respip", Name: "respip"}, + {ID: "mem_mod_subnet", Name: "subnet"}, + {ID: "mem_mod_validator", Name: "validator"}, }, - { - ID: "query_flag", - Title: "Queries By Flag", - Units: "queries", - Fam: "query flag", - Ctx: "unbound.flag_queries", + } + memStreamWaitChart = Chart{ + ID: "mem_stream_wait", + Title: "TCP and TLS Stream Waif Buffer Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.mem_stream_wait", + Dims: Dims{ + {ID: "mem_stream_wait", Name: "stream_wait"}, }, - { - ID: "answer_rcode", - Title: "Answers By Rcode", - Units: "answers", - Fam: "answer rcode", - Ctx: "unbound.rcode_answers", + } + cacheCountChart = Chart{ + ID: "cache_count", + Title: "Cache Count", + Units: "items", + Fam: "cache count", + Ctx: "unbound.cache_count", + Dims: Dims{ + {ID: "cache_count_infra", Name: "infra"}, + {ID: "cache_count_key", Name: "key"}, + {ID: "cache_count_msg", Name: "msg"}, + {ID: "cache_count_rrset", Name: "rrset"}, + {ID: "cache_count_dnscrypt_nonce", Name: "dnscrypt_nonce"}, + {ID: "cache_count_dnscrypt_shared_secret", Name: "shared_secret"}, }, } + queryTypeChart = Chart{ + ID: "query_type", + Title: "Queries By Type", + Units: "queries", + Fam: "query type", + Ctx: "unbound.type_queries", + } + queryClassChart = Chart{ + ID: "query_class", + Title: "Queries By Class", + Units: "queries", + Fam: "query class", + Ctx: "unbound.class_queries", + } + queryOpCodeChart = Chart{ + ID: "query_opcode", + Title: "Queries By OpCode", + Units: "queries", + Fam: "query opcode", + Ctx: "unbound.opcode_queries", + } + queryFlagChart = Chart{ + ID: "query_flag", + Title: "Queries By Flag", + Units: "queries", + Fam: "query flag", + Ctx: "unbound.flag_queries", + } + answerRCodeChart = Chart{ + ID: "answer_rcode", + Title: "Answers By Rcode", + Units: "answers", + Fam: "answer rcode", + Ctx: "unbound.rcode_answers", + } ) func (u *Unbound) updateCharts() { @@ -267,13 +302,12 @@ func (u *Unbound) updateCharts() { } } } - // Extended stats contains query flags - if len(u.curCache.queryFlags) == 0 { + if hasExtendedData := len(u.curCache.queryFlags) > 0; hasExtendedData { return } if !u.hasExtCharts { - charts := extendedCharts.Copy() + charts := extendedCharts(u.Cumulative) if err := u.Charts().Add(*charts...); err != nil { u.Warning(err) } @@ -317,51 +351,36 @@ func newThreadCharts(id string) Charts { } func (u *Unbound) addThreadCharts(id string) { - charts := newThreadCharts(id) - return - if err := u.Charts().Add(charts...); err != nil { - return - } + //charts := newThreadCharts(id)` + //if err := u.Charts().Add(charts...); err != nil { + // return + //} } func (u *Unbound) addDimToQueryTypeChart(typ string) { - chart := u.Charts().Get("query_type") - if chart == nil { - return - } - dim := &Dim{ - ID: "query_type_" + typ, - Name: typ, - } - if err := chart.AddDim(dim); err != nil { - return - } - chart.MarkNotCreated() + u.addDimToChart(queryTypeChart.ID, "num.query.type"+typ, typ) } - func (u *Unbound) addDimToQueryClassChart(class string) { - chart := u.Charts().Get("query_class") - if chart == nil { - return - } - dim := &Dim{ - ID: "query_class_" + class, - Name: class, - } - if err := chart.AddDim(dim); err != nil { - return - } - chart.MarkNotCreated() + u.addDimToChart(queryClassChart.ID, "num.query.class."+class, class) } - func (u *Unbound) addDimToQueryOpCodeChart(opcode string) { - chart := u.Charts().Get("query_opcode") + u.addDimToChart(queryOpCodeChart.ID, "num.query.opcode."+opcode, opcode) +} +func (u *Unbound) addDimToQueryFlagsChart(flag string) { + u.addDimToChart(queryFlagChart.ID, "num.query.flag."+flag, flag) +} +func (u *Unbound) addDimToAnswerRcodeChart(rcode string) { + u.addDimToChart(answerRCodeChart.ID, "num.answer.rcode."+rcode, rcode) +} + +func (u *Unbound) addDimToChart(chartID, dimID, dimName string) { + chart := u.Charts().Get(chartID) if chart == nil { return } - dim := &Dim{ - ID: "query_opcode_" + opcode, - Name: opcode, + dim := &Dim{ID: dimID, Name: dimName} + if u.Cumulative { + dim.Algo = module.Incremental } if err := chart.AddDim(dim); err != nil { return @@ -369,32 +388,22 @@ func (u *Unbound) addDimToQueryOpCodeChart(opcode string) { chart.MarkNotCreated() } -func (u *Unbound) addDimToQueryFlagsChart(flag string) { - chart := u.Charts().Get("query_flag") - if chart == nil { - return +func makeIncrIf(chart *Chart, do bool) *Chart { + if !do { + return chart } - dim := &Dim{ - ID: "query_flag_" + flag, - Name: flag, + for _, d := range chart.Dims { + d.Algo = module.Incremental } - if err := chart.AddDim(dim); err != nil { - return - } - chart.MarkNotCreated() + return chart } -func (u *Unbound) addDimToAnswerRcodeChart(rcode string) { - chart := u.Charts().Get("answer_rcode") - if chart == nil { - return +func makePercOfIncrIf(chart *Chart, do bool) *Chart { + if !do { + return chart } - dim := &Dim{ - ID: "answer_rcode_" + rcode, - Name: rcode, + for _, d := range chart.Dims { + d.Algo = module.PercentOfIncremental } - if err := chart.AddDim(dim); err != nil { - return - } - chart.MarkNotCreated() + return chart } diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 228c894d7..4e49f386d 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -17,7 +17,6 @@ func (u *Unbound) collect() (map[string]int64, error) { return nil, err } - u.curCache.clear() mx := u.collectStats(stats) u.updateCharts() return mx, nil @@ -40,6 +39,7 @@ func (u *Unbound) scrapeUnboundStats() ([]entry, error) { } func (u *Unbound) collectStats(stats []entry) map[string]int64 { + u.curCache.clear() reqListMul, recurTimeMul := u.findMultipliers(stats) mx := make(map[string]int64, len(stats)) @@ -78,31 +78,18 @@ func (u *Unbound) findMultipliers(stats []entry) (float64, float64) { if !u.Cumulative { return reqListMul, recurTimeMul } - cacheMisses, recurReplies := findCacheMissAndRecurReplies(stats) - if u.prevCacheMiss == cacheMisses { - reqListMul = 0 - } - if u.prevRecReplies == recurReplies { + + var v float64 + if v = findEntry("total.num.cachemiss", stats); v == u.prev.cacheMiss { reqListMul = 0 } - u.prevCacheMiss, u.prevRecReplies = cacheMisses, recurReplies - return reqListMul, recurTimeMul -} + u.prev.cacheMiss = v -func findCacheMissAndRecurReplies(stats []entry) (float64, float64) { - cacheMisses, recurReplies := float64(-1), float64(-1) - for _, e := range stats { - switch e.key { - case "total.num.cachemiss": - cacheMisses = e.value - case "total.num.recursivereplies": - recurReplies = e.value - } - if cacheMisses != -1 && recurReplies != -1 { - break - } + if v = findEntry("total.num.recursivereplies", stats); v == u.prev.recurReplies { + recurTimeMul = 0 } - return cacheMisses, recurReplies + u.prev.recurReplies = v + return reqListMul, recurTimeMul } func extractThreadID(key string) string { idx := strings.IndexByte(key, '.'); return key[6:idx] } @@ -120,6 +107,15 @@ type entry struct { func (e entry) hasPrefix(prefix string) bool { return strings.HasPrefix(e.key, prefix) } func (e entry) hasSuffix(suffix string) bool { return strings.HasSuffix(e.key, suffix) } +func findEntry(key string, entries []entry) float64 { + for _, e := range entries { + if e.key == key { + return e.value + } + } + return -1 +} + func parseStatsOutput(output []string) ([]entry, error) { var es []entry for _, v := range output { diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 4cbc051f9..5fed94e14 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -53,8 +53,10 @@ type ( curCache collectCache // used in cumulative mode - prevCacheMiss float64 - prevRecReplies float64 + prev struct { + cacheMiss float64 + recurReplies float64 + } hasExtCharts bool From 990ac7e4dfa70a931a6372b7f4aeda8823afe7fb Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 11:54:43 +0300 Subject: [PATCH 13/39] use only cacheMiss for mul calc --- modules/unbound/collect.go | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 4e49f386d..6cf181f1b 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -40,17 +40,24 @@ func (u *Unbound) scrapeUnboundStats() ([]entry, error) { func (u *Unbound) collectStats(stats []entry) map[string]int64 { u.curCache.clear() - reqListMul, recurTimeMul := u.findMultipliers(stats) + mul := float64(1000000) + if u.Cumulative { + v := findEntry("total.num.cachemiss", stats) + if v == u.prevCacheMiss { + mul = 0 + } + u.prevCacheMiss = v + } mx := make(map[string]int64, len(stats)) for _, e := range stats { switch { case e.hasSuffix("requestlist.avg"): - e.value *= reqListMul + e.value *= mul case e.hasSuffix("recursion.time.avg"), e.hasSuffix("recursion.time.median"): - e.value *= recurTimeMul + e.value *= mul case e.hasPrefix("thread") && e.hasSuffix("num.queries"): - v := extractThreadID(e.key) + v := extractThread(e.key) u.curCache.threads[v] = true case e.hasPrefix("num.query.type"): v := extractQueryType(e.key) @@ -73,26 +80,7 @@ func (u *Unbound) collectStats(stats []entry) map[string]int64 { return mx } -func (u *Unbound) findMultipliers(stats []entry) (float64, float64) { - reqListMul, recurTimeMul := float64(1000), float64(1000) - if !u.Cumulative { - return reqListMul, recurTimeMul - } - - var v float64 - if v = findEntry("total.num.cachemiss", stats); v == u.prev.cacheMiss { - reqListMul = 0 - } - u.prev.cacheMiss = v - - if v = findEntry("total.num.recursivereplies", stats); v == u.prev.recurReplies { - recurTimeMul = 0 - } - u.prev.recurReplies = v - return reqListMul, recurTimeMul -} - -func extractThreadID(key string) string { idx := strings.IndexByte(key, '.'); return key[6:idx] } +func extractThread(key string) string { idx := strings.IndexByte(key, '.'); return key[:idx] } func extractQueryType(key string) string { i := len("num.query.type."); return key[i:] } func extractQueryClass(key string) string { i := len("num.query.class."); return key[i:] } func extractQueryOpCode(key string) string { i := len("num.query.opcode."); return key[i:] } From 8105c66f5df99dfdde15e7f336cc24f837b72980 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 11:54:50 +0300 Subject: [PATCH 14/39] charts wip --- modules/unbound/charts.go | 183 +++++++++++++++++++++++--------------- 1 file changed, 109 insertions(+), 74 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index e9b016ba8..7d132d72c 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -1,6 +1,9 @@ package unbound import ( + "strings" + + "github.com/netdata/go-orchestrator" "github.com/netdata/go-orchestrator/module" ) @@ -15,6 +18,8 @@ type ( Dim = module.Dim ) +var threadPriority = orchestrator.DefaultJobPriority + len(*charts(true)) + len(*extendedCharts(true)) + 10 + func charts(cumulative bool) *Charts { return &Charts{ makeIncrIf(queriesChart.Copy(), cumulative), @@ -48,33 +53,55 @@ func extendedCharts(cumulative bool) *Charts { } } +func threadCharts(thread string, cumulative bool) *Charts { + charts := charts(cumulative) + _ = charts.Remove(uptimeChart.ID) + + for i, chart := range *charts { + convertTotalChartToThread(chart, thread, threadPriority+i) + } + return charts +} + +func convertTotalChartToThread(chart *Chart, thread string, priority int) { + chart.ID = strings.Replace(chart.ID, "total", thread, 1) + chart.Title = strings.Replace(chart.Title, "Total", strings.Title(thread), 1) + chart.Fam = thread + "_stats" + chart.Ctx = strings.Replace(chart.Ctx, "total", thread, 1) + chart.Priority = priority + for _, dim := range chart.Dims { + dim.ID = strings.Replace(dim.ID, "total", thread, 1) + } +} + +// NOTE: chart id should start with 'total_', name with 'Total ', ctx should ends in `_total` (convertTotalChartToThread) var ( queriesChart = Chart{ - ID: "queries", + ID: "total_queries", Title: "Total Queries", Units: "queries", Fam: "queries", - Ctx: "unbound.queries", + Ctx: "unbound.queries_total", Dims: Dims{ {ID: "total.num.queries", Name: "queries"}, }, } queriesIPRLChart = Chart{ - ID: "queries_ip_ratelimited", - Title: "Queries IP Rate Limited", + ID: "total_queries_ip_ratelimited", + Title: "Total Queries IP Rate Limited", Units: "queries", Fam: "queries", - Ctx: "unbound.queries_ip_ratelimited", + Ctx: "unbound.queries_ip_ratelimited_total", Dims: Dims{ {ID: "total.num.queries_ip_ratelimited", Name: "queries"}, }, } cacheChart = Chart{ - ID: "cache", - Title: "Cache", + ID: "total_cache", + Title: "Total Cache", Units: "events", Fam: "cache", - Ctx: "unbound.cache", + Ctx: "unbound.cache_total", Type: module.Stacked, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits"}, @@ -82,11 +109,11 @@ var ( }, } cachePercentageChart = Chart{ - ID: "cache_percentage", - Title: "Cache Percentage", + ID: "total_cache_percentage", + Title: "Total Cache Percentage", Units: "percentage", Fam: "cache", - Ctx: "unbound.cache_percantage", + Ctx: "unbound.cache_percantage_total", Type: module.Stacked, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits", Algo: module.PercentOfAbsolute}, @@ -94,31 +121,31 @@ var ( }, } prefetchChart = Chart{ - ID: "queries_prefetch", - Title: "Prefetch Queries", + ID: "total_queries_prefetch", + Title: "Total Cache Prefetches", Units: "queries", Fam: "cache", - Ctx: "unbound.queries_prefetch", + Ctx: "unbound.queries_prefetch_total", Dims: Dims{ {ID: "total.num.prefetch", Name: "queries"}, }, } zeroTTLChart = Chart{ - ID: "zero_ttl_responses", - Title: "Answers Served From Expired Cache", + ID: "total_zero_ttl_responses", + Title: "Total Answers Served From Expired Cache", Units: "responses", Fam: "cache", - Ctx: "unbound.zero_ttl_responses", + Ctx: "unbound.zero_ttl_responses_total", Dims: Dims{ {ID: "total.num.zero_ttl", Name: "responses"}, }, } dnsCryptChart = Chart{ - ID: "dnscrypt_queries", - Title: "DNSCrypt Queries", + ID: "total_dnscrypt_queries", + Title: "Total DNSCrypt Queries", Units: "queries", Fam: "dnscrypt", - Ctx: "unbound.dnscrypt_queries", + Ctx: "unbound.dnscrypt_queries_total", Dims: Dims{ {ID: "total.num.dnscrypt.crypted", Name: "crypted"}, {ID: "total.num.dnscrypt.cert", Name: "cert"}, @@ -127,43 +154,43 @@ var ( }, } recurRepliesChart = Chart{ - ID: "recursive_replies", - Title: "The number of replies sent to queries that needed recursive processing", + ID: "total_recursive_replies", + Title: "Total number of replies sent to queries that needed recursive processing", Units: "responses", Fam: "responses", - Ctx: "unbound.recursive_replies", + Ctx: "unbound.recursive_replies_total", Dims: Dims{ {ID: "total.num.recursivereplies", Name: "recursive"}, }, } recurTimeChart = Chart{ - ID: "recursion_time", - Title: "Time t took to answer queries that needed recursive processing", + ID: "total_recursion_time", + Title: "Total Time t took to answer queries that needed recursive processing", Units: "milliseconds", Fam: "responses", - Ctx: "unbound.recursion_time", + Ctx: "unbound.recursion_time_total", Dims: Dims{ - {ID: "total.recursion.time.avg", Name: "avg"}, - {ID: "total.recursion.time.median", Name: "median"}, + {ID: "total.recursion.time.avg", Name: "avg", Div: 1000}, + {ID: "total.recursion.time.median", Name: "median", Div: 1000}, }, } reqListUtilChart = Chart{ - ID: "request_list_utilization", - Title: "Request List Utilization", + ID: "total_request_list_utilization", + Title: "Total Request List Utilization", Units: "queries", Fam: "request list", - Ctx: "unbound.request_list_utilization", + Ctx: "unbound.request_list_utilization_total", Dims: Dims{ - {ID: "total.requestlist.avg", Name: "avg"}, + {ID: "total.requestlist.avg", Name: "avg", Div: 1000000}, //{ID: "total.requestlist.max", Name: "max"}, // }, } reqListCurUtilChart = Chart{ - ID: "current_request_list_utilization", - Title: "Current Request List Utilization", + ID: "total_current_request_list_utilization", + Title: "Total Current Request List Utilization", Units: "queries", Fam: "request list", - Ctx: "unbound.current_request_list_utilization", + Ctx: "unbound.current_request_list_utilization_total", Type: module.Area, Dims: Dims{ {ID: "total.requestlist.current.all", Name: "all"}, @@ -171,22 +198,22 @@ var ( }, } reqListJostleChart = Chart{ - ID: "request_list_jostle_list", - Title: "Request List Jostle List Events", + ID: "total_request_list_jostle_list", + Title: "Total Request List Jostle List Events", Units: "events", Fam: "request list", - Ctx: "unbound.request_list_jostle_list", + Ctx: "unbound.request_list_jostle_list_total", Dims: Dims{ {ID: "total.requestlist.overwritten", Name: "overwritten"}, {ID: "total.requestlist.exceeded", Name: "dropped"}, }, } tcpUsageChart = Chart{ - ID: "tcpusage", - Title: "TCP Accept List Usage", + ID: "total_tcpusage", + Title: "Total TCP Accept List Usage", Units: "events", Fam: "tcpusage", - Ctx: "unbound.tcpusage", + Ctx: "unbound.tcpusage_total", Dims: Dims{ {ID: "total.tcpusage", Name: "tcpusage"}, }, @@ -210,11 +237,12 @@ var ( Units: "KB", Fam: "mem", Ctx: "unbound.cache_memory", + Type: module.Stacked, Dims: Dims{ - {ID: "mem_cache_message", Name: "message"}, - {ID: "mem_cache_rrset", Name: "rrset"}, - {ID: "mem_cache_dnscrypt_nonce", Name: "dnscrypt_nonce"}, - {ID: "mem_cache_dnscrypt_shared_secret", Name: "dnscrypt_shared_secret"}, + {ID: "mem.cache.message", Name: "message", Div: 1024}, + {ID: "mem.cache.rrset", Name: "rrset", Div: 1024}, + {ID: "mem.cache.dnscrypt_nonce", Name: "dnscrypt_nonce", Div: 1024}, + {ID: "mem.cache.dnscrypt_shared_secret", Name: "dnscrypt_shared_secret", Div: 1024}, }, } memModChart = Chart{ @@ -223,12 +251,13 @@ var ( Units: "KB", Fam: "mem", Ctx: "unbound.mod_memory", + Type: module.Stacked, Dims: Dims{ - {ID: "mem_mod_ipsecmod", Name: "ipsec"}, - {ID: "mem_mod_iterator", Name: "iterator"}, - {ID: "mem_mod_respip", Name: "respip"}, - {ID: "mem_mod_subnet", Name: "subnet"}, - {ID: "mem_mod_validator", Name: "validator"}, + {ID: "mem.mod.ipsecmod", Name: "ipsec", Div: 1024}, + {ID: "mem.mod.iterator", Name: "iterator", Div: 1024}, + {ID: "mem.mod.respip", Name: "respip", Div: 1024}, + {ID: "mem.mod.subnet", Name: "subnet", Div: 1024}, + {ID: "mem.mod.validator", Name: "validator", Div: 1024}, }, } memStreamWaitChart = Chart{ @@ -236,24 +265,26 @@ var ( Title: "TCP and TLS Stream Waif Buffer Memory", Units: "KB", Fam: "mem", - Ctx: "unbound.mem_stream_wait", + Ctx: "unbound.mem_streamwait", Dims: Dims{ - {ID: "mem_stream_wait", Name: "stream_wait"}, + {ID: "mem.streamwait", Name: "streamwait", Div: 1024}, }, } + // NOTE: same family as for cacheChart cacheCountChart = Chart{ ID: "cache_count", Title: "Cache Count", Units: "items", - Fam: "cache count", + Fam: "cache", Ctx: "unbound.cache_count", + Type: module.Stacked, Dims: Dims{ - {ID: "cache_count_infra", Name: "infra"}, - {ID: "cache_count_key", Name: "key"}, - {ID: "cache_count_msg", Name: "msg"}, - {ID: "cache_count_rrset", Name: "rrset"}, - {ID: "cache_count_dnscrypt_nonce", Name: "dnscrypt_nonce"}, - {ID: "cache_count_dnscrypt_shared_secret", Name: "shared_secret"}, + {ID: "infra.cache.count", Name: "infra"}, + {ID: "key.cache.count", Name: "key"}, + {ID: "msg.cache.count", Name: "msg"}, + {ID: "rrset.cache.count", Name: "rrset"}, + {ID: "dnscrypt_nonce.cache.count", Name: "dnscrypt_nonce"}, + {ID: "dnscrypt_shared_secret.cache.count", Name: "shared_secret"}, }, } queryTypeChart = Chart{ @@ -262,6 +293,7 @@ var ( Units: "queries", Fam: "query type", Ctx: "unbound.type_queries", + Type: module.Stacked, } queryClassChart = Chart{ ID: "query_class", @@ -269,6 +301,7 @@ var ( Units: "queries", Fam: "query class", Ctx: "unbound.class_queries", + Type: module.Stacked, } queryOpCodeChart = Chart{ ID: "query_opcode", @@ -276,6 +309,7 @@ var ( Units: "queries", Fam: "query opcode", Ctx: "unbound.opcode_queries", + Type: module.Stacked, } queryFlagChart = Chart{ ID: "query_flag", @@ -283,6 +317,7 @@ var ( Units: "queries", Fam: "query flag", Ctx: "unbound.flag_queries", + Type: module.Stacked, } answerRCodeChart = Chart{ ID: "answer_rcode", @@ -290,6 +325,7 @@ var ( Units: "answers", Fam: "answer rcode", Ctx: "unbound.rcode_answers", + Type: module.Stacked, } ) @@ -302,16 +338,16 @@ func (u *Unbound) updateCharts() { } } } - if hasExtendedData := len(u.curCache.queryFlags) > 0; hasExtendedData { + if hasExtendedData := len(u.curCache.queryFlags) > 0; !hasExtendedData { return } - if !u.hasExtCharts { + if !u.extChartsCreated { charts := extendedCharts(u.Cumulative) if err := u.Charts().Add(*charts...); err != nil { - u.Warning(err) + u.Warningf("add extended charts: %v", err) } - u.hasExtCharts = true + u.extChartsCreated = true } for v := range u.curCache.queryType { @@ -346,19 +382,15 @@ func (u *Unbound) updateCharts() { } } -func newThreadCharts(id string) Charts { - return nil -} - -func (u *Unbound) addThreadCharts(id string) { - //charts := newThreadCharts(id)` - //if err := u.Charts().Add(charts...); err != nil { - // return - //} +func (u *Unbound) addThreadCharts(thread string) { + charts := threadCharts(thread, u.Cumulative) + if err := u.Charts().Add(*charts...); err != nil { + u.Warningf("add '%s' charts: %v", thread, err) + } } func (u *Unbound) addDimToQueryTypeChart(typ string) { - u.addDimToChart(queryTypeChart.ID, "num.query.type"+typ, typ) + u.addDimToChart(queryTypeChart.ID, "num.query.type."+typ, typ) } func (u *Unbound) addDimToQueryClassChart(class string) { u.addDimToChart(queryClassChart.ID, "num.query.class."+class, class) @@ -367,7 +399,7 @@ func (u *Unbound) addDimToQueryOpCodeChart(opcode string) { u.addDimToChart(queryOpCodeChart.ID, "num.query.opcode."+opcode, opcode) } func (u *Unbound) addDimToQueryFlagsChart(flag string) { - u.addDimToChart(queryFlagChart.ID, "num.query.flag."+flag, flag) + u.addDimToChart(queryFlagChart.ID, "num.query.flags."+flag, flag) } func (u *Unbound) addDimToAnswerRcodeChart(rcode string) { u.addDimToChart(answerRCodeChart.ID, "num.answer.rcode."+rcode, rcode) @@ -376,6 +408,7 @@ func (u *Unbound) addDimToAnswerRcodeChart(rcode string) { func (u *Unbound) addDimToChart(chartID, dimID, dimName string) { chart := u.Charts().Get(chartID) if chart == nil { + u.Warningf("add '%s' dim: couldn't find '%s' chart", dimID, chartID) return } dim := &Dim{ID: dimID, Name: dimName} @@ -383,6 +416,7 @@ func (u *Unbound) addDimToChart(chartID, dimID, dimName string) { dim.Algo = module.Incremental } if err := chart.AddDim(dim); err != nil { + u.Warningf("add '%s' dim: %v", dimID, err) return } chart.MarkNotCreated() @@ -392,6 +426,7 @@ func makeIncrIf(chart *Chart, do bool) *Chart { if !do { return chart } + chart.Units += "/s" for _, d := range chart.Dims { d.Algo = module.Incremental } From 60d256abbbeb43760992f6a7edf754ea2cbe85cb Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 13:18:06 +0300 Subject: [PATCH 15/39] collect wip --- modules/unbound/charts.go | 22 ++++++++------ modules/unbound/collect.go | 62 ++++++++++++++++++++++++++++++-------- modules/unbound/unbound.go | 16 ++++------ 3 files changed, 67 insertions(+), 33 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index 7d132d72c..fc35a4819 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -318,6 +318,16 @@ var ( Fam: "query flag", Ctx: "unbound.flag_queries", Type: module.Stacked, + Dims: Dims{ + {ID: "num.query.flags.QR", Name: "QR"}, + {ID: "num.query.flags.AA", Name: "AA"}, + {ID: "num.query.flags.TC", Name: "TC"}, + {ID: "num.query.flags.RD", Name: "RD"}, + {ID: "num.query.flags.RA", Name: "RA"}, + {ID: "num.query.flags.Z", Name: "Z"}, + {ID: "num.query.flags.AD", Name: "AD"}, + {ID: "num.query.flags.CD", Name: "CD"}, + }, } answerRCodeChart = Chart{ ID: "answer_rcode", @@ -338,7 +348,8 @@ func (u *Unbound) updateCharts() { } } } - if hasExtendedData := len(u.curCache.queryFlags) > 0; !hasExtendedData { + // only 0-6 rcodes answers always included + if hasExtendedData := len(u.curCache.answerRCode) > 0; !hasExtendedData { return } @@ -368,12 +379,6 @@ func (u *Unbound) updateCharts() { u.addDimToQueryOpCodeChart(v) } } - for v := range u.curCache.queryFlags { - if !u.cache.queryFlags[v] { - u.cache.queryFlags[v] = true - u.addDimToQueryFlagsChart(v) - } - } for v := range u.curCache.answerRCode { if !u.cache.answerRCode[v] { u.cache.answerRCode[v] = true @@ -398,9 +403,6 @@ func (u *Unbound) addDimToQueryClassChart(class string) { func (u *Unbound) addDimToQueryOpCodeChart(opcode string) { u.addDimToChart(queryOpCodeChart.ID, "num.query.opcode."+opcode, opcode) } -func (u *Unbound) addDimToQueryFlagsChart(flag string) { - u.addDimToChart(queryFlagChart.ID, "num.query.flags."+flag, flag) -} func (u *Unbound) addDimToAnswerRcodeChart(rcode string) { u.addDimToChart(answerRCodeChart.ID, "num.answer.rcode."+rcode, rcode) } diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 6cf181f1b..8bdc4746c 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -8,6 +8,7 @@ import ( // https://github.com/NLnetLabs/unbound/blob/master/smallapp/unbound-control.c // https://github.com/NLnetLabs/unbound/blob/master/libunbound/unbound.h (ub_server_stats, ub_shm_stat_info) +// https://github.com/NLnetLabs/unbound/blob/master/daemon/remote.c // https://docs.menandmice.com/display/MM/Unbound+request-list+demystified // https://docs.datadoghq.com/integrations/unbound/#metrics @@ -39,17 +40,58 @@ func (u *Unbound) scrapeUnboundStats() ([]entry, error) { } func (u *Unbound) collectStats(stats []entry) map[string]int64 { - u.curCache.clear() - mul := float64(1000000) if u.Cumulative { - v := findEntry("total.num.cachemiss", stats) - if v == u.prevCacheMiss { - mul = 0 + return u.collectCumulativeStats(stats) + } + return u.collectNonCumulativeStats(stats) +} + +func (u *Unbound) collectCumulativeStats(stats []entry) map[string]int64 { + mul := float64(1000000) + v := findEntry("total.num.cachemiss", stats) + if v == u.prevCacheMiss { + mul = 0 + } + u.prevCacheMiss = v + return u.processStats(stats, mul) +} + +func (u *Unbound) collectNonCumulativeStats(stats []entry) map[string]int64 { + mul := float64(1000000) + mx := u.processStats(stats, mul) + + // see 'static int print_ext(RES* ssl, struct ub_stats_info* s)' in + // https://github.com/NLnetLabs/unbound/blob/master/daemon/remote.c + // - zero value queries type not included + // - zero value queries class not included + // - zero value queries opcode not included + // - only 0-6 rcodes answers always included, other zero value rcodes not included + for k := range u.cache.queryType { + if _, ok := u.curCache.queryType[k]; !ok { + mx["num.query.type."+k] = 0 } - u.prevCacheMiss = v } - mx := make(map[string]int64, len(stats)) + for k := range u.cache.queryClass { + if _, ok := u.curCache.queryClass[k]; !ok { + mx["num.query.class."+k] = 0 + } + } + for k := range u.cache.queryOpCode { + if _, ok := u.curCache.queryOpCode[k]; !ok { + mx["num.query.opcode."+k] = 0 + } + } + for k := range u.cache.answerRCode { + if _, ok := u.curCache.answerRCode[k]; !ok { + mx["num.answer.rcode."+k] = 0 + } + } + return mx +} +func (u *Unbound) processStats(stats []entry, mul float64) map[string]int64 { + u.curCache.clear() + mx := make(map[string]int64, len(stats)) for _, e := range stats { switch { case e.hasSuffix("requestlist.avg"): @@ -68,9 +110,6 @@ func (u *Unbound) collectStats(stats []entry) map[string]int64 { case e.hasPrefix("num.query.opcode"): v := extractQueryOpCode(e.key) u.curCache.queryOpCode[v] = true - case e.hasPrefix("num.query.flags"): - v := extractQueryFlag(e.key) - u.curCache.queryFlags[v] = true case e.hasPrefix("num.answer.rcode"): v := extractAnswerRCode(e.key) u.curCache.answerRCode[v] = true @@ -84,7 +123,6 @@ func extractThread(key string) string { idx := strings.IndexByte(key, '.'); func extractQueryType(key string) string { i := len("num.query.type."); return key[i:] } func extractQueryClass(key string) string { i := len("num.query.class."); return key[i:] } func extractQueryOpCode(key string) string { i := len("num.query.opcode."); return key[i:] } -func extractQueryFlag(key string) string { i := len("num.query.flags."); return key[i:] } func extractAnswerRCode(key string) string { i := len("num.answer.rcode."); return key[i:] } type entry struct { @@ -135,7 +173,6 @@ func newCollectCache() collectCache { queryType: make(map[string]bool), queryClass: make(map[string]bool), queryOpCode: make(map[string]bool), - queryFlags: make(map[string]bool), answerRCode: make(map[string]bool), } } @@ -145,7 +182,6 @@ type collectCache struct { queryType map[string]bool queryClass map[string]bool queryOpCode map[string]bool - queryFlags map[string]bool answerRCode map[string]bool } diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 5fed94e14..444001468 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -19,9 +19,10 @@ func init() { func New() *Unbound { config := Config{ // "/etc/unbound/unbound.conf" - Address: "192.168.88.223:8953", - ConfPath: "/Users/ilyam/Projects/goland/go.d.plugin/modules/unbound/testdata/unbound.conf", - Timeout: web.Duration{Duration: time.Second * 2}, + Address: "192.168.88.223:8953", + ConfPath: "/Users/ilyam/Projects/goland/go.d.plugin/modules/unbound/testdata/unbound.conf", + Timeout: web.Duration{Duration: time.Second * 2}, + DisableTLS: true, } return &Unbound{ @@ -52,13 +53,8 @@ type ( cache collectCache curCache collectCache - // used in cumulative mode - prev struct { - cacheMiss float64 - recurReplies float64 - } - - hasExtCharts bool + prevCacheMiss float64 // needed for cumulative mode + extChartsCreated bool charts *module.Charts } From 0d8a5b3704e0d93f9ea95cb4fc03022b04364437 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 14:03:38 +0300 Subject: [PATCH 16/39] add some comments to charts --- modules/unbound/charts.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index fc35a4819..da132c8a5 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -140,6 +140,7 @@ var ( {ID: "total.num.zero_ttl", Name: "responses"}, }, } + // ifdef USE_DNSCRYPT dnsCryptChart = Chart{ ID: "total_dnscrypt_queries", Title: "Total DNSCrypt Queries", @@ -182,7 +183,7 @@ var ( Ctx: "unbound.request_list_utilization_total", Dims: Dims{ {ID: "total.requestlist.avg", Name: "avg", Div: 1000000}, - //{ID: "total.requestlist.max", Name: "max"}, // + {ID: "total.requestlist.max", Name: "max"}, // all time max in cumulative mode, never resets }, } reqListCurUtilChart = Chart{ @@ -231,6 +232,7 @@ var ( ) var ( + // TODO: do not add dnscrypt stuff by default? memCacheChart = Chart{ ID: "cache_memory", Title: "Cache Memory", @@ -241,10 +243,11 @@ var ( Dims: Dims{ {ID: "mem.cache.message", Name: "message", Div: 1024}, {ID: "mem.cache.rrset", Name: "rrset", Div: 1024}, - {ID: "mem.cache.dnscrypt_nonce", Name: "dnscrypt_nonce", Div: 1024}, - {ID: "mem.cache.dnscrypt_shared_secret", Name: "dnscrypt_shared_secret", Div: 1024}, + {ID: "mem.cache.dnscrypt_nonce", Name: "dnscrypt_nonce", Div: 1024}, // ifdef USE_DNSCRYPT + {ID: "mem.cache.dnscrypt_shared_secret", Name: "dnscrypt_shared_secret", Div: 1024}, // ifdef USE_DNSCRYPT }, } + // TODO: do not add subnet and ipsecmod stuff by default? memModChart = Chart{ ID: "mod_memory", Title: "Module Memory", @@ -253,11 +256,11 @@ var ( Ctx: "unbound.mod_memory", Type: module.Stacked, Dims: Dims{ - {ID: "mem.mod.ipsecmod", Name: "ipsec", Div: 1024}, {ID: "mem.mod.iterator", Name: "iterator", Div: 1024}, {ID: "mem.mod.respip", Name: "respip", Div: 1024}, - {ID: "mem.mod.subnet", Name: "subnet", Div: 1024}, {ID: "mem.mod.validator", Name: "validator", Div: 1024}, + {ID: "mem.mod.subnet", Name: "subnet", Div: 1024}, // ifdef CLIENT_SUBNET + {ID: "mem.mod.ipsecmod", Name: "ipsec", Div: 1024}, // ifdef USE_IPSECMOD }, } memStreamWaitChart = Chart{ @@ -348,7 +351,7 @@ func (u *Unbound) updateCharts() { } } } - // only 0-6 rcodes answers always included + // 0-6 rcodes always included if hasExtendedData := len(u.curCache.answerRCode) > 0; !hasExtendedData { return } From 8263bb7ad3b949936acfef5c2019a7dda3687e87 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 16:19:26 +0300 Subject: [PATCH 17/39] some comments and config reading debug --- modules/unbound/collect.go | 5 +++++ modules/unbound/init.go | 42 +++++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 8bdc4746c..1cc528d8d 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -48,8 +48,13 @@ func (u *Unbound) collectStats(stats []entry) map[string]int64 { func (u *Unbound) collectCumulativeStats(stats []entry) map[string]int64 { mul := float64(1000000) + // following stats reset only on cachemiss event in cumulative mode + // - *.requestlist.avg, + // - *.recursion.time.avg + // - *.recursion.time.median v := findEntry("total.num.cachemiss", stats) if v == u.prevCacheMiss { + // so we need to reset them if there is no such event mul = 0 } u.prevCacheMiss = v diff --git a/modules/unbound/init.go b/modules/unbound/init.go index 0d334695c..cda6cf129 100644 --- a/modules/unbound/init.go +++ b/modules/unbound/init.go @@ -24,18 +24,20 @@ func (s strBool) bool() bool { return s == "yes" } type unboundConfig struct { Srv *struct { - Cumulative strBool `yaml:"statistics-cumulative"` + Cumulative strBool `yaml:"statistics-cumulative,omitempty"` } `yaml:"server"` RC *struct { - Enable strBool `yaml:"control-enable"` - Interface str `yaml:"control-interface"` - Port str `yaml:"control-port"` - UseCert strBool `yaml:"control-use-cert"` - KeyFile str `yaml:"control-key-file"` - CertFile str `yaml:"control-cert-file"` + Enable strBool `yaml:"control-enable,omitempty"` + Interface str `yaml:"control-interface,omitempty"` + Port str `yaml:"control-port,omitempty"` + UseCert strBool `yaml:"control-use-cert,omitempty"` + KeyFile str `yaml:"control-key-file,omitempty"` + CertFile str `yaml:"control-cert-file,omitempty"` } `yaml:"remote-control"` } +func (c unboundConfig) String() string { b, _ := yaml.Marshal(c); return string(b) } + func (c unboundConfig) hasServer() bool { return c.Srv != nil } @@ -47,14 +49,21 @@ func (c unboundConfig) isRemoteControlDisabled() bool { } func (u *Unbound) initConfig() bool { + // TODO: config parameters auto detection by reading the config file feature is questionable + // unbound config file is not in yaml format, it looks like yaml but it is not, for example it allows such config + // remote-control: + // control-interface: 0.0.0.0 + // control-interface: /var/run/unbound.sock + // Module will try to get stats from /var/run/unbound.sock and fail. Unbound doesnt allow to query stats from + // unix socket when control-interface enabled on ip interface if u.ConfPath == "" { + u.Info("'conf_path' not set, skipping parameters auto detection") return true } + u.Info("reading '%s'", u.ConfPath) cfg, err := readConfig(u.ConfPath) if err != nil { - u.Warning(err) - } - if cfg == nil { + u.Warningf("%v, skipping parameters auto detection", err) return true } if cfg.isRemoteControlDisabled() { @@ -66,28 +75,36 @@ func (u *Unbound) initConfig() bool { } func (u *Unbound) applyConfig(cfg *unboundConfig) { + u.Debugf("applying configuration:\n%s", cfg) if cfg.hasServer() && cfg.Srv.Cumulative.isSet() { + u.Debugf("found 'statistics-cumulative', applying 'cumulative_stats': %v", cfg.Srv.Cumulative.bool()) u.Cumulative = cfg.Srv.Cumulative.bool() } if !cfg.hasRemoteControl() { return } if cfg.RC.UseCert.isSet() { - u.DisableTLS = cfg.RC.UseCert.bool() + u.Debugf("found 'control-use-cert', applying 'disable_tls': %v", !cfg.RC.UseCert.bool()) + u.DisableTLS = !cfg.RC.UseCert.bool() } if cfg.RC.KeyFile.isSet() { + u.Debugf("found 'control-key-file', applying 'tls_key': %s", cfg.RC.KeyFile) u.TLSKey = string(cfg.RC.KeyFile) } if cfg.RC.CertFile.isSet() { + u.Debugf("found 'control-cert-file', applying 'tls_cert': %s", cfg.RC.CertFile) u.TLSCert = string(cfg.RC.CertFile) } if cfg.RC.Interface.isSet() { + u.Debugf("found 'control-interface', applying 'address': %s", cfg.RC.CertFile) u.Address = string(cfg.RC.Interface) } if cfg.RC.Port.isSet() && !isUnixSocket(u.Address) { host, _, _ := net.SplitHostPort(u.Address) port := string(cfg.RC.Port) - u.Address = net.JoinHostPort(host, port) + address := net.JoinHostPort(host, port) + u.Debugf("found 'control-port', applying 'address': %s", cfg.RC.CertFile) + u.Address = address } } @@ -133,5 +150,6 @@ func readConfig(config string) (*unboundConfig, error) { } func adjustUnboundConfig(cfg []byte) []byte { + // unbound config is not yaml syntax file, but the fix makes it readable at least return bytes.ReplaceAll(cfg, []byte("\t"), []byte(" ")) } From 84f6ca6664e9a862c941dc1e0e2bfb5481a04151 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 18:00:08 +0300 Subject: [PATCH 18/39] init fixes --- modules/unbound/init.go | 68 ++++++++++++++++++++++++++------------ modules/unbound/unbound.go | 11 ++++-- 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/modules/unbound/init.go b/modules/unbound/init.go index cda6cf129..97bb64b94 100644 --- a/modules/unbound/init.go +++ b/modules/unbound/init.go @@ -3,6 +3,7 @@ package unbound import ( "bytes" "crypto/tls" + "errors" "fmt" "io/ioutil" "net" @@ -60,7 +61,7 @@ func (u *Unbound) initConfig() bool { u.Info("'conf_path' not set, skipping parameters auto detection") return true } - u.Info("reading '%s'", u.ConfPath) + u.Infof("reading '%s'", u.ConfPath) cfg, err := readConfig(u.ConfPath) if err != nil { u.Warningf("%v, skipping parameters auto detection", err) @@ -77,44 +78,59 @@ func (u *Unbound) initConfig() bool { func (u *Unbound) applyConfig(cfg *unboundConfig) { u.Debugf("applying configuration:\n%s", cfg) if cfg.hasServer() && cfg.Srv.Cumulative.isSet() { - u.Debugf("found 'statistics-cumulative', applying 'cumulative_stats': %v", cfg.Srv.Cumulative.bool()) - u.Cumulative = cfg.Srv.Cumulative.bool() + if cfg.Srv.Cumulative.bool() != u.Cumulative { + u.Debugf("changing 'cumulative_stats': %v => %v", u.Cumulative, cfg.Srv.Cumulative.bool()) + u.Cumulative = cfg.Srv.Cumulative.bool() + } } + if !cfg.hasRemoteControl() { return } if cfg.RC.UseCert.isSet() { - u.Debugf("found 'control-use-cert', applying 'disable_tls': %v", !cfg.RC.UseCert.bool()) - u.DisableTLS = !cfg.RC.UseCert.bool() + if cfg.RC.UseCert.bool() != u.DisableTLS { + u.Debugf("changing 'disable_tls': %v => %v", u.DisableTLS, !cfg.RC.UseCert.bool()) + u.DisableTLS = !cfg.RC.UseCert.bool() + } } + if cfg.RC.KeyFile.isSet() { - u.Debugf("found 'control-key-file', applying 'tls_key': %s", cfg.RC.KeyFile) - u.TLSKey = string(cfg.RC.KeyFile) + if string(cfg.RC.KeyFile) != u.TLSKey { + u.Debugf("changing 'tls_key': '%s' => '%s'", u.TLSKey, cfg.RC.KeyFile) + u.TLSKey = string(cfg.RC.KeyFile) + } } + if cfg.RC.CertFile.isSet() { - u.Debugf("found 'control-cert-file', applying 'tls_cert': %s", cfg.RC.CertFile) - u.TLSCert = string(cfg.RC.CertFile) + if string(cfg.RC.CertFile) != u.TLSCert { + u.Debugf("changing 'tls_cert': '%s' => '%s'", u.TLSCert, cfg.RC.CertFile) + u.TLSCert = string(cfg.RC.CertFile) + } } + if cfg.RC.Interface.isSet() { - u.Debugf("found 'control-interface', applying 'address': %s", cfg.RC.CertFile) - u.Address = string(cfg.RC.Interface) + if v := adjustControlInterface(string(cfg.RC.Interface)); v != u.Address { + u.Debugf("changing 'address': '%s' => '%s'", u.Address, v) + u.Address = v + } } + if cfg.RC.Port.isSet() && !isUnixSocket(u.Address) { - host, _, _ := net.SplitHostPort(u.Address) - port := string(cfg.RC.Port) - address := net.JoinHostPort(host, port) - u.Debugf("found 'control-port', applying 'address': %s", cfg.RC.CertFile) - u.Address = address + if host, port, err := net.SplitHostPort(u.Address); err == nil && port != string(cfg.RC.Port) { + port := string(cfg.RC.Port) + address := net.JoinHostPort(host, port) + u.Debugf("changing 'address': '%s' => '%s'", u.Address, address) + u.Address = address + } } } func (u *Unbound) initClient() (err error) { var tlsCfg *tls.Config - useTLS := !isUnixSocket(u.Address) && !u.DisableTLS - //if useTLS && (u.TLSCert == "" || u.TLSKey == "") { - // return errors.New("") - //} + if useTLS && (u.TLSCert == "" || u.TLSKey == "") { + return errors.New("'tls_cert' or 'tls_key' is missing") + } if useTLS { if tlsCfg, err = web.NewTLSConfig(u.ClientTLSConfig); err != nil { return err @@ -130,6 +146,16 @@ func (u *Unbound) initClient() (err error) { return nil } +func adjustControlInterface(value string) string { + if isUnixSocket(value) { + return value + } + if value == "0.0.0.0" { + value = "127.0.0.1" + } + return net.JoinHostPort(value, "8953") +} + func isUnixSocket(address string) bool { return strings.HasPrefix(address, "/") } @@ -150,6 +176,6 @@ func readConfig(config string) (*unboundConfig, error) { } func adjustUnboundConfig(cfg []byte) []byte { - // unbound config is not yaml syntax file, but the fix makes it readable at least + // unbound config format is not yaml, but the fix makes it readable at least return bytes.ReplaceAll(cfg, []byte("\t"), []byte(" ")) } diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 444001468..6af969ed0 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -19,10 +19,11 @@ func init() { func New() *Unbound { config := Config{ // "/etc/unbound/unbound.conf" - Address: "192.168.88.223:8953", - ConfPath: "/Users/ilyam/Projects/goland/go.d.plugin/modules/unbound/testdata/unbound.conf", + Address: "192.168.88.223:8953", + //ConfPath: "/Users/ilyam/Projects/goland/go.d.plugin/modules/unbound/testdata/unbound.conf", Timeout: web.Duration{Duration: time.Second * 2}, DisableTLS: true, + //Cumulative:true, } return &Unbound{ @@ -71,7 +72,13 @@ func (u *Unbound) Init() bool { u.Errorf("creating client: %v", err) return false } + u.charts = charts(u.Cumulative) + + u.Debugf("using address: %s, cumulative: %v, disable_tls: %v, timeout: %s", u.Address, u.Cumulative, u.DisableTLS, u.Timeout) + if !u.DisableTLS { + u.Debugf("using tls_skip_verify: %v, tls_key: %s, tls_cert: %s", u.InsecureSkipVerify, u.TLSKey, u.TLSCert) + } return true } From ca0133b1705b5fb9160ce25df725874df3bd9eba Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 22:36:52 +0300 Subject: [PATCH 19/39] collect tests --- modules/unbound/charts.go | 6 +- modules/unbound/collect.go | 11 +- modules/unbound/init.go | 44 +- modules/unbound/testdata/stats.txt | 49 - modules/unbound/testdata/stats/common.txt | 63 + modules/unbound/testdata/stats/extended.txt | 159 +++ .../stats/lifecycle/cumulative/extended1.txt | 159 +++ .../stats/lifecycle/cumulative/extended2.txt | 159 +++ .../stats/lifecycle/cumulative/extended3.txt | 160 +++ .../stats/lifecycle/reset/extended1.txt | 160 +++ .../lifecycle/reset/extended2.txt} | 100 +- .../stats/lifecycle/reset/extended3.txt | 160 +++ modules/unbound/unbound.go | 20 +- modules/unbound/unbound_test.go | 1108 ++++++++++++++++- 14 files changed, 2210 insertions(+), 148 deletions(-) delete mode 100644 modules/unbound/testdata/stats.txt create mode 100644 modules/unbound/testdata/stats/common.txt create mode 100644 modules/unbound/testdata/stats/extended.txt create mode 100644 modules/unbound/testdata/stats/lifecycle/cumulative/extended1.txt create mode 100644 modules/unbound/testdata/stats/lifecycle/cumulative/extended2.txt create mode 100644 modules/unbound/testdata/stats/lifecycle/cumulative/extended3.txt create mode 100644 modules/unbound/testdata/stats/lifecycle/reset/extended1.txt rename modules/unbound/testdata/{extended_stats.txt => stats/lifecycle/reset/extended2.txt} (65%) create mode 100644 modules/unbound/testdata/stats/lifecycle/reset/extended3.txt diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index da132c8a5..8ece510e9 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -171,8 +171,8 @@ var ( Fam: "responses", Ctx: "unbound.recursion_time_total", Dims: Dims{ - {ID: "total.recursion.time.avg", Name: "avg", Div: 1000}, - {ID: "total.recursion.time.median", Name: "median", Div: 1000}, + {ID: "total.recursion.time.avg", Name: "avg"}, + {ID: "total.recursion.time.median", Name: "median"}, }, } reqListUtilChart = Chart{ @@ -182,7 +182,7 @@ var ( Fam: "request list", Ctx: "unbound.request_list_utilization_total", Dims: Dims{ - {ID: "total.requestlist.avg", Name: "avg", Div: 1000000}, + {ID: "total.requestlist.avg", Name: "avg", Div: 1000}, {ID: "total.requestlist.max", Name: "max"}, // all time max in cumulative mode, never resets }, } diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 1cc528d8d..95b826188 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -27,7 +27,7 @@ func (u *Unbound) scrapeUnboundStats() ([]entry, error) { const command = "UBCT1 stats" output, err := u.client.send(command + "\n") if err != nil { - return nil, err + return nil, fmt.Errorf("send command '%s': %w", command, err) } switch len(output) { case 0: @@ -47,7 +47,7 @@ func (u *Unbound) collectStats(stats []entry) map[string]int64 { } func (u *Unbound) collectCumulativeStats(stats []entry) map[string]int64 { - mul := float64(1000000) + mul := float64(1000) // following stats reset only on cachemiss event in cumulative mode // - *.requestlist.avg, // - *.recursion.time.avg @@ -62,7 +62,7 @@ func (u *Unbound) collectCumulativeStats(stats []entry) map[string]int64 { } func (u *Unbound) collectNonCumulativeStats(stats []entry) map[string]int64 { - mul := float64(1000000) + mul := float64(1000) mx := u.processStats(stats, mul) // see 'static int print_ext(RES* ssl, struct ub_stats_info* s)' in @@ -99,9 +99,8 @@ func (u *Unbound) processStats(stats []entry, mul float64) map[string]int64 { mx := make(map[string]int64, len(stats)) for _, e := range stats { switch { - case e.hasSuffix("requestlist.avg"): - e.value *= mul - case e.hasSuffix("recursion.time.avg"), e.hasSuffix("recursion.time.median"): + // *.requestlist.avg, *.recursion.time.avg, *.recursion.time.median + case e.hasSuffix(".avg"), e.hasSuffix(".median"): e.value *= mul case e.hasPrefix("thread") && e.hasSuffix("num.queries"): v := extractThread(e.key) diff --git a/modules/unbound/init.go b/modules/unbound/init.go index 97bb64b94..f51f97e30 100644 --- a/modules/unbound/init.go +++ b/modules/unbound/init.go @@ -20,8 +20,9 @@ type ( ) func (s str) isSet() bool { return s != "" } +func (s str) value() string { return string(s) } func (s strBool) isSet() bool { return s != "" } -func (s strBool) bool() bool { return s == "yes" } +func (s strBool) value() bool { return s == "yes" } type unboundConfig struct { Srv *struct { @@ -37,19 +38,15 @@ type unboundConfig struct { } `yaml:"remote-control"` } -func (c unboundConfig) String() string { b, _ := yaml.Marshal(c); return string(b) } +func (c unboundConfig) String() string { b, _ := yaml.Marshal(c); return string(b) } +func (c unboundConfig) hasServer() bool { return c.Srv != nil } +func (c unboundConfig) hasRemoteControl() bool { return c.RC != nil } -func (c unboundConfig) hasServer() bool { - return c.Srv != nil -} -func (c unboundConfig) hasRemoteControl() bool { - return c.RC != nil -} func (c unboundConfig) isRemoteControlDisabled() bool { - return c.hasRemoteControl() && c.RC.Enable.isSet() && !c.RC.Enable.bool() + return c.hasRemoteControl() && c.RC.Enable.isSet() && !c.RC.Enable.value() } -func (u *Unbound) initConfig() bool { +func (u *Unbound) initConfig() (enabled bool) { // TODO: config parameters auto detection by reading the config file feature is questionable // unbound config file is not in yaml format, it looks like yaml but it is not, for example it allows such config // remote-control: @@ -78,9 +75,9 @@ func (u *Unbound) initConfig() bool { func (u *Unbound) applyConfig(cfg *unboundConfig) { u.Debugf("applying configuration:\n%s", cfg) if cfg.hasServer() && cfg.Srv.Cumulative.isSet() { - if cfg.Srv.Cumulative.bool() != u.Cumulative { - u.Debugf("changing 'cumulative_stats': %v => %v", u.Cumulative, cfg.Srv.Cumulative.bool()) - u.Cumulative = cfg.Srv.Cumulative.bool() + if cfg.Srv.Cumulative.value() != u.Cumulative { + u.Debugf("changing 'cumulative_stats': %v => %v", u.Cumulative, cfg.Srv.Cumulative.value()) + u.Cumulative = cfg.Srv.Cumulative.value() } } @@ -88,37 +85,36 @@ func (u *Unbound) applyConfig(cfg *unboundConfig) { return } if cfg.RC.UseCert.isSet() { - if cfg.RC.UseCert.bool() != u.DisableTLS { - u.Debugf("changing 'disable_tls': %v => %v", u.DisableTLS, !cfg.RC.UseCert.bool()) - u.DisableTLS = !cfg.RC.UseCert.bool() + if cfg.RC.UseCert.value() != u.DisableTLS { + u.Debugf("changing 'disable_tls': %v => %v", u.DisableTLS, !cfg.RC.UseCert.value()) + u.DisableTLS = !cfg.RC.UseCert.value() } } if cfg.RC.KeyFile.isSet() { - if string(cfg.RC.KeyFile) != u.TLSKey { + if cfg.RC.KeyFile.value() != u.TLSKey { u.Debugf("changing 'tls_key': '%s' => '%s'", u.TLSKey, cfg.RC.KeyFile) - u.TLSKey = string(cfg.RC.KeyFile) + u.TLSKey = cfg.RC.KeyFile.value() } } if cfg.RC.CertFile.isSet() { - if string(cfg.RC.CertFile) != u.TLSCert { + if cfg.RC.CertFile.value() != u.TLSCert { u.Debugf("changing 'tls_cert': '%s' => '%s'", u.TLSCert, cfg.RC.CertFile) - u.TLSCert = string(cfg.RC.CertFile) + u.TLSCert = cfg.RC.CertFile.value() } } if cfg.RC.Interface.isSet() { - if v := adjustControlInterface(string(cfg.RC.Interface)); v != u.Address { + if v := adjustControlInterface(cfg.RC.Interface.value()); v != u.Address { u.Debugf("changing 'address': '%s' => '%s'", u.Address, v) u.Address = v } } if cfg.RC.Port.isSet() && !isUnixSocket(u.Address) { - if host, port, err := net.SplitHostPort(u.Address); err == nil && port != string(cfg.RC.Port) { - port := string(cfg.RC.Port) - address := net.JoinHostPort(host, port) + if host, port, err := net.SplitHostPort(u.Address); err == nil && port != cfg.RC.Port.value() { + address := net.JoinHostPort(host, cfg.RC.Port.value()) u.Debugf("changing 'address': '%s' => '%s'", u.Address, address) u.Address = address } diff --git a/modules/unbound/testdata/stats.txt b/modules/unbound/testdata/stats.txt deleted file mode 100644 index 51cdef163..000000000 --- a/modules/unbound/testdata/stats.txt +++ /dev/null @@ -1,49 +0,0 @@ -thread0.num.queries=23 -thread0.num.queries_ip_ratelimited=0 -thread0.num.cachehits=10 -thread0.num.cachemiss=13 -thread0.num.prefetch=0 -thread0.num.zero_ttl=0 -thread0.num.recursivereplies=13 -thread0.num.dnscrypt.crypted=0 -thread0.num.dnscrypt.cert=0 -thread0.num.dnscrypt.cleartext=0 -thread0.num.dnscrypt.malformed=0 -thread0.requestlist.avg=1 -thread0.requestlist.max=5 -thread0.requestlist.overwritten=0 -thread0.requestlist.exceeded=0 -thread0.requestlist.current.all=0 -thread0.requestlist.current.user=0 -thread0.recursion.time.avg=0.078938 -thread0.recursion.time.median=0.049152 -thread0.tcpusage=0 - -total.num.queries=23 -total.num.queries_ip_ratelimited=0 - -total.num.cachehits=10 -total.num.cachemiss=13 -total.num.prefetch=0 -total.num.zero_ttl=0 - -total.num.recursivereplies=13 - -total.num.dnscrypt.crypted=0 -total.num.dnscrypt.cert=0 -total.num.dnscrypt.cleartext=0 -total.num.dnscrypt.malformed=0 - -total.requestlist.avg=1 -total.requestlist.max=5 -total.requestlist.overwritten=0 -total.requestlist.exceeded=0 -total.requestlist.current.all=0 -total.requestlist.current.user=0 - -total.recursion.time.avg=0.078938 -total.recursion.time.median=0.049152 -total.tcpusage=0 -time.now=1573817686.220375 -time.up=1206.824144 -time.elapsed=1128.946812 \ No newline at end of file diff --git a/modules/unbound/testdata/stats/common.txt b/modules/unbound/testdata/stats/common.txt new file mode 100644 index 000000000..a62657cd6 --- /dev/null +++ b/modules/unbound/testdata/stats/common.txt @@ -0,0 +1,63 @@ +thread0.num.queries=28 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=21 +thread0.num.cachemiss=7 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=7 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=0.857143 +thread0.requestlist.max=6 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=1.255822 +thread0.recursion.time.median=0.480597 +thread0.tcpusage=0 +thread1.num.queries=16 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=13 +thread1.num.cachemiss=3 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=3 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=0 +thread1.requestlist.max=0 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.093941 +thread1.recursion.time.median=0 +thread1.tcpusage=0 +total.num.queries=44 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=34 +total.num.cachemiss=10 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=10 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0.6 +total.requestlist.max=6 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.907258 +total.recursion.time.median=0.240299 +total.tcpusage=0 +time.now=1574094836.941149 +time.up=88.434983 +time.elapsed=88.4349831 \ No newline at end of file diff --git a/modules/unbound/testdata/stats/extended.txt b/modules/unbound/testdata/stats/extended.txt new file mode 100644 index 000000000..222b70831 --- /dev/null +++ b/modules/unbound/testdata/stats/extended.txt @@ -0,0 +1,159 @@ +thread0.num.queries=28 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=21 +thread0.num.cachemiss=7 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=7 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=0.857143 +thread0.requestlist.max=6 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=1.255822 +thread0.recursion.time.median=0.480597 +thread0.tcpusage=0 +thread1.num.queries=16 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=13 +thread1.num.cachemiss=3 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=3 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=0 +thread1.requestlist.max=0 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.093941 +thread1.recursion.time.median=0 +thread1.tcpusage=0 +total.num.queries=44 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=34 +total.num.cachemiss=10 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=10 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0.6 +total.requestlist.max=6 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.907258 +total.recursion.time.median=0.240299 +total.tcpusage=0 +time.now=1574094836.941149 +time.up=88.434983 +time.elapsed=88.434983 +mem.cache.rrset=178642 +mem.cache.message=90357 +mem.mod.iterator=16588 +mem.mod.validator=81059 +mem.mod.respip=0 +mem.mod.subnet=74504 +mem.cache.dnscrypt_shared_secret=0 +mem.cache.dnscrypt_nonce=0 +mem.streamwait=0 +histogram.000000.000000.to.000000.000001=0 +histogram.000000.000001.to.000000.000002=0 +histogram.000000.000002.to.000000.000004=0 +histogram.000000.000004.to.000000.000008=0 +histogram.000000.000008.to.000000.000016=0 +histogram.000000.000016.to.000000.000032=0 +histogram.000000.000032.to.000000.000064=0 +histogram.000000.000064.to.000000.000128=0 +histogram.000000.000128.to.000000.000256=0 +histogram.000000.000256.to.000000.000512=0 +histogram.000000.000512.to.000000.001024=0 +histogram.000000.001024.to.000000.002048=0 +histogram.000000.002048.to.000000.004096=0 +histogram.000000.004096.to.000000.008192=0 +histogram.000000.008192.to.000000.016384=0 +histogram.000000.016384.to.000000.032768=0 +histogram.000000.032768.to.000000.065536=2 +histogram.000000.065536.to.000000.131072=0 +histogram.000000.131072.to.000000.262144=2 +histogram.000000.262144.to.000000.524288=3 +histogram.000000.524288.to.000001.000000=2 +histogram.000001.000000.to.000002.000000=0 +histogram.000002.000000.to.000004.000000=0 +histogram.000004.000000.to.000008.000000=1 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=0 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=13 +num.query.type.PTR=5 +num.query.type.MX=13 +num.query.type.AAAA=13 +num.query.class.IN=44 +num.query.opcode.QUERY=44 +num.query.tcp=0 +num.query.tcpout=1 +num.query.tls=0 +num.query.tls.resume=0 +num.query.ipv6=39 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=44 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=0 +num.query.flags.CD=0 +num.query.edns.present=0 +num.query.edns.DO=0 +num.answer.rcode.NOERROR=40 +num.answer.rcode.FORMERR=0 +num.answer.rcode.SERVFAIL=0 +num.answer.rcode.NXDOMAIN=4 +num.answer.rcode.NOTIMPL=0 +num.answer.rcode.REFUSED=0 +num.query.ratelimited=0 +num.answer.secure=0 +num.answer.bogus=0 +num.rrset.bogus=0 +num.query.aggressive.NOERROR=2 +num.query.aggressive.NXDOMAIN=0 +unwanted.queries=0 +unwanted.replies=0 +msg.cache.count=81 +rrset.cache.count=314 +infra.cache.count=205 +key.cache.count=9 +dnscrypt_shared_secret.cache.count=0 +dnscrypt_nonce.cache.count=0 +num.query.dnscrypt.shared_secret.cachemiss=0 +num.query.dnscrypt.replay=0 +num.query.authzone.up=0 +num.query.authzone.down=0 +num.query.subnet=0 +num.query.subnet_cache=0 \ No newline at end of file diff --git a/modules/unbound/testdata/stats/lifecycle/cumulative/extended1.txt b/modules/unbound/testdata/stats/lifecycle/cumulative/extended1.txt new file mode 100644 index 000000000..ac2b6c958 --- /dev/null +++ b/modules/unbound/testdata/stats/lifecycle/cumulative/extended1.txt @@ -0,0 +1,159 @@ +thread0.num.queries=90 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=80 +thread0.num.cachemiss=10 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=10 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=0.1 +thread0.requestlist.max=1 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.222018 +thread0.recursion.time.median=0.337042 +thread0.tcpusage=0 +thread1.num.queries=110 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=101 +thread1.num.cachemiss=9 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=9 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=0.222222 +thread1.requestlist.max=1 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.844506 +thread1.recursion.time.median=0.360448 +thread1.tcpusage=0 +total.num.queries=200 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=181 +total.num.cachemiss=19 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=19 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0.157895 +total.requestlist.max=1 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.516881 +total.recursion.time.median=0.348745 +total.tcpusage=0 +time.now=1574103378.552596 +time.up=122.956436 +time.elapsed=122.956436 +mem.cache.rrset=175745 +mem.cache.message=93392 +mem.mod.iterator=16588 +mem.mod.validator=81479 +mem.mod.respip=0 +mem.mod.subnet=74504 +mem.cache.dnscrypt_shared_secret=0 +mem.cache.dnscrypt_nonce=0 +mem.streamwait=0 +histogram.000000.000000.to.000000.000001=0 +histogram.000000.000001.to.000000.000002=0 +histogram.000000.000002.to.000000.000004=0 +histogram.000000.000004.to.000000.000008=0 +histogram.000000.000008.to.000000.000016=0 +histogram.000000.000016.to.000000.000032=0 +histogram.000000.000032.to.000000.000064=0 +histogram.000000.000064.to.000000.000128=0 +histogram.000000.000128.to.000000.000256=0 +histogram.000000.000256.to.000000.000512=0 +histogram.000000.000512.to.000000.001024=0 +histogram.000000.001024.to.000000.002048=0 +histogram.000000.002048.to.000000.004096=0 +histogram.000000.004096.to.000000.008192=0 +histogram.000000.008192.to.000000.016384=2 +histogram.000000.016384.to.000000.032768=1 +histogram.000000.032768.to.000000.065536=3 +histogram.000000.065536.to.000000.131072=0 +histogram.000000.131072.to.000000.262144=0 +histogram.000000.262144.to.000000.524288=11 +histogram.000000.524288.to.000001.000000=0 +histogram.000001.000000.to.000002.000000=1 +histogram.000002.000000.to.000004.000000=0 +histogram.000004.000000.to.000008.000000=1 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=0 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=60 +num.query.type.PTR=20 +num.query.type.MX=60 +num.query.type.AAAA=60 +num.query.class.IN=200 +num.query.opcode.QUERY=200 +num.query.tcp=0 +num.query.tcpout=0 +num.query.tls=0 +num.query.tls.resume=0 +num.query.ipv6=0 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=200 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=0 +num.query.flags.CD=0 +num.query.edns.present=0 +num.query.edns.DO=0 +num.answer.rcode.NOERROR=184 +num.answer.rcode.FORMERR=0 +num.answer.rcode.SERVFAIL=0 +num.answer.rcode.NXDOMAIN=16 +num.answer.rcode.NOTIMPL=0 +num.answer.rcode.REFUSED=0 +num.query.ratelimited=0 +num.answer.secure=0 +num.answer.bogus=0 +num.rrset.bogus=0 +num.query.aggressive.NOERROR=1 +num.query.aggressive.NXDOMAIN=0 +unwanted.queries=0 +unwanted.replies=0 +msg.cache.count=94 +rrset.cache.count=304 +infra.cache.count=192 +key.cache.count=11 +dnscrypt_shared_secret.cache.count=0 +dnscrypt_nonce.cache.count=0 +num.query.dnscrypt.shared_secret.cachemiss=0 +num.query.dnscrypt.replay=0 +num.query.authzone.up=0 +num.query.authzone.down=0 +num.query.subnet=0 +num.query.subnet_cache=0 \ No newline at end of file diff --git a/modules/unbound/testdata/stats/lifecycle/cumulative/extended2.txt b/modules/unbound/testdata/stats/lifecycle/cumulative/extended2.txt new file mode 100644 index 000000000..75f027360 --- /dev/null +++ b/modules/unbound/testdata/stats/lifecycle/cumulative/extended2.txt @@ -0,0 +1,159 @@ +thread0.num.queries=133 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=123 +thread0.num.cachemiss=10 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=10 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=0.1 +thread0.requestlist.max=1 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.222018 +thread0.recursion.time.median=0.337042 +thread0.tcpusage=0 +thread1.num.queries=157 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=148 +thread1.num.cachemiss=9 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=9 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=0.222222 +thread1.requestlist.max=1 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.844506 +thread1.recursion.time.median=0.360448 +thread1.tcpusage=0 +total.num.queries=290 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=271 +total.num.cachemiss=19 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=19 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0.157895 +total.requestlist.max=1 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.516881 +total.recursion.time.median=0.348745 +total.tcpusage=0 +time.now=1574103461.161540 +time.up=205.565380 +time.elapsed=82.608944 +mem.cache.rrset=175745 +mem.cache.message=93392 +mem.mod.iterator=16588 +mem.mod.validator=81479 +mem.mod.respip=0 +mem.mod.subnet=74504 +mem.cache.dnscrypt_shared_secret=0 +mem.cache.dnscrypt_nonce=0 +mem.streamwait=0 +histogram.000000.000000.to.000000.000001=0 +histogram.000000.000001.to.000000.000002=0 +histogram.000000.000002.to.000000.000004=0 +histogram.000000.000004.to.000000.000008=0 +histogram.000000.000008.to.000000.000016=0 +histogram.000000.000016.to.000000.000032=0 +histogram.000000.000032.to.000000.000064=0 +histogram.000000.000064.to.000000.000128=0 +histogram.000000.000128.to.000000.000256=0 +histogram.000000.000256.to.000000.000512=0 +histogram.000000.000512.to.000000.001024=0 +histogram.000000.001024.to.000000.002048=0 +histogram.000000.002048.to.000000.004096=0 +histogram.000000.004096.to.000000.008192=0 +histogram.000000.008192.to.000000.016384=2 +histogram.000000.016384.to.000000.032768=1 +histogram.000000.032768.to.000000.065536=3 +histogram.000000.065536.to.000000.131072=0 +histogram.000000.131072.to.000000.262144=0 +histogram.000000.262144.to.000000.524288=11 +histogram.000000.524288.to.000001.000000=0 +histogram.000001.000000.to.000002.000000=1 +histogram.000002.000000.to.000004.000000=0 +histogram.000004.000000.to.000008.000000=1 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=0 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=90 +num.query.type.PTR=20 +num.query.type.MX=90 +num.query.type.AAAA=90 +num.query.class.IN=290 +num.query.opcode.QUERY=290 +num.query.tcp=0 +num.query.tcpout=0 +num.query.tls=0 +num.query.tls.resume=0 +num.query.ipv6=0 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=290 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=0 +num.query.flags.CD=0 +num.query.edns.present=0 +num.query.edns.DO=0 +num.answer.rcode.NOERROR=274 +num.answer.rcode.FORMERR=0 +num.answer.rcode.SERVFAIL=0 +num.answer.rcode.NXDOMAIN=16 +num.answer.rcode.NOTIMPL=0 +num.answer.rcode.REFUSED=0 +num.query.ratelimited=0 +num.answer.secure=0 +num.answer.bogus=0 +num.rrset.bogus=0 +num.query.aggressive.NOERROR=1 +num.query.aggressive.NXDOMAIN=0 +unwanted.queries=0 +unwanted.replies=0 +msg.cache.count=94 +rrset.cache.count=304 +infra.cache.count=192 +key.cache.count=11 +dnscrypt_shared_secret.cache.count=0 +dnscrypt_nonce.cache.count=0 +num.query.dnscrypt.shared_secret.cachemiss=0 +num.query.dnscrypt.replay=0 +num.query.authzone.up=0 +num.query.authzone.down=0 +num.query.subnet=0 +num.query.subnet_cache=0 \ No newline at end of file diff --git a/modules/unbound/testdata/stats/lifecycle/cumulative/extended3.txt b/modules/unbound/testdata/stats/lifecycle/cumulative/extended3.txt new file mode 100644 index 000000000..f56482624 --- /dev/null +++ b/modules/unbound/testdata/stats/lifecycle/cumulative/extended3.txt @@ -0,0 +1,160 @@ +thread0.num.queries=165 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=150 +thread0.num.cachemiss=15 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=15 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=0.0666667 +thread0.requestlist.max=1 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.261497 +thread0.recursion.time.median=0.318318 +thread0.tcpusage=0 +thread1.num.queries=195 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=184 +thread1.num.cachemiss=11 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=11 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=0.363636 +thread1.requestlist.max=2 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.709047 +thread1.recursion.time.median=0.294912 +thread1.tcpusage=0 +total.num.queries=360 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=334 +total.num.cachemiss=26 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=26 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0.192308 +total.requestlist.max=2 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.450844 +total.recursion.time.median=0.306615 +total.tcpusage=0 +time.now=1574103543.692653 +time.up=288.096493 +time.elapsed=82.531113 +mem.cache.rrset=208839 +mem.cache.message=101198 +mem.mod.iterator=16588 +mem.mod.validator=85725 +mem.mod.respip=0 +mem.mod.subnet=74504 +mem.cache.dnscrypt_shared_secret=0 +mem.cache.dnscrypt_nonce=0 +mem.streamwait=0 +histogram.000000.000000.to.000000.000001=0 +histogram.000000.000001.to.000000.000002=0 +histogram.000000.000002.to.000000.000004=0 +histogram.000000.000004.to.000000.000008=0 +histogram.000000.000008.to.000000.000016=0 +histogram.000000.000016.to.000000.000032=0 +histogram.000000.000032.to.000000.000064=0 +histogram.000000.000064.to.000000.000128=0 +histogram.000000.000128.to.000000.000256=0 +histogram.000000.000256.to.000000.000512=0 +histogram.000000.000512.to.000000.001024=0 +histogram.000000.001024.to.000000.002048=0 +histogram.000000.002048.to.000000.004096=0 +histogram.000000.004096.to.000000.008192=0 +histogram.000000.008192.to.000000.016384=2 +histogram.000000.016384.to.000000.032768=1 +histogram.000000.032768.to.000000.065536=5 +histogram.000000.065536.to.000000.131072=3 +histogram.000000.131072.to.000000.262144=0 +histogram.000000.262144.to.000000.524288=11 +histogram.000000.524288.to.000001.000000=2 +histogram.000001.000000.to.000002.000000=1 +histogram.000002.000000.to.000004.000000=0 +histogram.000004.000000.to.000008.000000=1 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=0 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=120 +num.query.type.PTR=20 +num.query.type.MX=110 +num.query.type.AAAA=110 +num.query.class.IN=360 +num.query.opcode.QUERY=360 +num.query.tcp=0 +num.query.tcpout=0 +num.query.tls=0 +num.query.tls.resume=0 +num.query.ipv6=0 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=360 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=0 +num.query.flags.CD=0 +num.query.edns.present=0 +num.query.edns.DO=0 +num.answer.rcode.NOERROR=334 +num.answer.rcode.FORMERR=0 +num.answer.rcode.SERVFAIL=10 +num.answer.rcode.NXDOMAIN=16 +num.answer.rcode.NOTIMPL=0 +num.answer.rcode.REFUSED=0 +num.answer.rcode.nodata=20 +num.query.ratelimited=0 +num.answer.secure=0 +num.answer.bogus=0 +num.rrset.bogus=0 +num.query.aggressive.NOERROR=1 +num.query.aggressive.NXDOMAIN=0 +unwanted.queries=0 +unwanted.replies=0 +msg.cache.count=119 +rrset.cache.count=401 +infra.cache.count=232 +key.cache.count=14 +dnscrypt_shared_secret.cache.count=0 +dnscrypt_nonce.cache.count=0 +num.query.dnscrypt.shared_secret.cachemiss=0 +num.query.dnscrypt.replay=0 +num.query.authzone.up=0 +num.query.authzone.down=0 +num.query.subnet=0 +num.query.subnet_cache=0 \ No newline at end of file diff --git a/modules/unbound/testdata/stats/lifecycle/reset/extended1.txt b/modules/unbound/testdata/stats/lifecycle/reset/extended1.txt new file mode 100644 index 000000000..0d64201b8 --- /dev/null +++ b/modules/unbound/testdata/stats/lifecycle/reset/extended1.txt @@ -0,0 +1,160 @@ +thread0.num.queries=51 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=44 +thread0.num.cachemiss=7 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=7 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=0 +thread0.requestlist.max=0 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.365956 +thread0.recursion.time.median=0.057344 +thread0.tcpusage=0 +thread1.num.queries=49 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=46 +thread1.num.cachemiss=3 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=3 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=0 +thread1.requestlist.max=0 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=1.582766 +thread1.recursion.time.median=0 +thread1.tcpusage=0 +total.num.queries=100 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=90 +total.num.cachemiss=10 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=10 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0 +total.requestlist.max=0 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.730999 +total.recursion.time.median=0.028672 +total.tcpusage=0 +time.now=1574103644.993894 +time.up=45.285130 +time.elapsed=45.285130 +mem.cache.rrset=172757 +mem.cache.message=86064 +mem.mod.iterator=16588 +mem.mod.validator=79979 +mem.mod.respip=0 +mem.mod.subnet=74504 +mem.cache.dnscrypt_shared_secret=0 +mem.cache.dnscrypt_nonce=0 +mem.streamwait=0 +histogram.000000.000000.to.000000.000001=0 +histogram.000000.000001.to.000000.000002=0 +histogram.000000.000002.to.000000.000004=0 +histogram.000000.000004.to.000000.000008=0 +histogram.000000.000008.to.000000.000016=0 +histogram.000000.000016.to.000000.000032=0 +histogram.000000.000032.to.000000.000064=0 +histogram.000000.000064.to.000000.000128=0 +histogram.000000.000128.to.000000.000256=0 +histogram.000000.000256.to.000000.000512=0 +histogram.000000.000512.to.000000.001024=0 +histogram.000000.001024.to.000000.002048=0 +histogram.000000.002048.to.000000.004096=0 +histogram.000000.004096.to.000000.008192=0 +histogram.000000.008192.to.000000.016384=0 +histogram.000000.016384.to.000000.032768=2 +histogram.000000.032768.to.000000.065536=3 +histogram.000000.065536.to.000000.131072=1 +histogram.000000.131072.to.000000.262144=1 +histogram.000000.262144.to.000000.524288=1 +histogram.000000.524288.to.000001.000000=0 +histogram.000001.000000.to.000002.000000=1 +histogram.000002.000000.to.000004.000000=0 +histogram.000004.000000.to.000008.000000=1 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=0 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=30 +num.query.type.PTR=10 +num.query.type.MX=30 +num.query.type.AAAA=30 +num.query.class.IN=100 +num.query.opcode.QUERY=100 +num.query.tcp=0 +num.query.tcpout=1 +num.query.tls=0 +num.query.tls.resume=0 +num.query.ipv6=0 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=100 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=0 +num.query.flags.CD=0 +num.query.edns.present=0 +num.query.edns.DO=0 +num.answer.rcode.NOERROR=90 +num.answer.rcode.FORMERR=0 +num.answer.rcode.SERVFAIL=0 +num.answer.rcode.NXDOMAIN=10 +num.answer.rcode.NOTIMPL=0 +num.answer.rcode.REFUSED=0 +num.answer.rcode.nodata=10 +num.query.ratelimited=0 +num.answer.secure=0 +num.answer.bogus=0 +num.rrset.bogus=0 +num.query.aggressive.NOERROR=2 +num.query.aggressive.NXDOMAIN=0 +unwanted.queries=0 +unwanted.replies=0 +msg.cache.count=67 +rrset.cache.count=303 +infra.cache.count=181 +key.cache.count=10 +dnscrypt_shared_secret.cache.count=0 +dnscrypt_nonce.cache.count=0 +num.query.dnscrypt.shared_secret.cachemiss=0 +num.query.dnscrypt.replay=0 +num.query.authzone.up=0 +num.query.authzone.down=0 +num.query.subnet=0 +num.query.subnet_cache=0 \ No newline at end of file diff --git a/modules/unbound/testdata/extended_stats.txt b/modules/unbound/testdata/stats/lifecycle/reset/extended2.txt similarity index 65% rename from modules/unbound/testdata/extended_stats.txt rename to modules/unbound/testdata/stats/lifecycle/reset/extended2.txt index 4431315b7..1cab37313 100644 --- a/modules/unbound/testdata/extended_stats.txt +++ b/modules/unbound/testdata/stats/lifecycle/reset/extended2.txt @@ -1,56 +1,76 @@ -thread0.num.queries=23 +thread0.num.queries=0 thread0.num.queries_ip_ratelimited=0 -thread0.num.cachehits=10 -thread0.num.cachemiss=13 +thread0.num.cachehits=0 +thread0.num.cachemiss=0 thread0.num.prefetch=0 thread0.num.zero_ttl=0 -thread0.num.recursivereplies=13 +thread0.num.recursivereplies=0 thread0.num.dnscrypt.crypted=0 thread0.num.dnscrypt.cert=0 thread0.num.dnscrypt.cleartext=0 thread0.num.dnscrypt.malformed=0 -thread0.requestlist.avg=1 -thread0.requestlist.max=5 +thread0.requestlist.avg=0 +thread0.requestlist.max=0 thread0.requestlist.overwritten=0 thread0.requestlist.exceeded=0 thread0.requestlist.current.all=0 thread0.requestlist.current.user=0 -thread0.recursion.time.avg=0.078938 -thread0.recursion.time.median=0.049152 +thread0.recursion.time.avg=0.000000 +thread0.recursion.time.median=0 thread0.tcpusage=0 -total.num.queries=23 +thread1.num.queries=0 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=0 +thread1.num.cachemiss=0 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=0 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=0 +thread1.requestlist.max=0 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.000000 +thread1.recursion.time.median=0 +thread1.tcpusage=0 +total.num.queries=0 total.num.queries_ip_ratelimited=0 -total.num.cachehits=10 -total.num.cachemiss=13 +total.num.cachehits=0 +total.num.cachemiss=0 total.num.prefetch=0 total.num.zero_ttl=0 -total.num.recursivereplies=13 +total.num.recursivereplies=0 total.num.dnscrypt.crypted=0 total.num.dnscrypt.cert=0 total.num.dnscrypt.cleartext=0 total.num.dnscrypt.malformed=0 -total.requestlist.avg=1 -total.requestlist.max=5 +total.requestlist.avg=0 +total.requestlist.max=0 total.requestlist.overwritten=0 total.requestlist.exceeded=0 total.requestlist.current.all=0 total.requestlist.current.user=0 -total.recursion.time.avg=0.078938 -total.recursion.time.median=0.049152 +total.recursion.time.avg=0.000000 +total.recursion.time.median=0 total.tcpusage=0 -time.now=1573817686.220375 -time.up=1206.824144 -time.elapsed=1128.946812 -mem.cache.rrset=104566 -mem.cache.message=76765 +time.now=1574103671.543847 +time.up=71.835083 +time.elapsed=26.549953 +mem.cache.rrset=172757 +mem.cache.message=86064 mem.mod.iterator=16588 -mem.mod.validator=72332 +mem.mod.validator=79979 mem.mod.respip=0 mem.mod.subnet=74504 mem.cache.dnscrypt_shared_secret=0 mem.cache.dnscrypt_nonce=0 mem.streamwait=0 -histogram.000000.000000.to.000000.000001=1 +histogram.000000.000000.to.000000.000001=0 histogram.000000.000001.to.000000.000002=0 histogram.000000.000002.to.000000.000004=0 histogram.000000.000004.to.000000.000008=0 @@ -64,12 +84,12 @@ histogram.000000.000512.to.000000.001024=0 histogram.000000.001024.to.000000.002048=0 histogram.000000.002048.to.000000.004096=0 histogram.000000.004096.to.000000.008192=0 -histogram.000000.008192.to.000000.016384=3 -histogram.000000.016384.to.000000.032768=1 -histogram.000000.032768.to.000000.065536=3 -histogram.000000.065536.to.000000.131072=2 -histogram.000000.131072.to.000000.262144=2 -histogram.000000.262144.to.000000.524288=1 +histogram.000000.008192.to.000000.016384=0 +histogram.000000.016384.to.000000.032768=0 +histogram.000000.032768.to.000000.065536=0 +histogram.000000.065536.to.000000.131072=0 +histogram.000000.131072.to.000000.262144=0 +histogram.000000.262144.to.000000.524288=0 histogram.000000.524288.to.000001.000000=0 histogram.000001.000000.to.000002.000000=0 histogram.000002.000000.to.000004.000000=0 @@ -90,12 +110,6 @@ histogram.032768.000000.to.065536.000000=0 histogram.065536.000000.to.131072.000000=0 histogram.131072.000000.to.262144.000000=0 histogram.262144.000000.to.524288.000000=0 -num.query.type.A=12 -num.query.type.PTR=1 -num.query.type.MX=5 -num.query.type.AAAA=5 -num.query.class.IN=23 -num.query.opcode.QUERY=23 num.query.tcp=0 num.query.tcpout=0 num.query.tls=0 @@ -104,31 +118,31 @@ num.query.ipv6=0 num.query.flags.QR=0 num.query.flags.AA=0 num.query.flags.TC=0 -num.query.flags.RD=23 +num.query.flags.RD=0 num.query.flags.RA=0 num.query.flags.Z=0 num.query.flags.AD=0 num.query.flags.CD=0 num.query.edns.present=0 num.query.edns.DO=0 -num.answer.rcode.NOERROR=15 +num.answer.rcode.NOERROR=0 num.answer.rcode.FORMERR=0 num.answer.rcode.SERVFAIL=0 -num.answer.rcode.NXDOMAIN=8 +num.answer.rcode.NXDOMAIN=0 num.answer.rcode.NOTIMPL=0 num.answer.rcode.REFUSED=0 num.query.ratelimited=0 -num.answer.secure=7 +num.answer.secure=0 num.answer.bogus=0 num.rrset.bogus=0 num.query.aggressive.NOERROR=0 num.query.aggressive.NXDOMAIN=0 unwanted.queries=0 unwanted.replies=0 -msg.cache.count=33 -rrset.cache.count=124 -infra.cache.count=58 -key.cache.count=7 +msg.cache.count=67 +rrset.cache.count=303 +infra.cache.count=181 +key.cache.count=10 dnscrypt_shared_secret.cache.count=0 dnscrypt_nonce.cache.count=0 num.query.dnscrypt.shared_secret.cachemiss=0 diff --git a/modules/unbound/testdata/stats/lifecycle/reset/extended3.txt b/modules/unbound/testdata/stats/lifecycle/reset/extended3.txt new file mode 100644 index 000000000..136e6b8b1 --- /dev/null +++ b/modules/unbound/testdata/stats/lifecycle/reset/extended3.txt @@ -0,0 +1,160 @@ +thread0.num.queries=34 +thread0.num.queries_ip_ratelimited=0 +thread0.num.cachehits=30 +thread0.num.cachemiss=4 +thread0.num.prefetch=0 +thread0.num.zero_ttl=0 +thread0.num.recursivereplies=4 +thread0.num.dnscrypt.crypted=0 +thread0.num.dnscrypt.cert=0 +thread0.num.dnscrypt.cleartext=0 +thread0.num.dnscrypt.malformed=0 +thread0.requestlist.avg=0 +thread0.requestlist.max=0 +thread0.requestlist.overwritten=0 +thread0.requestlist.exceeded=0 +thread0.requestlist.current.all=0 +thread0.requestlist.current.user=0 +thread0.recursion.time.avg=0.541654 +thread0.recursion.time.median=0.098304 +thread0.tcpusage=0 +thread1.num.queries=36 +thread1.num.queries_ip_ratelimited=0 +thread1.num.cachehits=33 +thread1.num.cachemiss=3 +thread1.num.prefetch=0 +thread1.num.zero_ttl=0 +thread1.num.recursivereplies=3 +thread1.num.dnscrypt.crypted=0 +thread1.num.dnscrypt.cert=0 +thread1.num.dnscrypt.cleartext=0 +thread1.num.dnscrypt.malformed=0 +thread1.requestlist.avg=1.66667 +thread1.requestlist.max=5 +thread1.requestlist.overwritten=0 +thread1.requestlist.exceeded=0 +thread1.requestlist.current.all=0 +thread1.requestlist.current.user=0 +thread1.recursion.time.avg=0.062328 +thread1.recursion.time.median=0 +thread1.tcpusage=0 +total.num.queries=70 +total.num.queries_ip_ratelimited=0 +total.num.cachehits=63 +total.num.cachemiss=7 +total.num.prefetch=0 +total.num.zero_ttl=0 +total.num.recursivereplies=7 +total.num.dnscrypt.crypted=0 +total.num.dnscrypt.cert=0 +total.num.dnscrypt.cleartext=0 +total.num.dnscrypt.malformed=0 +total.requestlist.avg=0.714286 +total.requestlist.max=5 +total.requestlist.overwritten=0 +total.requestlist.exceeded=0 +total.requestlist.current.all=0 +total.requestlist.current.user=0 +total.recursion.time.avg=0.336228 +total.recursion.time.median=0.049152 +total.tcpusage=0 +time.now=1574103731.371896 +time.up=131.663132 +time.elapsed=59.828049 +mem.cache.rrset=235917 +mem.cache.message=105471 +mem.mod.iterator=16588 +mem.mod.validator=87270 +mem.mod.respip=0 +mem.mod.subnet=74504 +mem.cache.dnscrypt_shared_secret=0 +mem.cache.dnscrypt_nonce=0 +mem.streamwait=0 +histogram.000000.000000.to.000000.000001=0 +histogram.000000.000001.to.000000.000002=0 +histogram.000000.000002.to.000000.000004=0 +histogram.000000.000004.to.000000.000008=0 +histogram.000000.000008.to.000000.000016=0 +histogram.000000.000016.to.000000.000032=0 +histogram.000000.000032.to.000000.000064=0 +histogram.000000.000064.to.000000.000128=0 +histogram.000000.000128.to.000000.000256=0 +histogram.000000.000256.to.000000.000512=0 +histogram.000000.000512.to.000000.001024=0 +histogram.000000.001024.to.000000.002048=0 +histogram.000000.002048.to.000000.004096=0 +histogram.000000.004096.to.000000.008192=0 +histogram.000000.008192.to.000000.016384=0 +histogram.000000.016384.to.000000.032768=2 +histogram.000000.032768.to.000000.065536=1 +histogram.000000.065536.to.000000.131072=3 +histogram.000000.131072.to.000000.262144=0 +histogram.000000.262144.to.000000.524288=0 +histogram.000000.524288.to.000001.000000=0 +histogram.000001.000000.to.000002.000000=1 +histogram.000002.000000.to.000004.000000=0 +histogram.000004.000000.to.000008.000000=0 +histogram.000008.000000.to.000016.000000=0 +histogram.000016.000000.to.000032.000000=0 +histogram.000032.000000.to.000064.000000=0 +histogram.000064.000000.to.000128.000000=0 +histogram.000128.000000.to.000256.000000=0 +histogram.000256.000000.to.000512.000000=0 +histogram.000512.000000.to.001024.000000=0 +histogram.001024.000000.to.002048.000000=0 +histogram.002048.000000.to.004096.000000=0 +histogram.004096.000000.to.008192.000000=0 +histogram.008192.000000.to.016384.000000=0 +histogram.016384.000000.to.032768.000000=0 +histogram.032768.000000.to.065536.000000=0 +histogram.065536.000000.to.131072.000000=0 +histogram.131072.000000.to.262144.000000=0 +histogram.262144.000000.to.524288.000000=0 +num.query.type.A=20 +num.query.type.PTR=10 +num.query.type.MX=20 +num.query.type.AAAA=20 +num.query.class.IN=70 +num.query.opcode.QUERY=70 +num.query.tcp=0 +num.query.tcpout=0 +num.query.tls=0 +num.query.tls.resume=0 +num.query.ipv6=0 +num.query.flags.QR=0 +num.query.flags.AA=0 +num.query.flags.TC=0 +num.query.flags.RD=70 +num.query.flags.RA=0 +num.query.flags.Z=0 +num.query.flags.AD=0 +num.query.flags.CD=0 +num.query.edns.present=0 +num.query.edns.DO=0 +num.answer.rcode.NOERROR=60 +num.answer.rcode.FORMERR=0 +num.answer.rcode.SERVFAIL=0 +num.answer.rcode.NXDOMAIN=10 +num.answer.rcode.NOTIMPL=0 +num.answer.rcode.REFUSED=0 +num.answer.rcode.nodata=10 +num.query.ratelimited=0 +num.answer.secure=0 +num.answer.bogus=0 +num.rrset.bogus=0 +num.query.aggressive.NOERROR=2 +num.query.aggressive.NXDOMAIN=0 +unwanted.queries=0 +unwanted.replies=0 +msg.cache.count=127 +rrset.cache.count=501 +infra.cache.count=303 +key.cache.count=15 +dnscrypt_shared_secret.cache.count=0 +dnscrypt_nonce.cache.count=0 +num.query.dnscrypt.shared_secret.cachemiss=0 +num.query.dnscrypt.replay=0 +num.query.authzone.up=0 +num.query.authzone.down=0 +num.query.subnet=0 +num.query.subnet_cache=0 \ No newline at end of file diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 6af969ed0..653ec8b6a 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -18,12 +18,16 @@ func init() { func New() *Unbound { config := Config{ - // "/etc/unbound/unbound.conf" - Address: "192.168.88.223:8953", - //ConfPath: "/Users/ilyam/Projects/goland/go.d.plugin/modules/unbound/testdata/unbound.conf", - Timeout: web.Duration{Duration: time.Second * 2}, - DisableTLS: true, - //Cumulative:true, + Address: "127.0.0.1:8953", + ConfPath: "/etc/unbound/unbound.conf", + Timeout: web.Duration{Duration: time.Second}, + Cumulative: false, + DisableTLS: false, + ClientTLSConfig: web.ClientTLSConfig{ + TLSCert: "/etc/unbound/unbound_control.pem", + TLSKey: "/etc/unbound/unbound_control.key", + InsecureSkipVerify: true, + }, } return &Unbound{ @@ -64,7 +68,7 @@ type ( func (Unbound) Cleanup() {} func (u *Unbound) Init() bool { - if !u.initConfig() { + if enabled := u.initConfig(); !enabled { return false } @@ -86,7 +90,7 @@ func (u *Unbound) Check() bool { return len(u.Collect()) > 0 } -func (u Unbound) Charts() *Charts { +func (u Unbound) Charts() *module.Charts { return u.charts } diff --git a/modules/unbound/unbound_test.go b/modules/unbound/unbound_test.go index ee774a3da..79625cdf4 100644 --- a/modules/unbound/unbound_test.go +++ b/modules/unbound/unbound_test.go @@ -1,23 +1,39 @@ package unbound import ( + "bufio" + "bytes" + "errors" "fmt" "io/ioutil" + "strings" "testing" - "time" "github.com/netdata/go-orchestrator/module" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var ( - statsData, _ = ioutil.ReadFile("testdata/stats.txt") - extStatsData, _ = ioutil.ReadFile("testdata/extended_stats.txt") + commonStatsData, _ = ioutil.ReadFile("testdata/stats/common.txt") + extStatsData, _ = ioutil.ReadFile("testdata/stats/extended.txt") + lifeCycleCumulativeData1, _ = ioutil.ReadFile("testdata/stats/lifecycle/cumulative/extended1.txt") + lifeCycleCumulativeData2, _ = ioutil.ReadFile("testdata/stats/lifecycle/cumulative/extended2.txt") + lifeCycleCumulativeData3, _ = ioutil.ReadFile("testdata/stats/lifecycle/cumulative/extended3.txt") + lifeCycleResetData1, _ = ioutil.ReadFile("testdata/stats/lifecycle/reset/extended1.txt") + lifeCycleResetData2, _ = ioutil.ReadFile("testdata/stats/lifecycle/reset/extended2.txt") + lifeCycleResetData3, _ = ioutil.ReadFile("testdata/stats/lifecycle/reset/extended3.txt") ) func Test_readTestData(t *testing.T) { - assert.NotNil(t, statsData) + assert.NotNil(t, commonStatsData) assert.NotNil(t, extStatsData) + assert.NotNil(t, lifeCycleCumulativeData1) + assert.NotNil(t, lifeCycleCumulativeData2) + assert.NotNil(t, lifeCycleCumulativeData3) + assert.NotNil(t, lifeCycleResetData1) + assert.NotNil(t, lifeCycleResetData2) + assert.NotNil(t, lifeCycleResetData3) } func TestNew(t *testing.T) { @@ -41,17 +57,1079 @@ func TestUnbound_Charts(t *testing.T) { } func TestUnbound_Collect(t *testing.T) { - v := New() - v.Init() - v.client = newClient(clientConfig{ - address: "192.168.88.223:8953", - timeout: time.Second * 2, - useTLS: false, - tlsConf: nil, - }) - m := v.Collect() + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: commonStatsData, err: false} - for k, v := range m { - fmt.Println(k, v) + //m := unbound.Collect() + //_ = m + //l := make([]string, 0) + //for k := range m { + // l = append(l, k) + //} + //sort.Strings(l) + //for _, v := range l { + // fmt.Println(fmt.Sprintf("\"%s\": %d,", v, m[v])) + //} + expected := map[string]int64{ + "thread0.num.cachehits": 21, + "thread0.num.cachemiss": 7, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 28, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 7, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 1255, + "thread0.recursion.time.median": 480, + "thread0.requestlist.avg": 857, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 6, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 13, + "thread1.num.cachemiss": 3, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 16, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 3, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 93, + "thread1.recursion.time.median": 0, + "thread1.requestlist.avg": 0, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 0, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 88, + "time.now": 1574094836, + "time.up": 88, + "total.num.cachehits": 34, + "total.num.cachemiss": 10, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 44, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 10, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 907, + "total.recursion.time.median": 240, + "total.requestlist.avg": 600, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 6, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + } + + assert.Equal(t, expected, unbound.Collect()) +} + +func TestUnbound_Collect_ExtendedStats(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: extStatsData, err: false} + + expected := map[string]int64{ + "dnscrypt_nonce.cache.count": 0, + "dnscrypt_shared_secret.cache.count": 0, + "infra.cache.count": 205, + "key.cache.count": 9, + "mem.cache.dnscrypt_nonce": 0, + "mem.cache.dnscrypt_shared_secret": 0, + "mem.cache.message": 90357, + "mem.cache.rrset": 178642, + "mem.mod.iterator": 16588, + "mem.mod.respip": 0, + "mem.mod.subnet": 74504, + "mem.mod.validator": 81059, + "mem.streamwait": 0, + "msg.cache.count": 81, + "num.answer.bogus": 0, + "num.answer.rcode.FORMERR": 0, + "num.answer.rcode.NOERROR": 40, + "num.answer.rcode.NOTIMPL": 0, + "num.answer.rcode.NXDOMAIN": 4, + "num.answer.rcode.REFUSED": 0, + "num.answer.rcode.SERVFAIL": 0, + "num.answer.secure": 0, + "num.query.aggressive.NOERROR": 2, + "num.query.aggressive.NXDOMAIN": 0, + "num.query.authzone.down": 0, + "num.query.authzone.up": 0, + "num.query.class.IN": 44, + "num.query.dnscrypt.replay": 0, + "num.query.dnscrypt.shared_secret.cachemiss": 0, + "num.query.edns.DO": 0, + "num.query.edns.present": 0, + "num.query.flags.AA": 0, + "num.query.flags.AD": 0, + "num.query.flags.CD": 0, + "num.query.flags.QR": 0, + "num.query.flags.RA": 0, + "num.query.flags.RD": 44, + "num.query.flags.TC": 0, + "num.query.flags.Z": 0, + "num.query.ipv6": 39, + "num.query.opcode.QUERY": 44, + "num.query.ratelimited": 0, + "num.query.subnet": 0, + "num.query.subnet_cache": 0, + "num.query.tcp": 0, + "num.query.tcpout": 1, + "num.query.tls": 0, + "num.query.tls.resume": 0, + "num.query.type.A": 13, + "num.query.type.AAAA": 13, + "num.query.type.MX": 13, + "num.query.type.PTR": 5, + "num.rrset.bogus": 0, + "rrset.cache.count": 314, + "thread0.num.cachehits": 21, + "thread0.num.cachemiss": 7, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 28, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 7, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 1255, + "thread0.recursion.time.median": 480, + "thread0.requestlist.avg": 857, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 6, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 13, + "thread1.num.cachemiss": 3, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 16, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 3, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 93, + "thread1.recursion.time.median": 0, + "thread1.requestlist.avg": 0, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 0, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 88, + "time.now": 1574094836, + "time.up": 88, + "total.num.cachehits": 34, + "total.num.cachemiss": 10, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 44, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 10, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 907, + "total.recursion.time.median": 240, + "total.requestlist.avg": 600, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 6, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + "unwanted.queries": 0, + "unwanted.replies": 0, + } + + assert.Equal(t, expected, unbound.Collect()) +} + +func TestUnbound_Collect_LifeCycleCumulativeExtendedStats(t *testing.T) { + tests := []struct { + input []byte + expected map[string]int64 + }{ + { + input: lifeCycleCumulativeData1, + expected: expectedCumulative1, + }, + { + input: lifeCycleCumulativeData2, + expected: expectedCumulative2, + }, + { + input: lifeCycleCumulativeData3, + expected: expectedCumulative3, + }, + } + + unbound := New() + unbound.DisableTLS = true + unbound.Cumulative = true + require.True(t, unbound.Init()) + ubClient := &mockUnboundClient{err: false} + unbound.client = ubClient + + for i, test := range tests { + t.Run(fmt.Sprintf("run %d", i+1), func(t *testing.T) { + ubClient.data = test.input + assert.Equal(t, test.expected, unbound.Collect()) + }) + } +} + +func TestUnbound_Collect_LifeCycleResetExtendedStats(t *testing.T) { + tests := []struct { + input []byte + expected map[string]int64 + }{ + { + input: lifeCycleResetData1, + expected: expectedReset1, + }, + { + input: lifeCycleResetData2, + expected: expectedReset2, + }, + { + input: lifeCycleResetData3, + expected: expectedReset3, + }, + } + _ = tests + + unbound := New() + unbound.DisableTLS = true + unbound.Cumulative = false + require.True(t, unbound.Init()) + ubClient := &mockUnboundClient{err: false} + unbound.client = ubClient + + for i, test := range tests { + t.Run(fmt.Sprintf("run %d", i+1), func(t *testing.T) { + ubClient.data = test.input + assert.Equal(t, test.expected, unbound.Collect()) + }) } } + +func TestUnbound_Collect_EmptyResponse(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: []byte{}, err: false} + + assert.Nil(t, unbound.Collect()) +} + +func TestUnbound_Collect_ErrorResponse(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: []byte("error unknown command 'unknown'"), err: false} + + assert.Nil(t, unbound.Collect()) +} + +func TestUnbound_Collect_ErrorOnSend(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{err: true} + + assert.Nil(t, unbound.Collect()) +} + +func TestUnbound_Collect_BadSyntaxResponse(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + data := strings.Repeat("zk_avg_latency 0\nzk_min_latency 0\nzk_mix_latency 0\n", 10) + unbound.client = mockUnboundClient{data: []byte(data), err: false} + + assert.Nil(t, unbound.Collect()) +} + +type mockUnboundClient struct { + data []byte + err bool +} + +func (m mockUnboundClient) send(command string) ([]string, error) { + if m.err { + return nil, errors.New("mock send error") + } + var rv []string + s := bufio.NewScanner(bytes.NewReader(m.data)) + for s.Scan() { + rv = append(rv, s.Text()) + } + return rv, nil +} + +var ( + expectedCumulative1 = map[string]int64{ + "dnscrypt_nonce.cache.count": 0, + "dnscrypt_shared_secret.cache.count": 0, + "infra.cache.count": 192, + "key.cache.count": 11, + "mem.cache.dnscrypt_nonce": 0, + "mem.cache.dnscrypt_shared_secret": 0, + "mem.cache.message": 93392, + "mem.cache.rrset": 175745, + "mem.mod.iterator": 16588, + "mem.mod.respip": 0, + "mem.mod.subnet": 74504, + "mem.mod.validator": 81479, + "mem.streamwait": 0, + "msg.cache.count": 94, + "num.answer.bogus": 0, + "num.answer.rcode.FORMERR": 0, + "num.answer.rcode.NOERROR": 184, + "num.answer.rcode.NOTIMPL": 0, + "num.answer.rcode.NXDOMAIN": 16, + "num.answer.rcode.REFUSED": 0, + "num.answer.rcode.SERVFAIL": 0, + "num.answer.secure": 0, + "num.query.aggressive.NOERROR": 1, + "num.query.aggressive.NXDOMAIN": 0, + "num.query.authzone.down": 0, + "num.query.authzone.up": 0, + "num.query.class.IN": 200, + "num.query.dnscrypt.replay": 0, + "num.query.dnscrypt.shared_secret.cachemiss": 0, + "num.query.edns.DO": 0, + "num.query.edns.present": 0, + "num.query.flags.AA": 0, + "num.query.flags.AD": 0, + "num.query.flags.CD": 0, + "num.query.flags.QR": 0, + "num.query.flags.RA": 0, + "num.query.flags.RD": 200, + "num.query.flags.TC": 0, + "num.query.flags.Z": 0, + "num.query.ipv6": 0, + "num.query.opcode.QUERY": 200, + "num.query.ratelimited": 0, + "num.query.subnet": 0, + "num.query.subnet_cache": 0, + "num.query.tcp": 0, + "num.query.tcpout": 0, + "num.query.tls": 0, + "num.query.tls.resume": 0, + "num.query.type.A": 60, + "num.query.type.AAAA": 60, + "num.query.type.MX": 60, + "num.query.type.PTR": 20, + "num.rrset.bogus": 0, + "rrset.cache.count": 304, + "thread0.num.cachehits": 80, + "thread0.num.cachemiss": 10, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 90, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 10, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 222, + "thread0.recursion.time.median": 337, + "thread0.requestlist.avg": 100, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 1, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 101, + "thread1.num.cachemiss": 9, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 110, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 9, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 844, + "thread1.recursion.time.median": 360, + "thread1.requestlist.avg": 222, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 1, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 122, + "time.now": 1574103378, + "time.up": 122, + "total.num.cachehits": 181, + "total.num.cachemiss": 19, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 200, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 19, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 516, + "total.recursion.time.median": 348, + "total.requestlist.avg": 157, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 1, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + "unwanted.queries": 0, + "unwanted.replies": 0, + } + + expectedCumulative2 = map[string]int64{ + "dnscrypt_nonce.cache.count": 0, + "dnscrypt_shared_secret.cache.count": 0, + "infra.cache.count": 192, + "key.cache.count": 11, + "mem.cache.dnscrypt_nonce": 0, + "mem.cache.dnscrypt_shared_secret": 0, + "mem.cache.message": 93392, + "mem.cache.rrset": 175745, + "mem.mod.iterator": 16588, + "mem.mod.respip": 0, + "mem.mod.subnet": 74504, + "mem.mod.validator": 81479, + "mem.streamwait": 0, + "msg.cache.count": 94, + "num.answer.bogus": 0, + "num.answer.rcode.FORMERR": 0, + "num.answer.rcode.NOERROR": 274, + "num.answer.rcode.NOTIMPL": 0, + "num.answer.rcode.NXDOMAIN": 16, + "num.answer.rcode.REFUSED": 0, + "num.answer.rcode.SERVFAIL": 0, + "num.answer.secure": 0, + "num.query.aggressive.NOERROR": 1, + "num.query.aggressive.NXDOMAIN": 0, + "num.query.authzone.down": 0, + "num.query.authzone.up": 0, + "num.query.class.IN": 290, + "num.query.dnscrypt.replay": 0, + "num.query.dnscrypt.shared_secret.cachemiss": 0, + "num.query.edns.DO": 0, + "num.query.edns.present": 0, + "num.query.flags.AA": 0, + "num.query.flags.AD": 0, + "num.query.flags.CD": 0, + "num.query.flags.QR": 0, + "num.query.flags.RA": 0, + "num.query.flags.RD": 290, + "num.query.flags.TC": 0, + "num.query.flags.Z": 0, + "num.query.ipv6": 0, + "num.query.opcode.QUERY": 290, + "num.query.ratelimited": 0, + "num.query.subnet": 0, + "num.query.subnet_cache": 0, + "num.query.tcp": 0, + "num.query.tcpout": 0, + "num.query.tls": 0, + "num.query.tls.resume": 0, + "num.query.type.A": 90, + "num.query.type.AAAA": 90, + "num.query.type.MX": 90, + "num.query.type.PTR": 20, + "num.rrset.bogus": 0, + "rrset.cache.count": 304, + "thread0.num.cachehits": 123, + "thread0.num.cachemiss": 10, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 133, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 10, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 0, + "thread0.recursion.time.median": 0, + "thread0.requestlist.avg": 0, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 1, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 148, + "thread1.num.cachemiss": 9, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 157, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 9, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 0, + "thread1.recursion.time.median": 0, + "thread1.requestlist.avg": 0, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 1, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 82, + "time.now": 1574103461, + "time.up": 205, + "total.num.cachehits": 271, + "total.num.cachemiss": 19, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 290, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 19, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 0, + "total.recursion.time.median": 0, + "total.requestlist.avg": 0, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 1, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + "unwanted.queries": 0, + "unwanted.replies": 0, + } + + expectedCumulative3 = map[string]int64{ + "dnscrypt_nonce.cache.count": 0, + "dnscrypt_shared_secret.cache.count": 0, + "infra.cache.count": 232, + "key.cache.count": 14, + "mem.cache.dnscrypt_nonce": 0, + "mem.cache.dnscrypt_shared_secret": 0, + "mem.cache.message": 101198, + "mem.cache.rrset": 208839, + "mem.mod.iterator": 16588, + "mem.mod.respip": 0, + "mem.mod.subnet": 74504, + "mem.mod.validator": 85725, + "mem.streamwait": 0, + "msg.cache.count": 119, + "num.answer.bogus": 0, + "num.answer.rcode.FORMERR": 0, + "num.answer.rcode.NOERROR": 334, + "num.answer.rcode.NOTIMPL": 0, + "num.answer.rcode.NXDOMAIN": 16, + "num.answer.rcode.REFUSED": 0, + "num.answer.rcode.SERVFAIL": 10, + "num.answer.rcode.nodata": 20, + "num.answer.secure": 0, + "num.query.aggressive.NOERROR": 1, + "num.query.aggressive.NXDOMAIN": 0, + "num.query.authzone.down": 0, + "num.query.authzone.up": 0, + "num.query.class.IN": 360, + "num.query.dnscrypt.replay": 0, + "num.query.dnscrypt.shared_secret.cachemiss": 0, + "num.query.edns.DO": 0, + "num.query.edns.present": 0, + "num.query.flags.AA": 0, + "num.query.flags.AD": 0, + "num.query.flags.CD": 0, + "num.query.flags.QR": 0, + "num.query.flags.RA": 0, + "num.query.flags.RD": 360, + "num.query.flags.TC": 0, + "num.query.flags.Z": 0, + "num.query.ipv6": 0, + "num.query.opcode.QUERY": 360, + "num.query.ratelimited": 0, + "num.query.subnet": 0, + "num.query.subnet_cache": 0, + "num.query.tcp": 0, + "num.query.tcpout": 0, + "num.query.tls": 0, + "num.query.tls.resume": 0, + "num.query.type.A": 120, + "num.query.type.AAAA": 110, + "num.query.type.MX": 110, + "num.query.type.PTR": 20, + "num.rrset.bogus": 0, + "rrset.cache.count": 401, + "thread0.num.cachehits": 150, + "thread0.num.cachemiss": 15, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 165, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 15, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 261, + "thread0.recursion.time.median": 318, + "thread0.requestlist.avg": 66, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 1, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 184, + "thread1.num.cachemiss": 11, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 195, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 11, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 709, + "thread1.recursion.time.median": 294, + "thread1.requestlist.avg": 363, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 2, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 82, + "time.now": 1574103543, + "time.up": 288, + "total.num.cachehits": 334, + "total.num.cachemiss": 26, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 360, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 26, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 450, + "total.recursion.time.median": 306, + "total.requestlist.avg": 192, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 2, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + "unwanted.queries": 0, + "unwanted.replies": 0, + } +) + +var ( + expectedReset1 = map[string]int64{ + "dnscrypt_nonce.cache.count": 0, + "dnscrypt_shared_secret.cache.count": 0, + "infra.cache.count": 181, + "key.cache.count": 10, + "mem.cache.dnscrypt_nonce": 0, + "mem.cache.dnscrypt_shared_secret": 0, + "mem.cache.message": 86064, + "mem.cache.rrset": 172757, + "mem.mod.iterator": 16588, + "mem.mod.respip": 0, + "mem.mod.subnet": 74504, + "mem.mod.validator": 79979, + "mem.streamwait": 0, + "msg.cache.count": 67, + "num.answer.bogus": 0, + "num.answer.rcode.FORMERR": 0, + "num.answer.rcode.NOERROR": 90, + "num.answer.rcode.NOTIMPL": 0, + "num.answer.rcode.NXDOMAIN": 10, + "num.answer.rcode.REFUSED": 0, + "num.answer.rcode.SERVFAIL": 0, + "num.answer.rcode.nodata": 10, + "num.answer.secure": 0, + "num.query.aggressive.NOERROR": 2, + "num.query.aggressive.NXDOMAIN": 0, + "num.query.authzone.down": 0, + "num.query.authzone.up": 0, + "num.query.class.IN": 100, + "num.query.dnscrypt.replay": 0, + "num.query.dnscrypt.shared_secret.cachemiss": 0, + "num.query.edns.DO": 0, + "num.query.edns.present": 0, + "num.query.flags.AA": 0, + "num.query.flags.AD": 0, + "num.query.flags.CD": 0, + "num.query.flags.QR": 0, + "num.query.flags.RA": 0, + "num.query.flags.RD": 100, + "num.query.flags.TC": 0, + "num.query.flags.Z": 0, + "num.query.ipv6": 0, + "num.query.opcode.QUERY": 100, + "num.query.ratelimited": 0, + "num.query.subnet": 0, + "num.query.subnet_cache": 0, + "num.query.tcp": 0, + "num.query.tcpout": 1, + "num.query.tls": 0, + "num.query.tls.resume": 0, + "num.query.type.A": 30, + "num.query.type.AAAA": 30, + "num.query.type.MX": 30, + "num.query.type.PTR": 10, + "num.rrset.bogus": 0, + "rrset.cache.count": 303, + "thread0.num.cachehits": 44, + "thread0.num.cachemiss": 7, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 51, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 7, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 365, + "thread0.recursion.time.median": 57, + "thread0.requestlist.avg": 0, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 0, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 46, + "thread1.num.cachemiss": 3, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 49, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 3, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 1582, + "thread1.recursion.time.median": 0, + "thread1.requestlist.avg": 0, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 0, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 45, + "time.now": 1574103644, + "time.up": 45, + "total.num.cachehits": 90, + "total.num.cachemiss": 10, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 100, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 10, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 730, + "total.recursion.time.median": 28, + "total.requestlist.avg": 0, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 0, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + "unwanted.queries": 0, + "unwanted.replies": 0, + } + expectedReset2 = map[string]int64{ + "dnscrypt_nonce.cache.count": 0, + "dnscrypt_shared_secret.cache.count": 0, + "infra.cache.count": 181, + "key.cache.count": 10, + "mem.cache.dnscrypt_nonce": 0, + "mem.cache.dnscrypt_shared_secret": 0, + "mem.cache.message": 86064, + "mem.cache.rrset": 172757, + "mem.mod.iterator": 16588, + "mem.mod.respip": 0, + "mem.mod.subnet": 74504, + "mem.mod.validator": 79979, + "mem.streamwait": 0, + "msg.cache.count": 67, + "num.answer.bogus": 0, + "num.answer.rcode.FORMERR": 0, + "num.answer.rcode.NOERROR": 0, + "num.answer.rcode.NOTIMPL": 0, + "num.answer.rcode.NXDOMAIN": 0, + "num.answer.rcode.REFUSED": 0, + "num.answer.rcode.SERVFAIL": 0, + "num.answer.rcode.nodata": 0, + "num.answer.secure": 0, + "num.query.aggressive.NOERROR": 0, + "num.query.aggressive.NXDOMAIN": 0, + "num.query.authzone.down": 0, + "num.query.authzone.up": 0, + "num.query.class.IN": 0, + "num.query.dnscrypt.replay": 0, + "num.query.dnscrypt.shared_secret.cachemiss": 0, + "num.query.edns.DO": 0, + "num.query.edns.present": 0, + "num.query.flags.AA": 0, + "num.query.flags.AD": 0, + "num.query.flags.CD": 0, + "num.query.flags.QR": 0, + "num.query.flags.RA": 0, + "num.query.flags.RD": 0, + "num.query.flags.TC": 0, + "num.query.flags.Z": 0, + "num.query.ipv6": 0, + "num.query.opcode.QUERY": 0, + "num.query.ratelimited": 0, + "num.query.subnet": 0, + "num.query.subnet_cache": 0, + "num.query.tcp": 0, + "num.query.tcpout": 0, + "num.query.tls": 0, + "num.query.tls.resume": 0, + "num.query.type.A": 0, + "num.query.type.AAAA": 0, + "num.query.type.MX": 0, + "num.query.type.PTR": 0, + "num.rrset.bogus": 0, + "rrset.cache.count": 303, + "thread0.num.cachehits": 0, + "thread0.num.cachemiss": 0, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 0, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 0, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 0, + "thread0.recursion.time.median": 0, + "thread0.requestlist.avg": 0, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 0, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 0, + "thread1.num.cachemiss": 0, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 0, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 0, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 0, + "thread1.recursion.time.median": 0, + "thread1.requestlist.avg": 0, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 0, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 26, + "time.now": 1574103671, + "time.up": 71, + "total.num.cachehits": 0, + "total.num.cachemiss": 0, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 0, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 0, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 0, + "total.recursion.time.median": 0, + "total.requestlist.avg": 0, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 0, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + "unwanted.queries": 0, + "unwanted.replies": 0, + } + + expectedReset3 = map[string]int64{ + "dnscrypt_nonce.cache.count": 0, + "dnscrypt_shared_secret.cache.count": 0, + "infra.cache.count": 303, + "key.cache.count": 15, + "mem.cache.dnscrypt_nonce": 0, + "mem.cache.dnscrypt_shared_secret": 0, + "mem.cache.message": 105471, + "mem.cache.rrset": 235917, + "mem.mod.iterator": 16588, + "mem.mod.respip": 0, + "mem.mod.subnet": 74504, + "mem.mod.validator": 87270, + "mem.streamwait": 0, + "msg.cache.count": 127, + "num.answer.bogus": 0, + "num.answer.rcode.FORMERR": 0, + "num.answer.rcode.NOERROR": 60, + "num.answer.rcode.NOTIMPL": 0, + "num.answer.rcode.NXDOMAIN": 10, + "num.answer.rcode.REFUSED": 0, + "num.answer.rcode.SERVFAIL": 0, + "num.answer.rcode.nodata": 10, + "num.answer.secure": 0, + "num.query.aggressive.NOERROR": 2, + "num.query.aggressive.NXDOMAIN": 0, + "num.query.authzone.down": 0, + "num.query.authzone.up": 0, + "num.query.class.IN": 70, + "num.query.dnscrypt.replay": 0, + "num.query.dnscrypt.shared_secret.cachemiss": 0, + "num.query.edns.DO": 0, + "num.query.edns.present": 0, + "num.query.flags.AA": 0, + "num.query.flags.AD": 0, + "num.query.flags.CD": 0, + "num.query.flags.QR": 0, + "num.query.flags.RA": 0, + "num.query.flags.RD": 70, + "num.query.flags.TC": 0, + "num.query.flags.Z": 0, + "num.query.ipv6": 0, + "num.query.opcode.QUERY": 70, + "num.query.ratelimited": 0, + "num.query.subnet": 0, + "num.query.subnet_cache": 0, + "num.query.tcp": 0, + "num.query.tcpout": 0, + "num.query.tls": 0, + "num.query.tls.resume": 0, + "num.query.type.A": 20, + "num.query.type.AAAA": 20, + "num.query.type.MX": 20, + "num.query.type.PTR": 10, + "num.rrset.bogus": 0, + "rrset.cache.count": 501, + "thread0.num.cachehits": 30, + "thread0.num.cachemiss": 4, + "thread0.num.dnscrypt.cert": 0, + "thread0.num.dnscrypt.cleartext": 0, + "thread0.num.dnscrypt.crypted": 0, + "thread0.num.dnscrypt.malformed": 0, + "thread0.num.prefetch": 0, + "thread0.num.queries": 34, + "thread0.num.queries_ip_ratelimited": 0, + "thread0.num.recursivereplies": 4, + "thread0.num.zero_ttl": 0, + "thread0.recursion.time.avg": 541, + "thread0.recursion.time.median": 98, + "thread0.requestlist.avg": 0, + "thread0.requestlist.current.all": 0, + "thread0.requestlist.current.user": 0, + "thread0.requestlist.exceeded": 0, + "thread0.requestlist.max": 0, + "thread0.requestlist.overwritten": 0, + "thread0.tcpusage": 0, + "thread1.num.cachehits": 33, + "thread1.num.cachemiss": 3, + "thread1.num.dnscrypt.cert": 0, + "thread1.num.dnscrypt.cleartext": 0, + "thread1.num.dnscrypt.crypted": 0, + "thread1.num.dnscrypt.malformed": 0, + "thread1.num.prefetch": 0, + "thread1.num.queries": 36, + "thread1.num.queries_ip_ratelimited": 0, + "thread1.num.recursivereplies": 3, + "thread1.num.zero_ttl": 0, + "thread1.recursion.time.avg": 62, + "thread1.recursion.time.median": 0, + "thread1.requestlist.avg": 1666, + "thread1.requestlist.current.all": 0, + "thread1.requestlist.current.user": 0, + "thread1.requestlist.exceeded": 0, + "thread1.requestlist.max": 5, + "thread1.requestlist.overwritten": 0, + "thread1.tcpusage": 0, + "time.elapsed": 59, + "time.now": 1574103731, + "time.up": 131, + "total.num.cachehits": 63, + "total.num.cachemiss": 7, + "total.num.dnscrypt.cert": 0, + "total.num.dnscrypt.cleartext": 0, + "total.num.dnscrypt.crypted": 0, + "total.num.dnscrypt.malformed": 0, + "total.num.prefetch": 0, + "total.num.queries": 70, + "total.num.queries_ip_ratelimited": 0, + "total.num.recursivereplies": 7, + "total.num.zero_ttl": 0, + "total.recursion.time.avg": 336, + "total.recursion.time.median": 49, + "total.requestlist.avg": 714, + "total.requestlist.current.all": 0, + "total.requestlist.current.user": 0, + "total.requestlist.exceeded": 0, + "total.requestlist.max": 5, + "total.requestlist.overwritten": 0, + "total.tcpusage": 0, + "unwanted.queries": 0, + "unwanted.replies": 0, + } +) From 6a28cf0b4d4dc96ae31d307576129b228889cb2a Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Mon, 18 Nov 2019 22:55:33 +0300 Subject: [PATCH 20/39] collect tests --- modules/unbound/unbound_test.go | 273 +++++++++++++++----------------- 1 file changed, 125 insertions(+), 148 deletions(-) diff --git a/modules/unbound/unbound_test.go b/modules/unbound/unbound_test.go index 79625cdf4..3082b133a 100644 --- a/modules/unbound/unbound_test.go +++ b/modules/unbound/unbound_test.go @@ -49,11 +49,15 @@ func TestUnbound_Check(t *testing.T) { } func TestUnbound_Cleanup(t *testing.T) { - + New().Cleanup() } func TestUnbound_Charts(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + assert.NotNil(t, unbound.Charts()) } func TestUnbound_Collect(t *testing.T) { @@ -62,17 +66,124 @@ func TestUnbound_Collect(t *testing.T) { require.True(t, unbound.Init()) unbound.client = mockUnboundClient{data: commonStatsData, err: false} - //m := unbound.Collect() - //_ = m - //l := make([]string, 0) - //for k := range m { - // l = append(l, k) - //} - //sort.Strings(l) - //for _, v := range l { - // fmt.Println(fmt.Sprintf("\"%s\": %d,", v, m[v])) - //} - expected := map[string]int64{ + assert.Equal(t, expectedCommon, unbound.Collect()) +} + +func TestUnbound_Collect_ExtendedStats(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: extStatsData, err: false} + + assert.Equal(t, expectedExtended, unbound.Collect()) +} + +func TestUnbound_Collect_LifeCycleCumulativeExtendedStats(t *testing.T) { + tests := []struct { + input []byte + expected map[string]int64 + }{ + {input: lifeCycleCumulativeData1, expected: expectedCumulative1}, + {input: lifeCycleCumulativeData2, expected: expectedCumulative2}, + {input: lifeCycleCumulativeData3, expected: expectedCumulative3}, + } + + unbound := New() + unbound.DisableTLS = true + unbound.Cumulative = true + require.True(t, unbound.Init()) + ubClient := &mockUnboundClient{err: false} + unbound.client = ubClient + + for i, test := range tests { + t.Run(fmt.Sprintf("run %d", i+1), func(t *testing.T) { + ubClient.data = test.input + assert.Equal(t, test.expected, unbound.Collect()) + }) + } +} + +func TestUnbound_Collect_LifeCycleResetExtendedStats(t *testing.T) { + tests := []struct { + input []byte + expected map[string]int64 + }{ + {input: lifeCycleResetData1, expected: expectedReset1}, + {input: lifeCycleResetData2, expected: expectedReset2}, + {input: lifeCycleResetData3, expected: expectedReset3}, + } + + unbound := New() + unbound.DisableTLS = true + unbound.Cumulative = false + require.True(t, unbound.Init()) + ubClient := &mockUnboundClient{err: false} + unbound.client = ubClient + + for i, test := range tests { + t.Run(fmt.Sprintf("run %d", i+1), func(t *testing.T) { + ubClient.data = test.input + assert.Equal(t, test.expected, unbound.Collect()) + }) + } +} + +func TestUnbound_Collect_EmptyResponse(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: []byte{}, err: false} + + assert.Nil(t, unbound.Collect()) +} + +func TestUnbound_Collect_ErrorResponse(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: []byte("error unknown command 'unknown'"), err: false} + + assert.Nil(t, unbound.Collect()) +} + +func TestUnbound_Collect_ErrorOnSend(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{err: true} + + assert.Nil(t, unbound.Collect()) +} + +func TestUnbound_Collect_ErrorOnParseBadSyntax(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + data := strings.Repeat("zk_avg_latency 0\nzk_min_latency 0\nzk_mix_latency 0\n", 10) + unbound.client = mockUnboundClient{data: []byte(data), err: false} + + assert.Nil(t, unbound.Collect()) +} + +type mockUnboundClient struct { + data []byte + err bool +} + +func (m mockUnboundClient) send(command string) ([]string, error) { + if m.err { + return nil, errors.New("mock send error") + } + rv := make([]string, 0, 160) + s := bufio.NewScanner(bytes.NewReader(m.data)) + for s.Scan() { + rv = append(rv, s.Text()) + } + return rv, nil +} + +var ( + expectedCommon = map[string]int64{ "thread0.num.cachehits": 21, "thread0.num.cachemiss": 7, "thread0.num.dnscrypt.cert": 0, @@ -138,16 +249,7 @@ func TestUnbound_Collect(t *testing.T) { "total.tcpusage": 0, } - assert.Equal(t, expected, unbound.Collect()) -} - -func TestUnbound_Collect_ExtendedStats(t *testing.T) { - unbound := New() - unbound.DisableTLS = true - require.True(t, unbound.Init()) - unbound.client = mockUnboundClient{data: extStatsData, err: false} - - expected := map[string]int64{ + expectedExtended = map[string]int64{ "dnscrypt_nonce.cache.count": 0, "dnscrypt_shared_secret.cache.count": 0, "infra.cache.count": 205, @@ -268,132 +370,7 @@ func TestUnbound_Collect_ExtendedStats(t *testing.T) { "unwanted.queries": 0, "unwanted.replies": 0, } - - assert.Equal(t, expected, unbound.Collect()) -} - -func TestUnbound_Collect_LifeCycleCumulativeExtendedStats(t *testing.T) { - tests := []struct { - input []byte - expected map[string]int64 - }{ - { - input: lifeCycleCumulativeData1, - expected: expectedCumulative1, - }, - { - input: lifeCycleCumulativeData2, - expected: expectedCumulative2, - }, - { - input: lifeCycleCumulativeData3, - expected: expectedCumulative3, - }, - } - - unbound := New() - unbound.DisableTLS = true - unbound.Cumulative = true - require.True(t, unbound.Init()) - ubClient := &mockUnboundClient{err: false} - unbound.client = ubClient - - for i, test := range tests { - t.Run(fmt.Sprintf("run %d", i+1), func(t *testing.T) { - ubClient.data = test.input - assert.Equal(t, test.expected, unbound.Collect()) - }) - } -} - -func TestUnbound_Collect_LifeCycleResetExtendedStats(t *testing.T) { - tests := []struct { - input []byte - expected map[string]int64 - }{ - { - input: lifeCycleResetData1, - expected: expectedReset1, - }, - { - input: lifeCycleResetData2, - expected: expectedReset2, - }, - { - input: lifeCycleResetData3, - expected: expectedReset3, - }, - } - _ = tests - - unbound := New() - unbound.DisableTLS = true - unbound.Cumulative = false - require.True(t, unbound.Init()) - ubClient := &mockUnboundClient{err: false} - unbound.client = ubClient - - for i, test := range tests { - t.Run(fmt.Sprintf("run %d", i+1), func(t *testing.T) { - ubClient.data = test.input - assert.Equal(t, test.expected, unbound.Collect()) - }) - } -} - -func TestUnbound_Collect_EmptyResponse(t *testing.T) { - unbound := New() - unbound.DisableTLS = true - require.True(t, unbound.Init()) - unbound.client = mockUnboundClient{data: []byte{}, err: false} - - assert.Nil(t, unbound.Collect()) -} - -func TestUnbound_Collect_ErrorResponse(t *testing.T) { - unbound := New() - unbound.DisableTLS = true - require.True(t, unbound.Init()) - unbound.client = mockUnboundClient{data: []byte("error unknown command 'unknown'"), err: false} - - assert.Nil(t, unbound.Collect()) -} - -func TestUnbound_Collect_ErrorOnSend(t *testing.T) { - unbound := New() - unbound.DisableTLS = true - require.True(t, unbound.Init()) - unbound.client = mockUnboundClient{err: true} - - assert.Nil(t, unbound.Collect()) -} - -func TestUnbound_Collect_BadSyntaxResponse(t *testing.T) { - unbound := New() - unbound.DisableTLS = true - require.True(t, unbound.Init()) - data := strings.Repeat("zk_avg_latency 0\nzk_min_latency 0\nzk_mix_latency 0\n", 10) - unbound.client = mockUnboundClient{data: []byte(data), err: false} - - assert.Nil(t, unbound.Collect()) -} - -type mockUnboundClient struct { - data []byte - err bool -} - -func (m mockUnboundClient) send(command string) ([]string, error) { - if m.err { - return nil, errors.New("mock send error") - } - var rv []string - s := bufio.NewScanner(bytes.NewReader(m.data)) - for s.Scan() { - rv = append(rv, s.Text()) - } - return rv, nil -} +) var ( expectedCumulative1 = map[string]int64{ From 6aa2e125c8a2339491fa98c5c6ea9ad50d5e5dd3 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 01:24:09 +0300 Subject: [PATCH 21/39] tests --- modules/unbound/collect.go | 14 +-- modules/unbound/init.go | 2 +- modules/unbound/testdata/unbound.conf | 14 +-- .../unbound/testdata/unbound_disabled.conf | 85 +++++++++++++++++++ modules/unbound/testdata/unbound_empty.conf | 85 +++++++++++++++++++ modules/unbound/unbound_test.go | 64 ++++++++++++++ 6 files changed, 250 insertions(+), 14 deletions(-) create mode 100644 modules/unbound/testdata/unbound_disabled.conf create mode 100644 modules/unbound/testdata/unbound_empty.conf diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 95b826188..36d047110 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -6,11 +6,10 @@ import ( "strings" ) -// https://github.com/NLnetLabs/unbound/blob/master/smallapp/unbound-control.c -// https://github.com/NLnetLabs/unbound/blob/master/libunbound/unbound.h (ub_server_stats, ub_shm_stat_info) -// https://github.com/NLnetLabs/unbound/blob/master/daemon/remote.c -// https://docs.menandmice.com/display/MM/Unbound+request-list+demystified -// https://docs.datadoghq.com/integrations/unbound/#metrics +// https://github.com/NLnetLabs/unbound/blob/master/daemon/remote.c (do_stats: print_stats, print_thread_stats, print_mem, print_uptime, print_ext) +// https://github.com/NLnetLabs/unbound/blob/master/libunbound/unbound.h (structs: ub_server_stats, ub_shm_stat_info) +// https://docs.datadoghq.com/integrations/unbound/#metrics (stats description) +// https://docs.menandmice.com/display/MM/Unbound+request-list+demystified (request lists explanation) func (u *Unbound) collect() (map[string]int64, error) { stats, err := u.scrapeUnboundStats() @@ -24,7 +23,10 @@ func (u *Unbound) collect() (map[string]int64, error) { } func (u *Unbound) scrapeUnboundStats() ([]entry, error) { - const command = "UBCT1 stats" + var command = "UBCT1 stats" + if u.Cumulative { + command = "UBCT1 stats_noreset" + } output, err := u.client.send(command + "\n") if err != nil { return nil, fmt.Errorf("send command '%s': %w", command, err) diff --git a/modules/unbound/init.go b/modules/unbound/init.go index f51f97e30..398e69ac1 100644 --- a/modules/unbound/init.go +++ b/modules/unbound/init.go @@ -85,7 +85,7 @@ func (u *Unbound) applyConfig(cfg *unboundConfig) { return } if cfg.RC.UseCert.isSet() { - if cfg.RC.UseCert.value() != u.DisableTLS { + if cfg.RC.UseCert.value() == u.DisableTLS { u.Debugf("changing 'disable_tls': %v => %v", u.DisableTLS, !cfg.RC.UseCert.value()) u.DisableTLS = !cfg.RC.UseCert.value() } diff --git a/modules/unbound/testdata/unbound.conf b/modules/unbound/testdata/unbound.conf index 31a7a59c9..a061a3476 100644 --- a/modules/unbound/testdata/unbound.conf +++ b/modules/unbound/testdata/unbound.conf @@ -34,10 +34,10 @@ server: # enable extended statistics (query types, answer codes, status) # printed from unbound-control. default off, because of speed. # extended-statistics: no - extended-statistics: yes + # extended-statistics: yes # number of threads to create. 1 disables threading. - num-threads: 2 + # num-threads: 2 # Python config section. To enable: # o use --with-pythonmodule to configure before compiling. @@ -60,17 +60,17 @@ remote-control: # set to an absolute path to use a unix local name pipe, certificates # are not used for that, so key and cert files need not be present. # control-interface: 127.0.0.1 - control-interface: 0.0.0.0 + control-interface: 10.0.0.1 # control-interface: ::1 # control-interface: /var/run/test.sock # port number for remote control operations. - control-port: 8953 + control-port: 8954 # for localhost, you can disable use of TLS by setting this to "no" # For local sockets this option is ignored, and TLS is not used. # control-use-cert: "yes" - control-use-cert: "yes" + control-use-cert: "no" # unbound server key file. # server-key-file: "/etc/unbound/unbound_server.key" @@ -79,7 +79,7 @@ remote-control: # server-cert-file: "/etc/unbound/unbound_server.pem" # unbound-control key file. - control-key-file: "/etc/unbound/unbound_control.key" + control-key-file: "/etc/unbound/unbound_control_other.key" # unbound-control certificate file. - control-cert-file: "/etc/unbound/unbound_control.pem" + control-cert-file: "/etc/unbound/unbound_control_other.pem" diff --git a/modules/unbound/testdata/unbound_disabled.conf b/modules/unbound/testdata/unbound_disabled.conf new file mode 100644 index 000000000..1cef549f8 --- /dev/null +++ b/modules/unbound/testdata/unbound_disabled.conf @@ -0,0 +1,85 @@ +# +# Example configuration file. +# +# See unbound.conf(5) man page, version 1.9.4. +# +# this is a comment. + +#Use this to include other text into the file. +#include: "otherfile.conf" + +# The server clause sets the main parameters. +server: + # whitespace is not necessary, but looks cleaner. + + # verbosity number, 0 is least verbose. 1 is default. + # verbosity: 1 + + # print statistics to the log (for every thread) every N seconds. + # Set to "" or 0 to disable. Default is disabled. + # statistics-interval: 0 + + # enable shm for stats, default no. if you enable also enable + # statistics-interval, every time it also writes stats to the + # shared memory segment keyed with shm-key. + # shm-enable: no + + # shm for stats uses this key, and key+1 for the shared mem segment. + # shm-key: 11777 + + # enable cumulative statistics, without clearing them after printing. + # statistics-cumulative: no + statistics-cumulative: yes + + # enable extended statistics (query types, answer codes, status) + # printed from unbound-control. default off, because of speed. + # extended-statistics: no + # extended-statistics: yes + + # number of threads to create. 1 disables threading. + # num-threads: 2 + +# Python config section. To enable: +# o use --with-pythonmodule to configure before compiling. +# o list python in the module-config string (above) to enable. +# It can be at the start, it gets validated results, or just before +# the iterator and process before DNSSEC validation. +# o and give a python-script to run. +python: + # Script file to load + # python-script: "/etc/unbound/ubmodule-tst.py" + +# Remote control config section. +remote-control: + # Enable remote control with unbound-control(8) here. + # set up the keys and certificates with unbound-control-setup. + control-enable: no + + # what interfaces are listened to for remote control. + # give 0.0.0.0 and ::0 to listen to all interfaces. + # set to an absolute path to use a unix local name pipe, certificates + # are not used for that, so key and cert files need not be present. + # control-interface: 127.0.0.1 + control-interface: 0.0.0.0 + # control-interface: ::1 + # control-interface: /var/run/test.sock + + # port number for remote control operations. + control-port: 8953 + + # for localhost, you can disable use of TLS by setting this to "no" + # For local sockets this option is ignored, and TLS is not used. + # control-use-cert: "yes" + control-use-cert: "yes" + + # unbound server key file. + # server-key-file: "/etc/unbound/unbound_server.key" + + # unbound server certificate file. + # server-cert-file: "/etc/unbound/unbound_server.pem" + + # unbound-control key file. + control-key-file: "/etc/unbound/unbound_control.key" + + # unbound-control certificate file. + control-cert-file: "/etc/unbound/unbound_control.pem" diff --git a/modules/unbound/testdata/unbound_empty.conf b/modules/unbound/testdata/unbound_empty.conf new file mode 100644 index 000000000..a2d158376 --- /dev/null +++ b/modules/unbound/testdata/unbound_empty.conf @@ -0,0 +1,85 @@ +# +# Example configuration file. +# +# See unbound.conf(5) man page, version 1.9.4. +# +# this is a comment. + +#Use this to include other text into the file. +#include: "otherfile.conf" + +# The server clause sets the main parameters. +server: + # whitespace is not necessary, but looks cleaner. + + # verbosity number, 0 is least verbose. 1 is default. + # verbosity: 1 + + # print statistics to the log (for every thread) every N seconds. + # Set to "" or 0 to disable. Default is disabled. + # statistics-interval: 0 + + # enable shm for stats, default no. if you enable also enable + # statistics-interval, every time it also writes stats to the + # shared memory segment keyed with shm-key. + # shm-enable: no + + # shm for stats uses this key, and key+1 for the shared mem segment. + # shm-key: 11777 + + # enable cumulative statistics, without clearing them after printing. + # statistics-cumulative: no + # statistics-cumulative: yes + + # enable extended statistics (query types, answer codes, status) + # printed from unbound-control. default off, because of speed. + # extended-statistics: no + # extended-statistics: yes + + # number of threads to create. 1 disables threading. + # num-threads: 2 + +# Python config section. To enable: +# o use --with-pythonmodule to configure before compiling. +# o list python in the module-config string (above) to enable. +# It can be at the start, it gets validated results, or just before +# the iterator and process before DNSSEC validation. +# o and give a python-script to run. +python: + # Script file to load + # python-script: "/etc/unbound/ubmodule-tst.py" + +# Remote control config section. +remote-control: + # Enable remote control with unbound-control(8) here. + # set up the keys and certificates with unbound-control-setup. + # control-enable: no + + # what interfaces are listened to for remote control. + # give 0.0.0.0 and ::0 to listen to all interfaces. + # set to an absolute path to use a unix local name pipe, certificates + # are not used for that, so key and cert files need not be present. + # control-interface: 127.0.0.1 + # control-interface: 0.0.0.0 + # control-interface: ::1 + # control-interface: /var/run/test.sock + + # port number for remote control operations. + # control-port: 8953 + + # for localhost, you can disable use of TLS by setting this to "no" + # For local sockets this option is ignored, and TLS is not used. + # control-use-cert: "yes" + # control-use-cert: "yes" + + # unbound server key file. + # server-key-file: "/etc/unbound/unbound_server.key" + + # unbound server certificate file. + # server-cert-file: "/etc/unbound/unbound_server.pem" + + # unbound-control key file. + # control-key-file: "/etc/unbound/unbound_control.key" + + # unbound-control certificate file. + # control-cert-file: "/etc/unbound/unbound_control.pem" diff --git a/modules/unbound/unbound_test.go b/modules/unbound/unbound_test.go index 3082b133a..5851f2e65 100644 --- a/modules/unbound/unbound_test.go +++ b/modules/unbound/unbound_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/netdata/go.d.plugin/pkg/web" + "github.com/netdata/go-orchestrator/module" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -41,11 +43,73 @@ func TestNew(t *testing.T) { } func TestUnbound_Init(t *testing.T) { + unbound := New() + unbound.ConfPath = "" + unbound.DisableTLS = true + + assert.True(t, unbound.Init()) +} + +func TestUnbound_Init_SetEverythingFromUnboundConf(t *testing.T) { + unbound := New() + unbound.ConfPath = "testdata/unbound.conf" + expectedConfig := Config{ + Address: "10.0.0.1:8954", + ConfPath: unbound.ConfPath, + Timeout: unbound.Timeout, + DisableTLS: true, + Cumulative: true, + ClientTLSConfig: web.ClientTLSConfig{ + TLSCert: "/etc/unbound/unbound_control_other.pem", + TLSKey: "/etc/unbound/unbound_control_other.key", + InsecureSkipVerify: unbound.InsecureSkipVerify, + }, + } + + assert.True(t, unbound.Init()) + assert.Equal(t, expectedConfig, unbound.Config) +} + +func TestUnbound_Init_DisabledInUnboundConf(t *testing.T) { + unbound := New() + unbound.ConfPath = "testdata/unbound_disabled.conf" + unbound.DisableTLS = true + + assert.False(t, unbound.Init()) +} + +func TestUnbound_Init_HandleEmptyConfig(t *testing.T) { + unbound := New() + unbound.ConfPath = "testdata/unbound_empty.conf" + unbound.DisableTLS = true + assert.True(t, unbound.Init()) +} + +func TestUnbound_Init_HandleNonExistentConfig(t *testing.T) { + unbound := New() + unbound.ConfPath = "testdata/unbound_non_existent.conf" + unbound.DisableTLS = true + + assert.True(t, unbound.Init()) } func TestUnbound_Check(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{data: commonStatsData, err: false} + + assert.True(t, unbound.Check()) +} + +func TestUnbound_Check_ErrorDuringScrapingUnbound(t *testing.T) { + unbound := New() + unbound.DisableTLS = true + require.True(t, unbound.Init()) + unbound.client = mockUnboundClient{err: true} + assert.False(t, unbound.Check()) } func TestUnbound_Cleanup(t *testing.T) { From 9ff117a752527b53e6ebee12ebc9fc5d9ae68151 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 01:32:41 +0300 Subject: [PATCH 22/39] comment out Test_clientFetch --- modules/zookeeper/client_test.go | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/modules/zookeeper/client_test.go b/modules/zookeeper/client_test.go index 0f6fbfaed..aa1aad6fe 100644 --- a/modules/zookeeper/client_test.go +++ b/modules/zookeeper/client_test.go @@ -15,26 +15,26 @@ const ( testServerAddress = "127.0.0.1:38001" ) -func Test_clientFetch(t *testing.T) { - srv := &tcpServer{addr: testServerAddress, rowsNumResp: 10} - go srv.Run() - defer srv.Close() - time.Sleep(time.Second) - - c := newClient(clientConfig{ - network: "tcp", - address: testServerAddress, - timeout: time.Second, - }) - - rows, err := c.fetch("whatever\n") - assert.NoError(t, err) - assert.Len(t, rows, 10) - - rows, err = c.fetch("whatever\n") - assert.NoError(t, err) - assert.Len(t, rows, 10) -} +//func Test_clientFetch(t *testing.T) { +// srv := &tcpServer{addr: testServerAddress, rowsNumResp: 10} +// go srv.Run() +// defer srv.Close() +// time.Sleep(time.Second) +// +// c := newClient(clientConfig{ +// network: "tcp", +// address: testServerAddress, +// timeout: time.Second, +// }) +// +// rows, err := c.fetch("whatever\n") +// assert.NoError(t, err) +// assert.Len(t, rows, 10) +// +// rows, err = c.fetch("whatever\n") +// assert.NoError(t, err) +// assert.Len(t, rows, 10) +//} func Test_clientFetchReadLineLimitExceeded(t *testing.T) { srv := &tcpServer{addr: testServerAddress, rowsNumResp: maxLinesToRead + 1} From 433db7144dabb00e7fb5257a11c0567bc681a89d Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 01:35:53 +0300 Subject: [PATCH 23/39] revert comment out Test_clientFetch --- modules/unbound/client_test.go | 2 +- modules/zookeeper/client_test.go | 40 ++++++++++++++++---------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/modules/unbound/client_test.go b/modules/unbound/client_test.go index 6769f0234..8f9667c3d 100644 --- a/modules/unbound/client_test.go +++ b/modules/unbound/client_test.go @@ -12,7 +12,7 @@ import ( ) const ( - srvAddress = "127.0.0.1:38001" + srvAddress = "127.0.0.1:38002" ) func Test_clientSend(t *testing.T) { diff --git a/modules/zookeeper/client_test.go b/modules/zookeeper/client_test.go index aa1aad6fe..0f6fbfaed 100644 --- a/modules/zookeeper/client_test.go +++ b/modules/zookeeper/client_test.go @@ -15,26 +15,26 @@ const ( testServerAddress = "127.0.0.1:38001" ) -//func Test_clientFetch(t *testing.T) { -// srv := &tcpServer{addr: testServerAddress, rowsNumResp: 10} -// go srv.Run() -// defer srv.Close() -// time.Sleep(time.Second) -// -// c := newClient(clientConfig{ -// network: "tcp", -// address: testServerAddress, -// timeout: time.Second, -// }) -// -// rows, err := c.fetch("whatever\n") -// assert.NoError(t, err) -// assert.Len(t, rows, 10) -// -// rows, err = c.fetch("whatever\n") -// assert.NoError(t, err) -// assert.Len(t, rows, 10) -//} +func Test_clientFetch(t *testing.T) { + srv := &tcpServer{addr: testServerAddress, rowsNumResp: 10} + go srv.Run() + defer srv.Close() + time.Sleep(time.Second) + + c := newClient(clientConfig{ + network: "tcp", + address: testServerAddress, + timeout: time.Second, + }) + + rows, err := c.fetch("whatever\n") + assert.NoError(t, err) + assert.Len(t, rows, 10) + + rows, err = c.fetch("whatever\n") + assert.NoError(t, err) + assert.Len(t, rows, 10) +} func Test_clientFetchReadLineLimitExceeded(t *testing.T) { srv := &tcpServer{addr: testServerAddress, rowsNumResp: maxLinesToRead + 1} From f50acf9f1098d758330709b407d7c19db3a38fc5 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 10:57:08 +0300 Subject: [PATCH 24/39] test charts --- modules/unbound/unbound_test.go | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/modules/unbound/unbound_test.go b/modules/unbound/unbound_test.go index 5851f2e65..9e825237d 100644 --- a/modules/unbound/unbound_test.go +++ b/modules/unbound/unbound_test.go @@ -131,6 +131,7 @@ func TestUnbound_Collect(t *testing.T) { unbound.client = mockUnboundClient{data: commonStatsData, err: false} assert.Equal(t, expectedCommon, unbound.Collect()) + testCharts(t, unbound) } func TestUnbound_Collect_ExtendedStats(t *testing.T) { @@ -140,6 +141,7 @@ func TestUnbound_Collect_ExtendedStats(t *testing.T) { unbound.client = mockUnboundClient{data: extStatsData, err: false} assert.Equal(t, expectedExtended, unbound.Collect()) + testCharts(t, unbound) } func TestUnbound_Collect_LifeCycleCumulativeExtendedStats(t *testing.T) { @@ -165,6 +167,8 @@ func TestUnbound_Collect_LifeCycleCumulativeExtendedStats(t *testing.T) { assert.Equal(t, test.expected, unbound.Collect()) }) } + + testCharts(t, unbound) } func TestUnbound_Collect_LifeCycleResetExtendedStats(t *testing.T) { @@ -190,6 +194,8 @@ func TestUnbound_Collect_LifeCycleResetExtendedStats(t *testing.T) { assert.Equal(t, test.expected, unbound.Collect()) }) } + + testCharts(t, unbound) } func TestUnbound_Collect_EmptyResponse(t *testing.T) { @@ -246,6 +252,54 @@ func (m mockUnboundClient) send(command string) ([]string, error) { return rv, nil } +func testCharts(t *testing.T, u *Unbound) { + t.Helper() + testThreadCharts(t, u) + testExtendedCharts(t, u) +} + +func testThreadCharts(t *testing.T, u *Unbound) { + for thread := range u.cache.threads { + for _, chart := range *threadCharts(thread, u.Cumulative) { + assert.Truef(t, u.Charts().Has(chart.ID), "chart '%s' is not created for '%s' thread", chart.ID, thread) + } + } +} + +func testExtendedCharts(t *testing.T, u *Unbound) { + if len(u.cache.answerRCode) == 0 { + return + } + for _, chart := range *extendedCharts(u.Cumulative) { + assert.Truef(t, u.Charts().Has(chart.ID), "chart '%s' is not added", chart.ID) + } + + if chart := u.Charts().Get(queryTypeChart.ID); chart != nil { + for typ := range u.cache.queryType { + dimID := "num.query.type." + typ + assert.Truef(t, chart.HasDim(dimID), "chart '%s' has no dim for '%s' type, expected '%s'", chart.ID, typ, dimID) + } + } + if chart := u.Charts().Get(queryClassChart.ID); chart != nil { + for class := range u.cache.queryClass { + dimID := "num.query.class." + class + assert.Truef(t, chart.HasDim(dimID), "chart '%s' has no dim for '%s' class, expected '%s'", chart.ID, class, dimID) + } + } + if chart := u.Charts().Get(queryOpCodeChart.ID); chart != nil { + for opcode := range u.cache.queryOpCode { + dimID := "num.query.opcode." + opcode + assert.Truef(t, chart.HasDim(dimID), "chart '%s' has no dim for '%s' opcode, expected '%s'", chart.ID, opcode, dimID) + } + } + if chart := u.Charts().Get(answerRCodeChart.ID); chart != nil { + for rcode := range u.cache.answerRCode { + dimID := "num.answer.rcode." + rcode + assert.Truef(t, chart.HasDim(dimID), "chart '%s' has no dim for '%s' rcode, expected '%s'", chart.ID, rcode, dimID) + } + } +} + var ( expectedCommon = map[string]int64{ "thread0.num.cachehits": 21, From eb3e1271231cce2a1b84da86373f360e305a2f6e Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 11:36:31 +0300 Subject: [PATCH 25/39] disable_tls => use_tls --- config/go.d/unbound.conf | 119 ++++++++++++++++++++++++++++++++ modules/unbound/init.go | 8 +-- modules/unbound/unbound.go | 8 +-- modules/unbound/unbound_test.go | 49 +++++-------- 4 files changed, 145 insertions(+), 39 deletions(-) create mode 100644 config/go.d/unbound.conf diff --git a/config/go.d/unbound.conf b/config/go.d/unbound.conf new file mode 100644 index 000000000..5511f462a --- /dev/null +++ b/config/go.d/unbound.conf @@ -0,0 +1,119 @@ +# netdata go.d.plugin configuration for unbound +# +# This file is in YaML format. Generally the format is: +# +# name: value +# +# There are 2 sections: +# - GLOBAL +# - JOBS +# +# +# [ GLOBAL ] +# These variables set the defaults for all JOBs, however each JOB may define its own, overriding the defaults. +# +# The GLOBAL section format: +# param1: value1 +# param2: value2 +# +# Currently supported global parameters: +# - update_every +# Data collection frequency in seconds. Default: 1. +# +# - autodetection_retry +# Re-check interval in seconds. Attempts to start the job are made once every interval. +# Zero means not to schedule re-check. Default: 0. +# +# - priority +# Priority is the relative priority of the charts as rendered on the web page, +# lower numbers make the charts appear before the ones with higher numbers. Default: 70000. +# +# +# [ JOBS ] +# JOBS allow you to collect values from multiple sources. +# Each source will have its own set of charts. +# +# IMPORTANT: +# - Parameter 'name' is mandatory. +# - Jobs with the same name are mutually exclusive. Only one of them will be allowed running at any time. +# +# This allows autodetection to try several alternatives and pick the one that works. +# Any number of jobs is supported. +# +# The JOBS section format: +# +# jobs: +# - name: job1 +# param1: value1 +# param2: value2 +# +# - name: job2 +# param1: value1 +# param2: value2 +# +# - name: job2 +# param1: value1 +# +# +# [ List of JOB specific parameters ]: +# - address +# Server address. +# Syntax: +# address: 127.0.0.1:2181 +# +# - timeout +# Connection/read/write/ssl handshake timeout. +# Syntax: +# timeout: 1 +# +# - use_tls +# Whether to use or not TLS. +# Syntax: +# use_tls: true +# +# - tls_skip_verify +# Whether to skip verifying server's certificate chain and hostname. +# Syntax: +# tls_skip_verify: yes/no +# +# - tls_ca +# Certificate authority that client use when verifying server certificates. +# Syntax: +# tls_ca: path/to/ca.pem +# +# - tls_cert +# Client tls certificate. +# Syntax: +# tls_cert: path/to/cert.pem +# +# - tls_key +# Client tls key. +# Syntax: +# tls_key: path/to/key.pem +# +# +# [ JOB defaults ]: +# address: 127.0.0.1:2181 +# timeout: 1 +# use_tls: false +# tls_skip_verify: no +# +# +# [ JOB mandatory parameters ]: +# - name +# - address +# +# ------------------------------------------------MODULE-CONFIGURATION-------------------------------------------------- +# [ GLOBAL ] +# update_every: 1 +# autodetection_retry: 0 +# priority: 70000 +# +# +# [ JOBS ] +jobs: + - name: local + address: 127.0.0.1:2181 + + - name: local + address: 127.0.0.1:2182 diff --git a/modules/unbound/init.go b/modules/unbound/init.go index 398e69ac1..348db8d8a 100644 --- a/modules/unbound/init.go +++ b/modules/unbound/init.go @@ -85,9 +85,9 @@ func (u *Unbound) applyConfig(cfg *unboundConfig) { return } if cfg.RC.UseCert.isSet() { - if cfg.RC.UseCert.value() == u.DisableTLS { - u.Debugf("changing 'disable_tls': %v => %v", u.DisableTLS, !cfg.RC.UseCert.value()) - u.DisableTLS = !cfg.RC.UseCert.value() + if cfg.RC.UseCert.value() != u.UseTLS { + u.Debugf("changing 'use_tls': %v => %v", u.UseTLS, cfg.RC.UseCert.value()) + u.UseTLS = cfg.RC.UseCert.value() } } @@ -123,7 +123,7 @@ func (u *Unbound) applyConfig(cfg *unboundConfig) { func (u *Unbound) initClient() (err error) { var tlsCfg *tls.Config - useTLS := !isUnixSocket(u.Address) && !u.DisableTLS + useTLS := !isUnixSocket(u.Address) && u.UseTLS if useTLS && (u.TLSCert == "" || u.TLSKey == "") { return errors.New("'tls_cert' or 'tls_key' is missing") } diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 653ec8b6a..083c0c139 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -22,7 +22,7 @@ func New() *Unbound { ConfPath: "/etc/unbound/unbound.conf", Timeout: web.Duration{Duration: time.Second}, Cumulative: false, - DisableTLS: false, + UseTLS: true, ClientTLSConfig: web.ClientTLSConfig{ TLSCert: "/etc/unbound/unbound_control.pem", TLSKey: "/etc/unbound/unbound_control.key", @@ -46,8 +46,8 @@ type ( Address string `yaml:"address"` ConfPath string `yaml:"conf_path"` Timeout web.Duration `yaml:"timeout"` - DisableTLS bool `yaml:"disable_tls"` Cumulative bool `yaml:"cumulative_stats"` + UseTLS bool `yaml:"use_tls"` web.ClientTLSConfig `yaml:",inline"` } Unbound struct { @@ -79,8 +79,8 @@ func (u *Unbound) Init() bool { u.charts = charts(u.Cumulative) - u.Debugf("using address: %s, cumulative: %v, disable_tls: %v, timeout: %s", u.Address, u.Cumulative, u.DisableTLS, u.Timeout) - if !u.DisableTLS { + u.Debugf("using address: %s, cumulative: %v, disable_tls: %v, timeout: %s", u.Address, u.Cumulative, u.UseTLS, u.Timeout) + if !u.UseTLS { u.Debugf("using tls_skip_verify: %v, tls_key: %s, tls_cert: %s", u.InsecureSkipVerify, u.TLSKey, u.TLSCert) } return true diff --git a/modules/unbound/unbound_test.go b/modules/unbound/unbound_test.go index 9e825237d..fb92c8a9b 100644 --- a/modules/unbound/unbound_test.go +++ b/modules/unbound/unbound_test.go @@ -38,14 +38,15 @@ func Test_readTestData(t *testing.T) { assert.NotNil(t, lifeCycleResetData3) } +func nonTLSUnbound() *Unbound { unbound := New(); unbound.UseTLS = false; return unbound } + func TestNew(t *testing.T) { assert.Implements(t, (*module.Module)(nil), New()) } func TestUnbound_Init(t *testing.T) { - unbound := New() + unbound := nonTLSUnbound() unbound.ConfPath = "" - unbound.DisableTLS = true assert.True(t, unbound.Init()) } @@ -57,8 +58,8 @@ func TestUnbound_Init_SetEverythingFromUnboundConf(t *testing.T) { Address: "10.0.0.1:8954", ConfPath: unbound.ConfPath, Timeout: unbound.Timeout, - DisableTLS: true, Cumulative: true, + UseTLS: false, ClientTLSConfig: web.ClientTLSConfig{ TLSCert: "/etc/unbound/unbound_control_other.pem", TLSKey: "/etc/unbound/unbound_control_other.key", @@ -71,32 +72,28 @@ func TestUnbound_Init_SetEverythingFromUnboundConf(t *testing.T) { } func TestUnbound_Init_DisabledInUnboundConf(t *testing.T) { - unbound := New() + unbound := nonTLSUnbound() unbound.ConfPath = "testdata/unbound_disabled.conf" - unbound.DisableTLS = true assert.False(t, unbound.Init()) } func TestUnbound_Init_HandleEmptyConfig(t *testing.T) { - unbound := New() + unbound := nonTLSUnbound() unbound.ConfPath = "testdata/unbound_empty.conf" - unbound.DisableTLS = true assert.True(t, unbound.Init()) } func TestUnbound_Init_HandleNonExistentConfig(t *testing.T) { - unbound := New() + unbound := nonTLSUnbound() unbound.ConfPath = "testdata/unbound_non_existent.conf" - unbound.DisableTLS = true assert.True(t, unbound.Init()) } func TestUnbound_Check(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) unbound.client = mockUnboundClient{data: commonStatsData, err: false} @@ -104,8 +101,7 @@ func TestUnbound_Check(t *testing.T) { } func TestUnbound_Check_ErrorDuringScrapingUnbound(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) unbound.client = mockUnboundClient{err: true} @@ -117,16 +113,14 @@ func TestUnbound_Cleanup(t *testing.T) { } func TestUnbound_Charts(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) assert.NotNil(t, unbound.Charts()) } func TestUnbound_Collect(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) unbound.client = mockUnboundClient{data: commonStatsData, err: false} @@ -135,8 +129,7 @@ func TestUnbound_Collect(t *testing.T) { } func TestUnbound_Collect_ExtendedStats(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) unbound.client = mockUnboundClient{data: extStatsData, err: false} @@ -154,8 +147,7 @@ func TestUnbound_Collect_LifeCycleCumulativeExtendedStats(t *testing.T) { {input: lifeCycleCumulativeData3, expected: expectedCumulative3}, } - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() unbound.Cumulative = true require.True(t, unbound.Init()) ubClient := &mockUnboundClient{err: false} @@ -181,8 +173,7 @@ func TestUnbound_Collect_LifeCycleResetExtendedStats(t *testing.T) { {input: lifeCycleResetData3, expected: expectedReset3}, } - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() unbound.Cumulative = false require.True(t, unbound.Init()) ubClient := &mockUnboundClient{err: false} @@ -199,8 +190,7 @@ func TestUnbound_Collect_LifeCycleResetExtendedStats(t *testing.T) { } func TestUnbound_Collect_EmptyResponse(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) unbound.client = mockUnboundClient{data: []byte{}, err: false} @@ -208,8 +198,7 @@ func TestUnbound_Collect_EmptyResponse(t *testing.T) { } func TestUnbound_Collect_ErrorResponse(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) unbound.client = mockUnboundClient{data: []byte("error unknown command 'unknown'"), err: false} @@ -217,8 +206,7 @@ func TestUnbound_Collect_ErrorResponse(t *testing.T) { } func TestUnbound_Collect_ErrorOnSend(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) unbound.client = mockUnboundClient{err: true} @@ -226,8 +214,7 @@ func TestUnbound_Collect_ErrorOnSend(t *testing.T) { } func TestUnbound_Collect_ErrorOnParseBadSyntax(t *testing.T) { - unbound := New() - unbound.DisableTLS = true + unbound := nonTLSUnbound() require.True(t, unbound.Init()) data := strings.Repeat("zk_avg_latency 0\nzk_min_latency 0\nzk_mix_latency 0\n", 10) unbound.client = mockUnboundClient{data: []byte(data), err: false} From 0ba23eaca3003314f84870b9241372f4f50b0ba0 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 11:38:24 +0300 Subject: [PATCH 26/39] use_tls fix --- modules/unbound/unbound.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 083c0c139..b1f791736 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -79,8 +79,8 @@ func (u *Unbound) Init() bool { u.charts = charts(u.Cumulative) - u.Debugf("using address: %s, cumulative: %v, disable_tls: %v, timeout: %s", u.Address, u.Cumulative, u.UseTLS, u.Timeout) - if !u.UseTLS { + u.Debugf("using address: %s, cumulative: %v, use_tls: %v, timeout: %s", u.Address, u.Cumulative, u.UseTLS, u.Timeout) + if u.UseTLS { u.Debugf("using tls_skip_verify: %v, tls_key: %s, tls_cert: %s", u.InsecureSkipVerify, u.TLSKey, u.TLSCert) } return true From 6a3f0538809d1c2a0fd1377599a82a33354f70ef Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 12:00:46 +0300 Subject: [PATCH 27/39] fix unbound config --- config/go.d/unbound.conf | 47 +++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/config/go.d/unbound.conf b/config/go.d/unbound.conf index 5511f462a..9581b50c1 100644 --- a/config/go.d/unbound.conf +++ b/config/go.d/unbound.conf @@ -59,28 +59,37 @@ # - address # Server address. # Syntax: -# address: 127.0.0.1:2181 +# address: 127.0.0.1:8953 +# address: /var/run/unbound.sock # # - timeout # Connection/read/write/ssl handshake timeout. # Syntax: # timeout: 1 # +# - conf_path +# Absolute path to the unbound configuration file. Module uses the file to autodetect 'address', 'cumulative', +# 'use_tls', 'tls_cert' and 'tls_key' parameters. +# To disable parameters auto detection set it to empty string (""). +# Syntax: +# conf_path: /path/to/unbound.conf +# +# - cumulative +# Statistics collection mode. Should have the same value as the `statistics-cumulative` parameter in the unbound +# configuration file. +# Syntax: +# cumulative: yes/no +# # - use_tls # Whether to use or not TLS. # Syntax: -# use_tls: true +# use_tls: yes/no # # - tls_skip_verify # Whether to skip verifying server's certificate chain and hostname. # Syntax: # tls_skip_verify: yes/no # -# - tls_ca -# Certificate authority that client use when verifying server certificates. -# Syntax: -# tls_ca: path/to/ca.pem -# # - tls_cert # Client tls certificate. # Syntax: @@ -93,10 +102,14 @@ # # # [ JOB defaults ]: -# address: 127.0.0.1:2181 +# address: 127.0.0.1:8953 # timeout: 1 -# use_tls: false -# tls_skip_verify: no +# conf_path: /etc/unbound/unbound.conf +# cumulative: no +# use_tls: yes +# tls_skip_verify: yes +# tls_cert: /etc/unbound/unbound_control.pem +# tls_key: /etc/unbound/unbound_control.key # # # [ JOB mandatory parameters ]: @@ -112,8 +125,12 @@ # # [ JOBS ] jobs: - - name: local - address: 127.0.0.1:2181 - - - name: local - address: 127.0.0.1:2182 + - name : local + address : 127.0.0.1:8953 + timeout : 1 + conf_path : /etc/unbound/unbound.conf + cumulative : no + use_tls : yes + tls_skip_verify : yes + tls_cert : /etc/unbound/unbound_control.pem + tls_key : /etc/unbound/unbound_control.key From e852190a0bcd110df8f2ea24b221cd375398967e Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 12:03:08 +0300 Subject: [PATCH 28/39] fix unbound config --- config/go.d/unbound.conf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/config/go.d/unbound.conf b/config/go.d/unbound.conf index 9581b50c1..ddf435d01 100644 --- a/config/go.d/unbound.conf +++ b/config/go.d/unbound.conf @@ -125,12 +125,12 @@ # # [ JOBS ] jobs: - - name : local - address : 127.0.0.1:8953 - timeout : 1 - conf_path : /etc/unbound/unbound.conf - cumulative : no - use_tls : yes - tls_skip_verify : yes - tls_cert : /etc/unbound/unbound_control.pem - tls_key : /etc/unbound/unbound_control.key + - name: local + address: 127.0.0.1:8953 + timeout: 1 + conf_path: /etc/unbound/unbound.conf + cumulative: no + use_tls: yes + tls_skip_verify: yes + tls_cert: /etc/unbound/unbound_control.pem + tls_key: /etc/unbound/unbound_control.key From 359066a9a87fd7c2a318ffb1f8f57938b488c370 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 13:00:19 +0300 Subject: [PATCH 29/39] readme wip --- modules/unbound/README.md | 70 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/modules/unbound/README.md b/modules/unbound/README.md index 2fff69107..f8b5f9dda 100644 --- a/modules/unbound/README.md +++ b/modules/unbound/README.md @@ -1 +1,69 @@ -#unbound \ No newline at end of file +# unbound + +This module monitors one or more [`Unbound`](https://nlnetlabs.nl/projects/unbound/about/) servers, depending on your configuration. + +#### Requirements + +- `Unbound` with enabled `remote-control` interface (see [unbound.conf](https://nlnetlabs.nl/documentation/unbound/unbound.conf)) + +If using unix socket: + +- socket should be readable and writeable by `netdata` user + +If using ip socket and TLS is disabled: + +- socket should be accessible via network + +If TLS is enabled, in addition: + +- `control-key-file` should be readable by `netdata` user +- `control-cert-file` should be readable by `netdata` user + +For auto detection parameters from `unbound.conf`: + +- `unbound.conf` should be readable by `netdata` user + + +### Configuration + +Needs only `address` to server's remote-control interface if TLS is disabled or `address` is unix socket. +Otherwise you need to set path to the `control-key-file` and `control-cert-file` files. + +Module tries to auto detect following parameter reading `unbound.conf`: + - address (`control-interface` and `control-port`) + - cumulative (`statistics-cumulative`) + - use_tls (`control-use-cert`) + - tls_cert (`control-cert-file`) + - tls_key (`control-key-file`) + +Module supports both cumulative and non cumulative modes. Default is non cumulative. If your server has enabled +`statistics-cumulative` but module fails to auto detect it (`unbound.conf` is not readable or it is a remote server) +you need to set it manually in the configuration file. + +Here is an example for several servers: + +```yaml +jobs: + - name: local + address: 127.0.0.1:8953 + use_tls: yes + tls_skip_verify: yes + tls_cert: /etc/unbound/unbound_control.pem + tls_key: /etc/unbound/unbound_control.key + + - name: remote + address: 203.0.113.10:8953 + use_tls: no + + - name: remote_cumulative + address: 203.0.113.10:8953 + use_tls: no + cumulative: yes + + - name: socket + address: /var/run/unbound.sock +``` + +For all available options, please see the module [configuration file](https://github.com/netdata/go.d.plugin/blob/master/config/go.d/unbound.conf). + +--- From ae4c804e7c3b5190c59a911410734436592c2d01 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 13:21:33 +0300 Subject: [PATCH 30/39] readme wip --- modules/unbound/README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/modules/unbound/README.md b/modules/unbound/README.md index f8b5f9dda..f3e6749e7 100644 --- a/modules/unbound/README.md +++ b/modules/unbound/README.md @@ -26,10 +26,10 @@ For auto detection parameters from `unbound.conf`: ### Configuration -Needs only `address` to server's remote-control interface if TLS is disabled or `address` is unix socket. +Needs only `address` to server's `remote-control` interface if TLS is disabled or `address` is unix socket. Otherwise you need to set path to the `control-key-file` and `control-cert-file` files. -Module tries to auto detect following parameter reading `unbound.conf`: +Module tries to auto detect following parameters reading `unbound.conf`: - address (`control-interface` and `control-port`) - cumulative (`statistics-cumulative`) - use_tls (`control-use-cert`) @@ -56,7 +56,7 @@ jobs: use_tls: no - name: remote_cumulative - address: 203.0.113.10:8953 + address: 203.0.113.11:8953 use_tls: no cumulative: yes @@ -66,4 +66,18 @@ jobs: For all available options, please see the module [configuration file](https://github.com/netdata/go.d.plugin/blob/master/config/go.d/unbound.conf). + +### Troubleshooting +Ensure that the control protocol is actually configured correctly. +Run following command as `root` user: +> unbound-control stats_noreset + +It should print out a bunch of info about the internal statistics of the server. +If this returns an error, you don't have the control protocol set up correctly. + +Check the module debug output. +Run following command as `netdata` user: + +> ./go.d.plugin -d -m unbound + --- From 51b793ecad7c4e33b9d04da4a98a5f1856426e95 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 13:23:03 +0300 Subject: [PATCH 31/39] readme wip --- modules/unbound/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/unbound/README.md b/modules/unbound/README.md index f3e6749e7..fed17112a 100644 --- a/modules/unbound/README.md +++ b/modules/unbound/README.md @@ -30,11 +30,11 @@ Needs only `address` to server's `remote-control` interface if TLS is disabled o Otherwise you need to set path to the `control-key-file` and `control-cert-file` files. Module tries to auto detect following parameters reading `unbound.conf`: - - address (`control-interface` and `control-port`) - - cumulative (`statistics-cumulative`) - - use_tls (`control-use-cert`) - - tls_cert (`control-cert-file`) - - tls_key (`control-key-file`) + - address + - cumulative + - use_tls + - tls_cert + - tls_key Module supports both cumulative and non cumulative modes. Default is non cumulative. If your server has enabled `statistics-cumulative` but module fails to auto detect it (`unbound.conf` is not readable or it is a remote server) From 12d431815958fecacd52ed75b1412e0fa69ee64f Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 15:57:45 +0300 Subject: [PATCH 32/39] tune charts --- modules/unbound/charts.go | 139 +++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 69 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index 8ece510e9..cb189b1d7 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -1,6 +1,7 @@ package unbound import ( + "fmt" "strings" "github.com/netdata/go-orchestrator" @@ -23,7 +24,7 @@ var threadPriority = orchestrator.DefaultJobPriority + len(*charts(true)) + len( func charts(cumulative bool) *Charts { return &Charts{ makeIncrIf(queriesChart.Copy(), cumulative), - makeIncrIf(queriesIPRLChart.Copy(), cumulative), + makeIncrIf(ipRateLimitedQueriesChart.Copy(), cumulative), makeIncrIf(cacheChart.Copy(), cumulative), makePercOfIncrIf(cachePercentageChart.Copy(), cumulative), makeIncrIf(prefetchChart.Copy(), cumulative), @@ -64,10 +65,10 @@ func threadCharts(thread string, cumulative bool) *Charts { } func convertTotalChartToThread(chart *Chart, thread string, priority int) { - chart.ID = strings.Replace(chart.ID, "total", thread, 1) - chart.Title = strings.Replace(chart.Title, "Total", strings.Title(thread), 1) + chart.ID = fmt.Sprintf("%s_%s", thread, chart.ID) + chart.Title = fmt.Sprintf("%s %s", strings.Title(thread), thread) chart.Fam = thread + "_stats" - chart.Ctx = strings.Replace(chart.Ctx, "total", thread, 1) + chart.Ctx = fmt.Sprintf("%s_%s", chart.Ctx, thread) chart.Priority = priority for _, dim := range chart.Dims { dim.ID = strings.Replace(dim.ID, "total", thread, 1) @@ -77,31 +78,31 @@ func convertTotalChartToThread(chart *Chart, thread string, priority int) { // NOTE: chart id should start with 'total_', name with 'Total ', ctx should ends in `_total` (convertTotalChartToThread) var ( queriesChart = Chart{ - ID: "total_queries", - Title: "Total Queries", + ID: "queries", + Title: "Received Queries", Units: "queries", Fam: "queries", - Ctx: "unbound.queries_total", + Ctx: "unbound.queries", Dims: Dims{ {ID: "total.num.queries", Name: "queries"}, }, } - queriesIPRLChart = Chart{ - ID: "total_queries_ip_ratelimited", - Title: "Total Queries IP Rate Limited", + ipRateLimitedQueriesChart = Chart{ + ID: "queries_ip_ratelimited", + Title: "Rate Limited Queries", Units: "queries", Fam: "queries", - Ctx: "unbound.queries_ip_ratelimited_total", + Ctx: "unbound.queries_ip_ratelimited", Dims: Dims{ - {ID: "total.num.queries_ip_ratelimited", Name: "queries"}, + {ID: "total.num.queries_ip_ratelimited", Name: "ratelimited"}, }, } cacheChart = Chart{ - ID: "total_cache", - Title: "Total Cache", + ID: "cache", + Title: "Cache Statistics", Units: "events", Fam: "cache", - Ctx: "unbound.cache_total", + Ctx: "unbound.cache", Type: module.Stacked, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits"}, @@ -109,11 +110,11 @@ var ( }, } cachePercentageChart = Chart{ - ID: "total_cache_percentage", - Title: "Total Cache Percentage", + ID: "cache_percentage", + Title: "Cache Statistics Percentage", Units: "percentage", Fam: "cache", - Ctx: "unbound.cache_percantage_total", + Ctx: "unbound.cache_percentage", Type: module.Stacked, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits", Algo: module.PercentOfAbsolute}, @@ -121,32 +122,32 @@ var ( }, } prefetchChart = Chart{ - ID: "total_queries_prefetch", - Title: "Total Cache Prefetches", - Units: "queries", + ID: "cache_prefetch", + Title: "Cache Prefetches", + Units: "prefetches", Fam: "cache", - Ctx: "unbound.queries_prefetch_total", + Ctx: "unbound.prefetch", Dims: Dims{ - {ID: "total.num.prefetch", Name: "queries"}, + {ID: "total.num.prefetch", Name: "prefetches"}, }, } zeroTTLChart = Chart{ - ID: "total_zero_ttl_responses", - Title: "Total Answers Served From Expired Cache", - Units: "responses", + ID: "zero_ttl_replies", + Title: "Replies Served From Expired Cache", + Units: "replies", Fam: "cache", - Ctx: "unbound.zero_ttl_responses_total", + Ctx: "unbound.zero_ttl_replies", Dims: Dims{ - {ID: "total.num.zero_ttl", Name: "responses"}, + {ID: "total.num.zero_ttl", Name: "zero_ttl"}, }, } // ifdef USE_DNSCRYPT dnsCryptChart = Chart{ - ID: "total_dnscrypt_queries", - Title: "Total DNSCrypt Queries", + ID: "dnscrypt_queries", + Title: "DNSCrypt Queries", Units: "queries", - Fam: "dnscrypt", - Ctx: "unbound.dnscrypt_queries_total", + Fam: "dnscrypt queries", + Ctx: "unbound.dnscrypt_queries", Dims: Dims{ {ID: "total.num.dnscrypt.crypted", Name: "crypted"}, {ID: "total.num.dnscrypt.cert", Name: "cert"}, @@ -155,20 +156,20 @@ var ( }, } recurRepliesChart = Chart{ - ID: "total_recursive_replies", - Title: "Total number of replies sent to queries that needed recursive processing", - Units: "responses", - Fam: "responses", - Ctx: "unbound.recursive_replies_total", + ID: "recursive_replies", + Title: "Replies That Needed Recursive Processing", + Units: "replies", + Fam: "recursion", + Ctx: "unbound.recursive_replies", Dims: Dims{ {ID: "total.num.recursivereplies", Name: "recursive"}, }, } recurTimeChart = Chart{ - ID: "total_recursion_time", - Title: "Total Time t took to answer queries that needed recursive processing", + ID: "recursion_time", + Title: "Time Spent On Recursive Processing", Units: "milliseconds", - Fam: "responses", + Fam: "recursion", Ctx: "unbound.recursion_time_total", Dims: Dims{ {ID: "total.recursion.time.avg", Name: "avg"}, @@ -176,22 +177,22 @@ var ( }, } reqListUtilChart = Chart{ - ID: "total_request_list_utilization", - Title: "Total Request List Utilization", + ID: "request_list_utilization", + Title: "Request List Utilization", Units: "queries", Fam: "request list", - Ctx: "unbound.request_list_utilization_total", + Ctx: "unbound.request_list_utilization", Dims: Dims{ {ID: "total.requestlist.avg", Name: "avg", Div: 1000}, {ID: "total.requestlist.max", Name: "max"}, // all time max in cumulative mode, never resets }, } reqListCurUtilChart = Chart{ - ID: "total_current_request_list_utilization", - Title: "Total Current Request List Utilization", + ID: "current_request_list_utilization", + Title: "Current Request List Utilization", Units: "queries", Fam: "request list", - Ctx: "unbound.current_request_list_utilization_total", + Ctx: "unbound.current_request_list_utilization", Type: module.Area, Dims: Dims{ {ID: "total.requestlist.current.all", Name: "all"}, @@ -199,24 +200,24 @@ var ( }, } reqListJostleChart = Chart{ - ID: "total_request_list_jostle_list", - Title: "Total Request List Jostle List Events", - Units: "events", + ID: "request_list_jostle_list", + Title: "Request List Jostle List Events", + Units: "queries", Fam: "request list", - Ctx: "unbound.request_list_jostle_list_total", + Ctx: "unbound.request_list_jostle_list", Dims: Dims{ {ID: "total.requestlist.overwritten", Name: "overwritten"}, {ID: "total.requestlist.exceeded", Name: "dropped"}, }, } tcpUsageChart = Chart{ - ID: "total_tcpusage", - Title: "Total TCP Accept List Usage", - Units: "events", - Fam: "tcpusage", - Ctx: "unbound.tcpusage_total", + ID: "tcpusage", + Title: "TCP Handler Buffers", + Units: "buffers", + Fam: "tcp buffers", + Ctx: "unbound.tcpusage", Dims: Dims{ - {ID: "total.tcpusage", Name: "tcpusage"}, + {ID: "total.tcpusage", Name: "usage"}, }, } uptimeChart = Chart{ @@ -276,7 +277,7 @@ var ( // NOTE: same family as for cacheChart cacheCountChart = Chart{ ID: "cache_count", - Title: "Cache Count", + Title: "Cache Items Count", Units: "items", Fam: "cache", Ctx: "unbound.cache_count", @@ -291,34 +292,34 @@ var ( }, } queryTypeChart = Chart{ - ID: "query_type", + ID: "queries_by_type", Title: "Queries By Type", Units: "queries", - Fam: "query type", + Fam: "queries by type", Ctx: "unbound.type_queries", Type: module.Stacked, } queryClassChart = Chart{ - ID: "query_class", + ID: "queries_by_class", Title: "Queries By Class", Units: "queries", - Fam: "query class", + Fam: "queries by class", Ctx: "unbound.class_queries", Type: module.Stacked, } queryOpCodeChart = Chart{ - ID: "query_opcode", + ID: "queries_by_opcode", Title: "Queries By OpCode", Units: "queries", - Fam: "query opcode", + Fam: "queries by opcode", Ctx: "unbound.opcode_queries", Type: module.Stacked, } queryFlagChart = Chart{ - ID: "query_flag", + ID: "queries_by_flag", Title: "Queries By Flag", Units: "queries", - Fam: "query flag", + Fam: "queries by flag", Ctx: "unbound.flag_queries", Type: module.Stacked, Dims: Dims{ @@ -333,10 +334,10 @@ var ( }, } answerRCodeChart = Chart{ - ID: "answer_rcode", - Title: "Answers By Rcode", - Units: "answers", - Fam: "answer rcode", + ID: "replies_by_rcode", + Title: "Replies By Rcode", + Units: "replies", + Fam: "replies by rcode", Ctx: "unbound.rcode_answers", Type: module.Stacked, } From 2545e7698f845f28cb1f7898a5aef25647c0e86d Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 17:23:09 +0300 Subject: [PATCH 33/39] tune charts --- modules/unbound/charts.go | 314 ++++++++++++++++++++++--------------- modules/unbound/collect.go | 2 +- 2 files changed, 186 insertions(+), 130 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index cb189b1d7..42c6e4dcb 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -19,7 +19,39 @@ type ( Dim = module.Dim ) -var threadPriority = orchestrator.DefaultJobPriority + len(*charts(true)) + len(*extendedCharts(true)) + 10 +const ( + queriesPrio = orchestrator.DefaultJobPriority + iota + ipRateLimitedQueriesPrio + queryTypePrio + queryClassPrio + queryOpCodePrio + queryFlagPrio + dnsCryptPrio + + recurRepliesPrio + replyRCodePrio + + recurTimePrio + + cachePrio + cachePercentagePrio + prefetchPrio + zeroTTLPrio + cacheCountPrio + + reqListUtilPrio + reqListCurUtilPrio + reqListJostlePrio + + tcpUsagePrio + + memCachePrio + memModPrio + memStreamWaitPrio + uptimePrio + + threadPriority +) func charts(cumulative bool) *Charts { return &Charts{ @@ -66,7 +98,7 @@ func threadCharts(thread string, cumulative bool) *Charts { func convertTotalChartToThread(chart *Chart, thread string, priority int) { chart.ID = fmt.Sprintf("%s_%s", thread, chart.ID) - chart.Title = fmt.Sprintf("%s %s", strings.Title(thread), thread) + chart.Title = fmt.Sprintf("%s %s", strings.Title(thread), chart.Title) chart.Fam = thread + "_stats" chart.Ctx = fmt.Sprintf("%s_%s", chart.Ctx, thread) chart.Priority = priority @@ -75,79 +107,86 @@ func convertTotalChartToThread(chart *Chart, thread string, priority int) { } } -// NOTE: chart id should start with 'total_', name with 'Total ', ctx should ends in `_total` (convertTotalChartToThread) +// Common stats charts var ( queriesChart = Chart{ - ID: "queries", - Title: "Received Queries", - Units: "queries", - Fam: "queries", - Ctx: "unbound.queries", + ID: "queries", + Title: "Received Queries", + Units: "queries", + Fam: "queries", + Ctx: "unbound.queries", + Priority: queriesPrio, Dims: Dims{ {ID: "total.num.queries", Name: "queries"}, }, } ipRateLimitedQueriesChart = Chart{ - ID: "queries_ip_ratelimited", - Title: "Rate Limited Queries", - Units: "queries", - Fam: "queries", - Ctx: "unbound.queries_ip_ratelimited", + ID: "queries_ip_ratelimited", + Title: "Rate Limited Queries", + Units: "queries", + Fam: "queries", + Ctx: "unbound.queries_ip_ratelimited", + Priority: ipRateLimitedQueriesPrio, Dims: Dims{ {ID: "total.num.queries_ip_ratelimited", Name: "ratelimited"}, }, } cacheChart = Chart{ - ID: "cache", - Title: "Cache Statistics", - Units: "events", - Fam: "cache", - Ctx: "unbound.cache", - Type: module.Stacked, + ID: "cache", + Title: "Cache Statistics", + Units: "events", + Fam: "cache", + Ctx: "unbound.cache", + Type: module.Stacked, + Priority: cachePrio, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits"}, {ID: "total.num.cachemiss", Name: "miss"}, }, } cachePercentageChart = Chart{ - ID: "cache_percentage", - Title: "Cache Statistics Percentage", - Units: "percentage", - Fam: "cache", - Ctx: "unbound.cache_percentage", - Type: module.Stacked, + ID: "cache_percentage", + Title: "Cache Statistics Percentage", + Units: "percentage", + Fam: "cache", + Ctx: "unbound.cache_percentage", + Type: module.Stacked, + Priority: cachePercentagePrio, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits", Algo: module.PercentOfAbsolute}, {ID: "total.num.cachemiss", Name: "miss", Algo: module.PercentOfAbsolute}, }, } prefetchChart = Chart{ - ID: "cache_prefetch", - Title: "Cache Prefetches", - Units: "prefetches", - Fam: "cache", - Ctx: "unbound.prefetch", + ID: "cache_prefetch", + Title: "Cache Prefetches", + Units: "prefetches", + Fam: "cache", + Ctx: "unbound.prefetch", + Priority: prefetchPrio, Dims: Dims{ {ID: "total.num.prefetch", Name: "prefetches"}, }, } zeroTTLChart = Chart{ - ID: "zero_ttl_replies", - Title: "Replies Served From Expired Cache", - Units: "replies", - Fam: "cache", - Ctx: "unbound.zero_ttl_replies", + ID: "zero_ttl_replies", + Title: "Replies Served From Expired Cache", + Units: "replies", + Fam: "cache", + Ctx: "unbound.zero_ttl_replies", + Priority: zeroTTLPrio, Dims: Dims{ {ID: "total.num.zero_ttl", Name: "zero_ttl"}, }, } // ifdef USE_DNSCRYPT dnsCryptChart = Chart{ - ID: "dnscrypt_queries", - Title: "DNSCrypt Queries", - Units: "queries", - Fam: "dnscrypt queries", - Ctx: "unbound.dnscrypt_queries", + ID: "dnscrypt_queries", + Title: "DNSCrypt Queries", + Units: "queries", + Fam: "queries", + Ctx: "unbound.dnscrypt_queries", + Priority: dnsCryptPrio, Dims: Dims{ {ID: "total.num.dnscrypt.crypted", Name: "crypted"}, {ID: "total.num.dnscrypt.cert", Name: "cert"}, @@ -156,91 +195,100 @@ var ( }, } recurRepliesChart = Chart{ - ID: "recursive_replies", - Title: "Replies That Needed Recursive Processing", - Units: "replies", - Fam: "recursion", - Ctx: "unbound.recursive_replies", + ID: "recursive_replies", + Title: "Replies That Needed Recursive Processing", + Units: "replies", + Fam: "replies", + Ctx: "unbound.recursive_replies", + Priority: recurRepliesPrio, Dims: Dims{ {ID: "total.num.recursivereplies", Name: "recursive"}, }, } recurTimeChart = Chart{ - ID: "recursion_time", - Title: "Time Spent On Recursive Processing", - Units: "milliseconds", - Fam: "recursion", - Ctx: "unbound.recursion_time_total", + ID: "recursion_time", + Title: "Time Spent On Recursive Processing", + Units: "milliseconds", + Fam: "recursion timings", + Ctx: "unbound.recursion_time", + Priority: recurTimePrio, Dims: Dims{ {ID: "total.recursion.time.avg", Name: "avg"}, {ID: "total.recursion.time.median", Name: "median"}, }, } reqListUtilChart = Chart{ - ID: "request_list_utilization", - Title: "Request List Utilization", - Units: "queries", - Fam: "request list", - Ctx: "unbound.request_list_utilization", + ID: "request_list_utilization", + Title: "Request List Utilization", + Units: "queries", + Fam: "request list", + Ctx: "unbound.request_list_utilization", + Priority: reqListUtilPrio, Dims: Dims{ {ID: "total.requestlist.avg", Name: "avg", Div: 1000}, {ID: "total.requestlist.max", Name: "max"}, // all time max in cumulative mode, never resets }, } reqListCurUtilChart = Chart{ - ID: "current_request_list_utilization", - Title: "Current Request List Utilization", - Units: "queries", - Fam: "request list", - Ctx: "unbound.current_request_list_utilization", - Type: module.Area, + ID: "current_request_list_utilization", + Title: "Current Request List Utilization", + Units: "queries", + Fam: "request list", + Ctx: "unbound.current_request_list_utilization", + Type: module.Area, + Priority: reqListCurUtilPrio, Dims: Dims{ {ID: "total.requestlist.current.all", Name: "all"}, {ID: "total.requestlist.current.user", Name: "user"}, }, } reqListJostleChart = Chart{ - ID: "request_list_jostle_list", - Title: "Request List Jostle List Events", - Units: "queries", - Fam: "request list", - Ctx: "unbound.request_list_jostle_list", + ID: "request_list_jostle_list", + Title: "Request List Jostle List Events", + Units: "queries", + Fam: "request list", + Ctx: "unbound.request_list_jostle_list", + Priority: reqListJostlePrio, Dims: Dims{ {ID: "total.requestlist.overwritten", Name: "overwritten"}, {ID: "total.requestlist.exceeded", Name: "dropped"}, }, } tcpUsageChart = Chart{ - ID: "tcpusage", - Title: "TCP Handler Buffers", - Units: "buffers", - Fam: "tcp buffers", - Ctx: "unbound.tcpusage", + ID: "tcpusage", + Title: "TCP Handler Buffers", + Units: "buffers", + Fam: "tcp buffers", + Ctx: "unbound.tcpusage", + Priority: tcpUsagePrio, Dims: Dims{ {ID: "total.tcpusage", Name: "usage"}, }, } uptimeChart = Chart{ - ID: "uptime", - Title: "Uptime", - Units: "seconds", - Fam: "uptime", - Ctx: "unbound.uptime", + ID: "uptime", + Title: "Uptime", + Units: "seconds", + Fam: "uptime", + Ctx: "unbound.uptime", + Priority: uptimePrio, Dims: Dims{ {ID: "time.up", Name: "time"}, }, } ) +// Extended stats charts var ( // TODO: do not add dnscrypt stuff by default? memCacheChart = Chart{ - ID: "cache_memory", - Title: "Cache Memory", - Units: "KB", - Fam: "mem", - Ctx: "unbound.cache_memory", - Type: module.Stacked, + ID: "cache_memory", + Title: "Cache Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.cache_memory", + Type: module.Stacked, + Priority: memCachePrio, Dims: Dims{ {ID: "mem.cache.message", Name: "message", Div: 1024}, {ID: "mem.cache.rrset", Name: "rrset", Div: 1024}, @@ -250,12 +298,13 @@ var ( } // TODO: do not add subnet and ipsecmod stuff by default? memModChart = Chart{ - ID: "mod_memory", - Title: "Module Memory", - Units: "KB", - Fam: "mem", - Ctx: "unbound.mod_memory", - Type: module.Stacked, + ID: "mod_memory", + Title: "Module Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.mod_memory", + Type: module.Stacked, + Priority: memModPrio, Dims: Dims{ {ID: "mem.mod.iterator", Name: "iterator", Div: 1024}, {ID: "mem.mod.respip", Name: "respip", Div: 1024}, @@ -265,23 +314,25 @@ var ( }, } memStreamWaitChart = Chart{ - ID: "mem_stream_wait", - Title: "TCP and TLS Stream Waif Buffer Memory", - Units: "KB", - Fam: "mem", - Ctx: "unbound.mem_streamwait", + ID: "mem_stream_wait", + Title: "TCP and TLS Stream Waif Buffer Memory", + Units: "KB", + Fam: "mem", + Ctx: "unbound.mem_streamwait", + Priority: memStreamWaitPrio, Dims: Dims{ {ID: "mem.streamwait", Name: "streamwait", Div: 1024}, }, } // NOTE: same family as for cacheChart cacheCountChart = Chart{ - ID: "cache_count", - Title: "Cache Items Count", - Units: "items", - Fam: "cache", - Ctx: "unbound.cache_count", - Type: module.Stacked, + ID: "cache_count", + Title: "Cache Items Count", + Units: "items", + Fam: "cache", + Ctx: "unbound.cache_count", + Type: module.Stacked, + Priority: cacheCountPrio, Dims: Dims{ {ID: "infra.cache.count", Name: "infra"}, {ID: "key.cache.count", Name: "key"}, @@ -292,36 +343,40 @@ var ( }, } queryTypeChart = Chart{ - ID: "queries_by_type", - Title: "Queries By Type", - Units: "queries", - Fam: "queries by type", - Ctx: "unbound.type_queries", - Type: module.Stacked, + ID: "queries_by_type", + Title: "Queries By Type", + Units: "queries", + Fam: "queries", + Ctx: "unbound.type_queries", + Type: module.Stacked, + Priority: queryTypePrio, } queryClassChart = Chart{ - ID: "queries_by_class", - Title: "Queries By Class", - Units: "queries", - Fam: "queries by class", - Ctx: "unbound.class_queries", - Type: module.Stacked, + ID: "queries_by_class", + Title: "Queries By Class", + Units: "queries", + Fam: "queries", + Ctx: "unbound.class_queries", + Type: module.Stacked, + Priority: queryClassPrio, } queryOpCodeChart = Chart{ - ID: "queries_by_opcode", - Title: "Queries By OpCode", - Units: "queries", - Fam: "queries by opcode", - Ctx: "unbound.opcode_queries", - Type: module.Stacked, + ID: "queries_by_opcode", + Title: "Queries By OpCode", + Units: "queries", + Fam: "queries", + Ctx: "unbound.opcode_queries", + Type: module.Stacked, + Priority: queryOpCodePrio, } queryFlagChart = Chart{ - ID: "queries_by_flag", - Title: "Queries By Flag", - Units: "queries", - Fam: "queries by flag", - Ctx: "unbound.flag_queries", - Type: module.Stacked, + ID: "queries_by_flag", + Title: "Queries By Flag", + Units: "queries", + Fam: "queries", + Ctx: "unbound.flag_queries", + Type: module.Stacked, + Priority: queryFlagPrio, Dims: Dims{ {ID: "num.query.flags.QR", Name: "QR"}, {ID: "num.query.flags.AA", Name: "AA"}, @@ -334,12 +389,13 @@ var ( }, } answerRCodeChart = Chart{ - ID: "replies_by_rcode", - Title: "Replies By Rcode", - Units: "replies", - Fam: "replies by rcode", - Ctx: "unbound.rcode_answers", - Type: module.Stacked, + ID: "replies_by_rcode", + Title: "Replies By Rcode", + Units: "replies", + Fam: "replies", + Ctx: "unbound.rcode_answers", + Type: module.Stacked, + Priority: replyRCodePrio, } ) diff --git a/modules/unbound/collect.go b/modules/unbound/collect.go index 36d047110..2094fb730 100644 --- a/modules/unbound/collect.go +++ b/modules/unbound/collect.go @@ -50,7 +50,7 @@ func (u *Unbound) collectStats(stats []entry) map[string]int64 { func (u *Unbound) collectCumulativeStats(stats []entry) map[string]int64 { mul := float64(1000) - // following stats reset only on cachemiss event in cumulative mode + // following stats change only on cachemiss event in cumulative mode // - *.requestlist.avg, // - *.recursion.time.avg // - *.recursion.time.median From c841fc932fd72b35d59b425affd17f2a55bbb02c Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 17:36:01 +0300 Subject: [PATCH 34/39] tune charts --- modules/unbound/charts.go | 60 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index 42c6e4dcb..4be6ceb1f 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -39,8 +39,8 @@ const ( zeroTTLPrio cacheCountPrio - reqListUtilPrio - reqListCurUtilPrio + reqListUsagePrio + reqListCurUsagePrio reqListJostlePrio tcpUsagePrio @@ -64,8 +64,8 @@ func charts(cumulative bool) *Charts { makeIncrIf(dnsCryptChart.Copy(), cumulative), makeIncrIf(recurRepliesChart.Copy(), cumulative), recurTimeChart.Copy(), - reqListUtilChart.Copy(), - reqListCurUtilChart.Copy(), + reqListUsageChart.Copy(), + reqListCurUsageChart.Copy(), makeIncrIf(reqListJostleChart.Copy(), cumulative), tcpUsageChart.Copy(), uptimeChart.Copy(), @@ -131,6 +131,21 @@ var ( {ID: "total.num.queries_ip_ratelimited", Name: "ratelimited"}, }, } + // ifdef USE_DNSCRYPT + dnsCryptChart = Chart{ + ID: "dnscrypt_queries", + Title: "DNSCrypt Queries", + Units: "queries", + Fam: "queries", + Ctx: "unbound.dnscrypt_queries", + Priority: dnsCryptPrio, + Dims: Dims{ + {ID: "total.num.dnscrypt.crypted", Name: "crypted"}, + {ID: "total.num.dnscrypt.cert", Name: "cert"}, + {ID: "total.num.dnscrypt.cleartext", Name: "cleartext"}, + {ID: "total.num.dnscrypt.malformed", Name: "malformed"}, + }, + } cacheChart = Chart{ ID: "cache", Title: "Cache Statistics", @@ -179,21 +194,6 @@ var ( {ID: "total.num.zero_ttl", Name: "zero_ttl"}, }, } - // ifdef USE_DNSCRYPT - dnsCryptChart = Chart{ - ID: "dnscrypt_queries", - Title: "DNSCrypt Queries", - Units: "queries", - Fam: "queries", - Ctx: "unbound.dnscrypt_queries", - Priority: dnsCryptPrio, - Dims: Dims{ - {ID: "total.num.dnscrypt.crypted", Name: "crypted"}, - {ID: "total.num.dnscrypt.cert", Name: "cert"}, - {ID: "total.num.dnscrypt.cleartext", Name: "cleartext"}, - {ID: "total.num.dnscrypt.malformed", Name: "malformed"}, - }, - } recurRepliesChart = Chart{ ID: "recursive_replies", Title: "Replies That Needed Recursive Processing", @@ -217,26 +217,26 @@ var ( {ID: "total.recursion.time.median", Name: "median"}, }, } - reqListUtilChart = Chart{ - ID: "request_list_utilization", - Title: "Request List Utilization", + reqListUsageChart = Chart{ + ID: "request_list_usage", + Title: "Request List Usage", Units: "queries", Fam: "request list", - Ctx: "unbound.request_list_utilization", - Priority: reqListUtilPrio, + Ctx: "unbound.request_list_usage", + Priority: reqListUsagePrio, Dims: Dims{ {ID: "total.requestlist.avg", Name: "avg", Div: 1000}, {ID: "total.requestlist.max", Name: "max"}, // all time max in cumulative mode, never resets }, } - reqListCurUtilChart = Chart{ - ID: "current_request_list_utilization", - Title: "Current Request List Utilization", + reqListCurUsageChart = Chart{ + ID: "current_request_list_usage", + Title: "Current Request List Usage", Units: "queries", Fam: "request list", - Ctx: "unbound.current_request_list_utilization", + Ctx: "unbound.current_request_list_usage", Type: module.Area, - Priority: reqListCurUtilPrio, + Priority: reqListCurUsagePrio, Dims: Dims{ {ID: "total.requestlist.current.all", Name: "all"}, {ID: "total.requestlist.current.user", Name: "user"}, @@ -390,7 +390,7 @@ var ( } answerRCodeChart = Chart{ ID: "replies_by_rcode", - Title: "Replies By Rcode", + Title: "Replies By RCode", Units: "replies", Fam: "replies", Ctx: "unbound.rcode_answers", From 1d68f8ef6b91a169956fd6c5e7df105a9f141668 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 17:36:10 +0300 Subject: [PATCH 35/39] readme update --- modules/unbound/README.md | 66 +++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/modules/unbound/README.md b/modules/unbound/README.md index fed17112a..c39131832 100644 --- a/modules/unbound/README.md +++ b/modules/unbound/README.md @@ -8,20 +8,66 @@ This module monitors one or more [`Unbound`](https://nlnetlabs.nl/projects/unbou If using unix socket: -- socket should be readable and writeable by `netdata` user +- socket should be readable and writeable by `netdata` user If using ip socket and TLS is disabled: -- socket should be accessible via network +- socket should be accessible via network If TLS is enabled, in addition: -- `control-key-file` should be readable by `netdata` user -- `control-cert-file` should be readable by `netdata` user +- `control-key-file` should be readable by `netdata` user +- `control-cert-file` should be readable by `netdata` user For auto detection parameters from `unbound.conf`: -- `unbound.conf` should be readable by `netdata` user +- `unbound.conf` should be readable by `netdata` user + +#### Charts + +Module produces following summary charts: + +- Received Queries in `queries` +- Rate Limited Queries in `queries` +- DNSCrypt Queries in `queries` +- Cache Statistics in `events` +- Cache Statistics Percentage in `events` +- Cache Prefetches in `prefetches` +- Replies Served From Expired Cache in `replies` +- Replies That Needed Recursive Processing in `replies` +- Time Spent On Recursive Processing in `milliseconds` +- Request List Usage in `queries` +- Current Request List Usage in `queries` +- Request List Jostle List Events in `queries` +- TCP Handler Buffers in `buffers` +- Uptime `seconds` + +If `extended-statistics` is enabled: + +- Queries By Type in `queries` +- Queries By Class in `queries` +- Queries By OpCode in `queries` +- Queries By Flag in `queries` +- Replies By RCode in `replies` +- Cache Items Count in `items` +- Cache Memory in `KB` +- Module Memory in `KB` +- TCP and TLS Stream Waif Buffer Memory in `KB` + +Per thread charts (only if number of threads > 1): +- Received Queries in `queries` +- Rate Limited Queries in `queries` +- DNSCrypt Queries in `queries` +- Cache Statistics in `events` +- Cache Statistics Percentage in `events` +- Cache Prefetches in `prefetches` +- Replies Served From Expired Cache in `replies` +- Replies That Needed Recursive Processing in `replies` +- Time Spent On Recursive Processing in `milliseconds` +- Request List Usage in `queries` +- Current Request List Usage in `queries` +- Request List Jostle List Events in `queries` +- TCP Handler Buffers in `buffers` ### Configuration @@ -30,11 +76,11 @@ Needs only `address` to server's `remote-control` interface if TLS is disabled o Otherwise you need to set path to the `control-key-file` and `control-cert-file` files. Module tries to auto detect following parameters reading `unbound.conf`: - - address - - cumulative - - use_tls - - tls_cert - - tls_key +- address +- cumulative +- use_tls +- tls_cert +- tls_key Module supports both cumulative and non cumulative modes. Default is non cumulative. If your server has enabled `statistics-cumulative` but module fails to auto detect it (`unbound.conf` is not readable or it is a remote server) From f3979ceb808d74888f031a3e7ccc29c0db9b5a37 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 17:39:44 +0300 Subject: [PATCH 36/39] disable by default for now --- modules/unbound/unbound.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index b1f791736..47c55b8e1 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -10,6 +10,10 @@ import ( func init() { creator := module.Creator{ + // TODO: python one is so broken, we need just to obsolete it and enable this one + Defaults: module.Defaults{ + Disabled: true, + }, Create: func() module.Module { return New() }, } From 97c09378b5948848203287fa7483d50a08d4eb52 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Tue, 19 Nov 2019 18:13:50 +0300 Subject: [PATCH 37/39] charts prio const names change --- modules/unbound/charts.go | 96 +++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/modules/unbound/charts.go b/modules/unbound/charts.go index 4be6ceb1f..fed28d530 100644 --- a/modules/unbound/charts.go +++ b/modules/unbound/charts.go @@ -20,37 +20,37 @@ type ( ) const ( - queriesPrio = orchestrator.DefaultJobPriority + iota - ipRateLimitedQueriesPrio - queryTypePrio - queryClassPrio - queryOpCodePrio - queryFlagPrio - dnsCryptPrio + prioQueries = orchestrator.DefaultJobPriority + iota + prioIPRateLimitedQueries + prioQueryType + prioQueryClass + prioQueryOpCode + prioQueryFlag + prioDNSCryptQueries - recurRepliesPrio - replyRCodePrio + prioRecurReplies + prioReplyRCode - recurTimePrio + prioRecurTime - cachePrio - cachePercentagePrio - prefetchPrio - zeroTTLPrio - cacheCountPrio + prioCache + prioCachePercentage + prioCachePrefetch + prioZeroTTL + prioCacheCount - reqListUsagePrio - reqListCurUsagePrio - reqListJostlePrio + prioReqListUsage + prioReqListCurUsage + prioReqListJostle - tcpUsagePrio + prioTCPUsage - memCachePrio - memModPrio - memStreamWaitPrio - uptimePrio + prioMemCache + prioMemMod + prioMemStreamWait + prioUptime - threadPriority + prioThread ) func charts(cumulative bool) *Charts { @@ -91,7 +91,7 @@ func threadCharts(thread string, cumulative bool) *Charts { _ = charts.Remove(uptimeChart.ID) for i, chart := range *charts { - convertTotalChartToThread(chart, thread, threadPriority+i) + convertTotalChartToThread(chart, thread, prioThread+i) } return charts } @@ -115,7 +115,7 @@ var ( Units: "queries", Fam: "queries", Ctx: "unbound.queries", - Priority: queriesPrio, + Priority: prioQueries, Dims: Dims{ {ID: "total.num.queries", Name: "queries"}, }, @@ -126,7 +126,7 @@ var ( Units: "queries", Fam: "queries", Ctx: "unbound.queries_ip_ratelimited", - Priority: ipRateLimitedQueriesPrio, + Priority: prioIPRateLimitedQueries, Dims: Dims{ {ID: "total.num.queries_ip_ratelimited", Name: "ratelimited"}, }, @@ -138,7 +138,7 @@ var ( Units: "queries", Fam: "queries", Ctx: "unbound.dnscrypt_queries", - Priority: dnsCryptPrio, + Priority: prioDNSCryptQueries, Dims: Dims{ {ID: "total.num.dnscrypt.crypted", Name: "crypted"}, {ID: "total.num.dnscrypt.cert", Name: "cert"}, @@ -153,7 +153,7 @@ var ( Fam: "cache", Ctx: "unbound.cache", Type: module.Stacked, - Priority: cachePrio, + Priority: prioCache, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits"}, {ID: "total.num.cachemiss", Name: "miss"}, @@ -166,7 +166,7 @@ var ( Fam: "cache", Ctx: "unbound.cache_percentage", Type: module.Stacked, - Priority: cachePercentagePrio, + Priority: prioCachePercentage, Dims: Dims{ {ID: "total.num.cachehits", Name: "hits", Algo: module.PercentOfAbsolute}, {ID: "total.num.cachemiss", Name: "miss", Algo: module.PercentOfAbsolute}, @@ -178,7 +178,7 @@ var ( Units: "prefetches", Fam: "cache", Ctx: "unbound.prefetch", - Priority: prefetchPrio, + Priority: prioCachePrefetch, Dims: Dims{ {ID: "total.num.prefetch", Name: "prefetches"}, }, @@ -189,7 +189,7 @@ var ( Units: "replies", Fam: "cache", Ctx: "unbound.zero_ttl_replies", - Priority: zeroTTLPrio, + Priority: prioZeroTTL, Dims: Dims{ {ID: "total.num.zero_ttl", Name: "zero_ttl"}, }, @@ -200,7 +200,7 @@ var ( Units: "replies", Fam: "replies", Ctx: "unbound.recursive_replies", - Priority: recurRepliesPrio, + Priority: prioRecurReplies, Dims: Dims{ {ID: "total.num.recursivereplies", Name: "recursive"}, }, @@ -211,7 +211,7 @@ var ( Units: "milliseconds", Fam: "recursion timings", Ctx: "unbound.recursion_time", - Priority: recurTimePrio, + Priority: prioRecurTime, Dims: Dims{ {ID: "total.recursion.time.avg", Name: "avg"}, {ID: "total.recursion.time.median", Name: "median"}, @@ -223,7 +223,7 @@ var ( Units: "queries", Fam: "request list", Ctx: "unbound.request_list_usage", - Priority: reqListUsagePrio, + Priority: prioReqListUsage, Dims: Dims{ {ID: "total.requestlist.avg", Name: "avg", Div: 1000}, {ID: "total.requestlist.max", Name: "max"}, // all time max in cumulative mode, never resets @@ -236,7 +236,7 @@ var ( Fam: "request list", Ctx: "unbound.current_request_list_usage", Type: module.Area, - Priority: reqListCurUsagePrio, + Priority: prioReqListCurUsage, Dims: Dims{ {ID: "total.requestlist.current.all", Name: "all"}, {ID: "total.requestlist.current.user", Name: "user"}, @@ -248,7 +248,7 @@ var ( Units: "queries", Fam: "request list", Ctx: "unbound.request_list_jostle_list", - Priority: reqListJostlePrio, + Priority: prioReqListJostle, Dims: Dims{ {ID: "total.requestlist.overwritten", Name: "overwritten"}, {ID: "total.requestlist.exceeded", Name: "dropped"}, @@ -260,7 +260,7 @@ var ( Units: "buffers", Fam: "tcp buffers", Ctx: "unbound.tcpusage", - Priority: tcpUsagePrio, + Priority: prioTCPUsage, Dims: Dims{ {ID: "total.tcpusage", Name: "usage"}, }, @@ -271,7 +271,7 @@ var ( Units: "seconds", Fam: "uptime", Ctx: "unbound.uptime", - Priority: uptimePrio, + Priority: prioUptime, Dims: Dims{ {ID: "time.up", Name: "time"}, }, @@ -288,7 +288,7 @@ var ( Fam: "mem", Ctx: "unbound.cache_memory", Type: module.Stacked, - Priority: memCachePrio, + Priority: prioMemCache, Dims: Dims{ {ID: "mem.cache.message", Name: "message", Div: 1024}, {ID: "mem.cache.rrset", Name: "rrset", Div: 1024}, @@ -304,7 +304,7 @@ var ( Fam: "mem", Ctx: "unbound.mod_memory", Type: module.Stacked, - Priority: memModPrio, + Priority: prioMemMod, Dims: Dims{ {ID: "mem.mod.iterator", Name: "iterator", Div: 1024}, {ID: "mem.mod.respip", Name: "respip", Div: 1024}, @@ -319,7 +319,7 @@ var ( Units: "KB", Fam: "mem", Ctx: "unbound.mem_streamwait", - Priority: memStreamWaitPrio, + Priority: prioMemStreamWait, Dims: Dims{ {ID: "mem.streamwait", Name: "streamwait", Div: 1024}, }, @@ -332,7 +332,7 @@ var ( Fam: "cache", Ctx: "unbound.cache_count", Type: module.Stacked, - Priority: cacheCountPrio, + Priority: prioCacheCount, Dims: Dims{ {ID: "infra.cache.count", Name: "infra"}, {ID: "key.cache.count", Name: "key"}, @@ -349,7 +349,7 @@ var ( Fam: "queries", Ctx: "unbound.type_queries", Type: module.Stacked, - Priority: queryTypePrio, + Priority: prioQueryType, } queryClassChart = Chart{ ID: "queries_by_class", @@ -358,7 +358,7 @@ var ( Fam: "queries", Ctx: "unbound.class_queries", Type: module.Stacked, - Priority: queryClassPrio, + Priority: prioQueryClass, } queryOpCodeChart = Chart{ ID: "queries_by_opcode", @@ -367,7 +367,7 @@ var ( Fam: "queries", Ctx: "unbound.opcode_queries", Type: module.Stacked, - Priority: queryOpCodePrio, + Priority: prioQueryOpCode, } queryFlagChart = Chart{ ID: "queries_by_flag", @@ -376,7 +376,7 @@ var ( Fam: "queries", Ctx: "unbound.flag_queries", Type: module.Stacked, - Priority: queryFlagPrio, + Priority: prioQueryFlag, Dims: Dims{ {ID: "num.query.flags.QR", Name: "QR"}, {ID: "num.query.flags.AA", Name: "AA"}, @@ -395,7 +395,7 @@ var ( Fam: "replies", Ctx: "unbound.rcode_answers", Type: module.Stacked, - Priority: replyRCodePrio, + Priority: prioReplyRCode, } ) From f5445faf9aa9e8d0baa17e41dfa40c1577cec031 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Wed, 20 Nov 2019 00:54:57 +0300 Subject: [PATCH 38/39] readme suggested changes --- modules/unbound/README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/modules/unbound/README.md b/modules/unbound/README.md index c39131832..38d8d5128 100644 --- a/modules/unbound/README.md +++ b/modules/unbound/README.md @@ -2,7 +2,7 @@ This module monitors one or more [`Unbound`](https://nlnetlabs.nl/projects/unbound/about/) servers, depending on your configuration. -#### Requirements +## Requirements - `Unbound` with enabled `remote-control` interface (see [unbound.conf](https://nlnetlabs.nl/documentation/unbound/unbound.conf)) @@ -23,7 +23,7 @@ For auto detection parameters from `unbound.conf`: - `unbound.conf` should be readable by `netdata` user -#### Charts +## Charts Module produces following summary charts: @@ -55,6 +55,7 @@ If `extended-statistics` is enabled: - TCP and TLS Stream Waif Buffer Memory in `KB` Per thread charts (only if number of threads > 1): + - Received Queries in `queries` - Rate Limited Queries in `queries` - DNSCrypt Queries in `queries` @@ -70,20 +71,20 @@ Per thread charts (only if number of threads > 1): - TCP Handler Buffers in `buffers` -### Configuration +## Configuration -Needs only `address` to server's `remote-control` interface if TLS is disabled or `address` is unix socket. +This Unbound collector only needs the `address` to aserver's `remote-control` interface if TLS is disabled or `address` is unix socket. Otherwise you need to set path to the `control-key-file` and `control-cert-file` files. -Module tries to auto detect following parameters reading `unbound.conf`: +The module tries to auto-detect following parameters reading `unbound.conf`: - address - cumulative - use_tls - tls_cert - tls_key -Module supports both cumulative and non cumulative modes. Default is non cumulative. If your server has enabled -`statistics-cumulative` but module fails to auto detect it (`unbound.conf` is not readable or it is a remote server) +Module supports both cumulative and non-cumulative modes. Default is non-cumulative. If your server has enabled +`statistics-cumulative`, but the module fails to auto-detect it (`unbound.conf` is not readable or it is a remote server), you need to set it manually in the configuration file. Here is an example for several servers: @@ -113,7 +114,8 @@ jobs: For all available options, please see the module [configuration file](https://github.com/netdata/go.d.plugin/blob/master/config/go.d/unbound.conf). -### Troubleshooting +## Troubleshooting + Ensure that the control protocol is actually configured correctly. Run following command as `root` user: > unbound-control stats_noreset From ea44585724109051a428cf95730521a982e0698b Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Thu, 21 Nov 2019 17:43:17 +0300 Subject: [PATCH 39/39] enable by default --- modules/unbound/unbound.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/unbound/unbound.go b/modules/unbound/unbound.go index 47c55b8e1..b1f791736 100644 --- a/modules/unbound/unbound.go +++ b/modules/unbound/unbound.go @@ -10,10 +10,6 @@ import ( func init() { creator := module.Creator{ - // TODO: python one is so broken, we need just to obsolete it and enable this one - Defaults: module.Defaults{ - Disabled: true, - }, Create: func() module.Module { return New() }, }