From 9629c6ed18b2cfc13dca1597a8ae15314bdd4733 Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Tue, 12 Nov 2019 12:22:13 +0200 Subject: [PATCH 1/4] Added OOM notification for Memory controller Signed-off-by: bpopovschi --- v2/manager.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/v2/manager.go b/v2/manager.go index a7137190..30f8bb67 100644 --- a/v2/manager.go +++ b/v2/manager.go @@ -25,8 +25,11 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "time" + "golang.org/x/sys/unix" + "github.com/containerd/cgroups/v2/stats" "github.com/pkg/errors" ) @@ -411,3 +414,19 @@ func (c *Manager) freeze(path string, state State) error { time.Sleep(1 * time.Millisecond) } } + +func (c *Manager) OOMEventFD(rootPath string) (uintptr, error) { + fpath := filepath.Join(rootPath, "memory.events") + fd, err := syscall.InotifyInit() + if err != nil { + return 0, fmt.Errorf("Failed to create inotify fd") + } + defer syscall.Close(fd) + wd, err := syscall.InotifyAddWatch(fd, fpath, unix.IN_MODIFY) + if wd < 0 { + return 0, fmt.Errorf("Failed to add inotify watch for %q", fpath) + } + defer syscall.InotifyRmWatch(fd, uint32(wd)) + + return uintptr(fd), nil +} From eb898ed08d0acb48db00b8bd01cc9227b001641b Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Tue, 12 Nov 2019 15:58:34 +0200 Subject: [PATCH 2/4] Added event channel for oom event subscription Signed-off-by: bpopovschi --- v2/manager.go | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/v2/manager.go b/v2/manager.go index 30f8bb67..ab6f396b 100644 --- a/v2/manager.go +++ b/v2/manager.go @@ -20,6 +20,7 @@ import ( "bufio" "fmt" "io/ioutil" + "log" "math" "os" "path/filepath" @@ -39,10 +40,14 @@ const ( controllersFile = "cgroup.controllers" ) +var oomEventCount = 0 + type cgValuer interface { Values() []Value } +type Event struct{} + // Resources for a cgroups v2 unified hierarchy type Resources struct { CPU *CPU @@ -415,8 +420,8 @@ func (c *Manager) freeze(path string, state State) error { } } -func (c *Manager) OOMEventFD(rootPath string) (uintptr, error) { - fpath := filepath.Join(rootPath, "memory.events") +func (c *Manager) OOMEventFD() (uintptr, error) { + fpath := filepath.Join(c.path, "memory.events") fd, err := syscall.InotifyInit() if err != nil { return 0, fmt.Errorf("Failed to create inotify fd") @@ -430,3 +435,34 @@ func (c *Manager) OOMEventFD(rootPath string) (uintptr, error) { return uintptr(fd), nil } + +func (c *Manager) EventChan() (<-chan Event, error) { + fd, err := c.OOMEventFD() + if err != nil { + return nil, fmt.Errorf("Failed to create oom event fd") + } + ec := make(chan Event) + go c.waitForOOMEvents(int(fd), ec) + + return ec, nil +} + +func (c *Manager) waitForOOMEvents(fd int, ec chan<- Event) { + for { + buffer := make([]byte, syscall.SizeofInotifyEvent*10) + bytesRead, err := syscall.Read(fd, buffer) + if err != nil { + log.Fatal(err) + } + var out map[string]interface{} + if bytesRead >= syscall.SizeofInotifyEvent { + if err := readStatsFile(c.path, "memory.events", out); err != nil { + val, ok := out["oom"] + if ok && val.(int) > oomEventCount { + oomEventCount = val.(int) + ec <- Event{} + } + } + } + } +} From 4aad508d1059e09c03caef3ed781781605ab0020 Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Wed, 13 Nov 2019 09:53:41 +0200 Subject: [PATCH 3/4] Added events struct and error channel Signed-off-by: bpopovschi --- v2/manager.go | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/v2/manager.go b/v2/manager.go index ab6f396b..bc1db7c0 100644 --- a/v2/manager.go +++ b/v2/manager.go @@ -20,7 +20,6 @@ import ( "bufio" "fmt" "io/ioutil" - "log" "math" "os" "path/filepath" @@ -46,7 +45,13 @@ type cgValuer interface { Values() []Value } -type Event struct{} +type Event struct { + Low uint64 + High uint64 + Max uint64 + OOM uint64 + OOMKill uint64 +} // Resources for a cgroups v2 unified hierarchy type Resources struct { @@ -420,48 +425,52 @@ func (c *Manager) freeze(path string, state State) error { } } -func (c *Manager) OOMEventFD() (uintptr, error) { +func (c *Manager) MemoryEventFD() (uintptr, error) { fpath := filepath.Join(c.path, "memory.events") fd, err := syscall.InotifyInit() if err != nil { - return 0, fmt.Errorf("Failed to create inotify fd") + return 0, errors.Errorf("Failed to create inotify fd") } defer syscall.Close(fd) wd, err := syscall.InotifyAddWatch(fd, fpath, unix.IN_MODIFY) if wd < 0 { - return 0, fmt.Errorf("Failed to add inotify watch for %q", fpath) + return 0, errors.Errorf("Failed to add inotify watch for %q", fpath) } defer syscall.InotifyRmWatch(fd, uint32(wd)) return uintptr(fd), nil } -func (c *Manager) EventChan() (<-chan Event, error) { - fd, err := c.OOMEventFD() - if err != nil { - return nil, fmt.Errorf("Failed to create oom event fd") - } +func (c *Manager) EventChan() (<-chan Event, <-chan error) { ec := make(chan Event) - go c.waitForOOMEvents(int(fd), ec) + errCh := make(chan error) + go c.waitForEvents(ec, errCh) return ec, nil } -func (c *Manager) waitForOOMEvents(fd int, ec chan<- Event) { +func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { + fd, err := c.MemoryEventFD() + if err != nil { + errCh <- errors.Errorf("Failed to create memory event fd") + } for { buffer := make([]byte, syscall.SizeofInotifyEvent*10) - bytesRead, err := syscall.Read(fd, buffer) + bytesRead, err := syscall.Read(int(fd), buffer) if err != nil { - log.Fatal(err) + errCh <- err } var out map[string]interface{} if bytesRead >= syscall.SizeofInotifyEvent { if err := readStatsFile(c.path, "memory.events", out); err != nil { - val, ok := out["oom"] - if ok && val.(int) > oomEventCount { - oomEventCount = val.(int) - ec <- Event{} + e := Event{ + High: out["high"].(uint64), + Low: out["low"].(uint64), + Max: out["max"].(uint64), + OOM: out["oom"].(uint64), + OOMKill: out["oom_kill"].(uint64), } + ec <- e } } } From d03d5167784d274aa37013826c6a09ded7d6b978 Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Wed, 13 Nov 2019 09:54:52 +0200 Subject: [PATCH 4/4] Remove global var Signed-off-by: bpopovschi --- v2/manager.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/manager.go b/v2/manager.go index bc1db7c0..a9510867 100644 --- a/v2/manager.go +++ b/v2/manager.go @@ -39,8 +39,6 @@ const ( controllersFile = "cgroup.controllers" ) -var oomEventCount = 0 - type cgValuer interface { Values() []Value }