diff --git a/pkg/cgroups/blkio.go b/pkg/cgroups/blkio.go deleted file mode 100644 index e157e6faf..000000000 --- a/pkg/cgroups/blkio.go +++ /dev/null @@ -1,151 +0,0 @@ -//go:build !linux -// +build !linux - -package cgroups - -import ( - "bufio" - "errors" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -type blkioHandler struct{} - -func getBlkioHandler() *blkioHandler { - return &blkioHandler{} -} - -// Apply set the specified constraints -func (c *blkioHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.BlockIO == nil { - return nil - } - return fmt.Errorf("blkio apply function not implemented yet") -} - -// Create the cgroup -func (c *blkioHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - return false, nil - } - return ctr.createCgroupDirectory(Blkio) -} - -// Destroy the cgroup -func (c *blkioHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(Blkio)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *blkioHandler) Stat(ctr *CgroupControl, m *Metrics) error { - var ioServiceBytesRecursive []BlkIOEntry - - if ctr.cgroup2 { - // more details on the io.stat file format:X https://facebookmicrosites.github.io/cgroup2/docs/io-controller.html - values, err := readCgroup2MapFile(ctr, "io.stat") - if err != nil { - return err - } - for k, v := range values { - d := strings.Split(k, ":") - if len(d) != 2 { - continue - } - minor, err := strconv.ParseUint(d[0], 10, 0) - if err != nil { - return err - } - major, err := strconv.ParseUint(d[1], 10, 0) - if err != nil { - return err - } - - for _, item := range v { - d := strings.Split(item, "=") - if len(d) != 2 { - continue - } - op := d[0] - - // Accommodate the cgroup v1 naming - switch op { - case "rbytes": - op = "read" - case "wbytes": - op = "write" - } - - value, err := strconv.ParseUint(d[1], 10, 0) - if err != nil { - return err - } - - entry := BlkIOEntry{ - Op: op, - Major: major, - Minor: minor, - Value: value, - } - ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) - } - } - } else { - BlkioRoot := ctr.getCgroupv1Path(Blkio) - - p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive") - f, err := os.Open(p) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return fmt.Errorf("open %s: %w", p, err) - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Fields(line) - if len(parts) < 3 { - continue - } - d := strings.Split(parts[0], ":") - if len(d) != 2 { - continue - } - minor, err := strconv.ParseUint(d[0], 10, 0) - if err != nil { - return err - } - major, err := strconv.ParseUint(d[1], 10, 0) - if err != nil { - return err - } - - op := parts[1] - - value, err := strconv.ParseUint(parts[2], 10, 0) - if err != nil { - return err - } - entry := BlkIOEntry{ - Op: op, - Major: major, - Minor: minor, - Value: value, - } - ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) - } - if err := scanner.Err(); err != nil { - return fmt.Errorf("parse %s: %w", p, err) - } - } - m.Blkio = BlkioMetrics{IoServiceBytesRecursive: ioServiceBytesRecursive} - return nil -} diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go deleted file mode 100644 index 10b70b8f9..000000000 --- a/pkg/cgroups/cgroups.go +++ /dev/null @@ -1,614 +0,0 @@ -//go:build !linux -// +build !linux - -package cgroups - -import ( - "bufio" - "context" - "errors" - "fmt" - "math" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/containers/storage/pkg/unshare" - systemdDbus "github.com/coreos/go-systemd/v22/dbus" - "github.com/godbus/dbus/v5" - spec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" -) - -var ( - // ErrCgroupDeleted means the cgroup was deleted - ErrCgroupDeleted = errors.New("cgroup deleted") - // ErrCgroupV1Rootless means the cgroup v1 were attempted to be used in rootless environment - ErrCgroupV1Rootless = errors.New("no support for CGroups V1 in rootless environments") - ErrStatCgroup = errors.New("no cgroup available for gathering user statistics") -) - -// CgroupControl controls a cgroup hierarchy -type CgroupControl struct { - cgroup2 bool - path string - systemd bool - // List of additional cgroup subsystems joined that - // do not have a custom handler. - additionalControllers []controller -} - -// CPUUsage keeps stats for the CPU usage (unit: nanoseconds) -type CPUUsage struct { - Kernel uint64 - Total uint64 - PerCPU []uint64 -} - -// MemoryUsage keeps stats for the memory usage -type MemoryUsage struct { - Usage uint64 - Limit uint64 -} - -// CPUMetrics keeps stats for the CPU usage -type CPUMetrics struct { - Usage CPUUsage -} - -// BlkIOEntry describes an entry in the blkio stats -type BlkIOEntry struct { - Op string - Major uint64 - Minor uint64 - Value uint64 -} - -// BlkioMetrics keeps usage stats for the blkio cgroup controller -type BlkioMetrics struct { - IoServiceBytesRecursive []BlkIOEntry -} - -// MemoryMetrics keeps usage stats for the memory cgroup controller -type MemoryMetrics struct { - Usage MemoryUsage -} - -// PidsMetrics keeps usage stats for the pids cgroup controller -type PidsMetrics struct { - Current uint64 -} - -// Metrics keeps usage stats for the cgroup controllers -type Metrics struct { - CPU CPUMetrics - Blkio BlkioMetrics - Memory MemoryMetrics - Pids PidsMetrics -} - -type controller struct { - name string - symlink bool -} - -type controllerHandler interface { - Create(*CgroupControl) (bool, error) - Apply(*CgroupControl, *spec.LinuxResources) error - Destroy(*CgroupControl) error - Stat(*CgroupControl, *Metrics) error -} - -const ( - cgroupRoot = "/sys/fs/cgroup" - // CPU is the cpu controller - CPU = "cpu" - // CPUAcct is the cpuacct controller - CPUAcct = "cpuacct" - // CPUset is the cpuset controller - CPUset = "cpuset" - // Memory is the memory controller - Memory = "memory" - // Pids is the pids controller - Pids = "pids" - // Blkio is the blkio controller - Blkio = "blkio" -) - -var handlers map[string]controllerHandler - -func init() { - handlers = make(map[string]controllerHandler) - handlers[CPU] = getCPUHandler() - handlers[CPUset] = getCpusetHandler() - handlers[Memory] = getMemoryHandler() - handlers[Pids] = getPidsHandler() - handlers[Blkio] = getBlkioHandler() -} - -// getAvailableControllers get the available controllers -func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) { - if cgroup2 { - controllers := []controller{} - controllersFile := cgroupRoot + "/cgroup.controllers" - // rootless cgroupv2: check available controllers for current user, systemd or servicescope will inherit - if unshare.IsRootless() { - userSlice, err := getCgroupPathForCurrentProcess() - if err != nil { - return controllers, err - } - // userSlice already contains '/' so not adding here - basePath := cgroupRoot + userSlice - controllersFile = fmt.Sprintf("%s/cgroup.controllers", basePath) - } - controllersFileBytes, err := os.ReadFile(controllersFile) - if err != nil { - return nil, fmt.Errorf("failed while reading controllers for cgroup v2: %w", err) - } - for _, controllerName := range strings.Fields(string(controllersFileBytes)) { - c := controller{ - name: controllerName, - symlink: false, - } - controllers = append(controllers, c) - } - return controllers, nil - } - - subsystems, _ := cgroupV1GetAllSubsystems() - controllers := []controller{} - // cgroupv1 and rootless: No subsystem is available: delegation is unsafe. - if unshare.IsRootless() { - return controllers, nil - } - - for _, name := range subsystems { - if _, found := exclude[name]; found { - continue - } - fileInfo, err := os.Stat(cgroupRoot + "/" + name) - if err != nil { - continue - } - c := controller{ - name: name, - symlink: !fileInfo.IsDir(), - } - controllers = append(controllers, c) - } - - return controllers, nil -} - -// GetAvailableControllers get string:bool map of all the available controllers -func GetAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]string, error) { - availableControllers, err := getAvailableControllers(exclude, cgroup2) - if err != nil { - return nil, err - } - controllerList := []string{} - for _, controller := range availableControllers { - controllerList = append(controllerList, controller.name) - } - - return controllerList, nil -} - -func cgroupV1GetAllSubsystems() ([]string, error) { - f, err := os.Open("/proc/cgroups") - if err != nil { - return nil, err - } - defer f.Close() - - subsystems := []string{} - - s := bufio.NewScanner(f) - for s.Scan() { - text := s.Text() - if text[0] != '#' { - parts := strings.Fields(text) - if len(parts) >= 4 && parts[3] != "0" { - subsystems = append(subsystems, parts[0]) - } - } - } - if err := s.Err(); err != nil { - return nil, err - } - return subsystems, nil -} - -func getCgroupPathForCurrentProcess() (string, error) { - path := fmt.Sprintf("/proc/%d/cgroup", os.Getpid()) - f, err := os.Open(path) - if err != nil { - return "", err - } - defer f.Close() - - cgroupPath := "" - s := bufio.NewScanner(f) - for s.Scan() { - text := s.Text() - procEntries := strings.SplitN(text, "::", 2) - // set process cgroupPath only if entry is valid - if len(procEntries) > 1 { - cgroupPath = procEntries[1] - } - } - if err := s.Err(); err != nil { - return cgroupPath, err - } - return cgroupPath, nil -} - -// getCgroupv1Path is a helper function to get the cgroup v1 path -func (c *CgroupControl) getCgroupv1Path(name string) string { - return filepath.Join(cgroupRoot, name, c.path) -} - -// initialize initializes the specified hierarchy -func (c *CgroupControl) initialize() (err error) { - createdSoFar := map[string]controllerHandler{} - defer func() { - if err != nil { - for name, ctr := range createdSoFar { - if err := ctr.Destroy(c); err != nil { - logrus.Warningf("error cleaning up controller %s for %s", name, c.path) - } - } - } - }() - if c.cgroup2 { - if err := createCgroupv2Path(filepath.Join(cgroupRoot, c.path)); err != nil { - return fmt.Errorf("creating cgroup path %s: %w", c.path, err) - } - } - for name, handler := range handlers { - created, err := handler.Create(c) - if err != nil { - return err - } - if created { - createdSoFar[name] = handler - } - } - - if !c.cgroup2 { - // We won't need to do this for cgroup v2 - for _, ctr := range c.additionalControllers { - if ctr.symlink { - continue - } - path := c.getCgroupv1Path(ctr.name) - if err := os.MkdirAll(path, 0o755); err != nil { - return fmt.Errorf("creating cgroup path for %s: %w", ctr.name, err) - } - } - } - - return nil -} - -func readFileAsUint64(path string) (uint64, error) { - data, err := os.ReadFile(path) - if err != nil { - return 0, err - } - v := cleanString(string(data)) - if v == "max" { - return math.MaxUint64, nil - } - ret, err := strconv.ParseUint(v, 10, 64) - if err != nil { - return ret, fmt.Errorf("parse %s from %s: %w", v, path, err) - } - return ret, nil -} - -func readFileByKeyAsUint64(path, key string) (uint64, error) { - content, err := os.ReadFile(path) - if err != nil { - return 0, err - } - for _, line := range strings.Split(string(content), "\n") { - fields := strings.SplitN(line, " ", 2) - if fields[0] == key { - v := cleanString(string(fields[1])) - if v == "max" { - return math.MaxUint64, nil - } - ret, err := strconv.ParseUint(v, 10, 64) - if err != nil { - return ret, fmt.Errorf("parse %s from %s: %w", v, path, err) - } - return ret, nil - } - } - - return 0, fmt.Errorf("no key named %s from %s", key, path) -} - -// New creates a new cgroup control -func New(path string, resources *spec.LinuxResources) (*CgroupControl, error) { - cgroup2, err := IsCgroup2UnifiedMode() - if err != nil { - return nil, err - } - control := &CgroupControl{ - cgroup2: cgroup2, - path: path, - } - - if !cgroup2 { - controllers, err := getAvailableControllers(handlers, false) - if err != nil { - return nil, err - } - control.additionalControllers = controllers - } - - if err := control.initialize(); err != nil { - return nil, err - } - - return control, nil -} - -// NewSystemd creates a new cgroup control -func NewSystemd(path string) (*CgroupControl, error) { - cgroup2, err := IsCgroup2UnifiedMode() - if err != nil { - return nil, err - } - control := &CgroupControl{ - cgroup2: cgroup2, - path: path, - systemd: true, - } - return control, nil -} - -// Load loads an existing cgroup control -func Load(path string) (*CgroupControl, error) { - cgroup2, err := IsCgroup2UnifiedMode() - if err != nil { - return nil, err - } - control := &CgroupControl{ - cgroup2: cgroup2, - path: path, - systemd: false, - } - if !cgroup2 { - controllers, err := getAvailableControllers(handlers, false) - if err != nil { - return nil, err - } - control.additionalControllers = controllers - } - if !cgroup2 { - oneExists := false - // check that the cgroup exists at least under one controller - for name := range handlers { - p := control.getCgroupv1Path(name) - if _, err := os.Stat(p); err == nil { - oneExists = true - break - } - } - - // if there is no controller at all, raise an error - if !oneExists { - if unshare.IsRootless() { - return nil, ErrCgroupV1Rootless - } - // compatible with the error code - // used by containerd/cgroups - return nil, ErrCgroupDeleted - } - } - return control, nil -} - -// CreateSystemdUnit creates the systemd cgroup -func (c *CgroupControl) CreateSystemdUnit(path string) error { - if !c.systemd { - return fmt.Errorf("the cgroup controller is not using systemd") - } - - conn, err := systemdDbus.NewWithContext(context.TODO()) - if err != nil { - return err - } - defer conn.Close() - - return systemdCreate(path, conn) -} - -// GetUserConnection returns a user connection to D-BUS -func GetUserConnection(uid int) (*systemdDbus.Conn, error) { - return systemdDbus.NewConnection(func() (*dbus.Conn, error) { - return dbusAuthConnection(uid, dbus.SessionBusPrivateNoAutoStartup) - }) -} - -// CreateSystemdUserUnit creates the systemd cgroup for the specified user -func (c *CgroupControl) CreateSystemdUserUnit(path string, uid int) error { - if !c.systemd { - return fmt.Errorf("the cgroup controller is not using systemd") - } - - conn, err := GetUserConnection(uid) - if err != nil { - return err - } - defer conn.Close() - - return systemdCreate(path, conn) -} - -func dbusAuthConnection(uid int, createBus func(opts ...dbus.ConnOption) (*dbus.Conn, error)) (*dbus.Conn, error) { - conn, err := createBus() - if err != nil { - return nil, err - } - - methods := []dbus.Auth{dbus.AuthExternal(strconv.Itoa(uid))} - - err = conn.Auth(methods) - if err != nil { - conn.Close() - return nil, err - } - if err := conn.Hello(); err != nil { - return nil, err - } - - return conn, nil -} - -// Delete cleans a cgroup -func (c *CgroupControl) Delete() error { - return c.DeleteByPath(c.path) -} - -// DeleteByPathConn deletes the specified cgroup path using the specified -// dbus connection if needed. -func (c *CgroupControl) DeleteByPathConn(path string, conn *systemdDbus.Conn) error { - if c.systemd { - return systemdDestroyConn(path, conn) - } - if c.cgroup2 { - return rmDirRecursively(filepath.Join(cgroupRoot, c.path)) - } - var lastError error - for _, h := range handlers { - if err := h.Destroy(c); err != nil { - lastError = err - } - } - - for _, ctr := range c.additionalControllers { - if ctr.symlink { - continue - } - p := c.getCgroupv1Path(ctr.name) - if err := rmDirRecursively(p); err != nil { - lastError = fmt.Errorf("remove %s: %w", p, err) - } - } - return lastError -} - -// DeleteByPath deletes the specified cgroup path -func (c *CgroupControl) DeleteByPath(path string) error { - if c.systemd { - conn, err := systemdDbus.NewWithContext(context.TODO()) - if err != nil { - return err - } - defer conn.Close() - return c.DeleteByPathConn(path, conn) - } - return c.DeleteByPathConn(path, nil) -} - -// Update updates the cgroups -func (c *CgroupControl) Update(resources *spec.LinuxResources) error { - for _, h := range handlers { - if err := h.Apply(c, resources); err != nil { - return err - } - } - return nil -} - -// AddPid moves the specified pid to the cgroup -func (c *CgroupControl) AddPid(pid int) error { - pidString := []byte(fmt.Sprintf("%d\n", pid)) - - if c.cgroup2 { - p := filepath.Join(cgroupRoot, c.path, "cgroup.procs") - if err := os.WriteFile(p, pidString, 0o644); err != nil { - return fmt.Errorf("write %s: %w", p, err) - } - return nil - } - - names := make([]string, 0, len(handlers)) - for n := range handlers { - names = append(names, n) - } - - for _, c := range c.additionalControllers { - if !c.symlink { - names = append(names, c.name) - } - } - - for _, n := range names { - // If we aren't using cgroup2, we won't write correctly to unified hierarchy - if !c.cgroup2 && n == "unified" { - continue - } - p := filepath.Join(c.getCgroupv1Path(n), "tasks") - if err := os.WriteFile(p, pidString, 0o644); err != nil { - return fmt.Errorf("write %s: %w", p, err) - } - } - return nil -} - -// Stat returns usage statistics for the cgroup -func (c *CgroupControl) Stat() (*Metrics, error) { - m := Metrics{} - found := false - for _, h := range handlers { - if err := h.Stat(c, &m); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return nil, err - } - logrus.Warningf("Failed to retrieve cgroup stats: %v", err) - continue - } - found = true - } - if !found { - return nil, ErrStatCgroup - } - return &m, nil -} - -func readCgroupMapPath(path string) (map[string][]string, error) { - ret := map[string][]string{} - f, err := os.Open(path) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return ret, nil - } - return nil, fmt.Errorf("open file %s: %w", path, err) - } - defer f.Close() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Fields(line) - if len(parts) < 2 { - continue - } - ret[parts[0]] = parts[1:] - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("parsing file %s: %w", path, err) - } - return ret, nil -} - -func readCgroup2MapFile(ctr *CgroupControl, name string) (map[string][]string, error) { - p := filepath.Join(cgroupRoot, ctr.path, name) - - return readCgroupMapPath(p) -} diff --git a/pkg/cgroups/cgroups_linux.go b/pkg/cgroups/cgroups_linux.go index e778b0e89..a5713a1a7 100644 --- a/pkg/cgroups/cgroups_linux.go +++ b/pkg/cgroups/cgroups_linux.go @@ -5,6 +5,7 @@ package cgroups import ( "bufio" + "bytes" "context" "errors" "fmt" @@ -572,3 +573,132 @@ func readCgroup2MapFile(ctr *CgroupControl, name string) (map[string][]string, e return readCgroupMapPath(p) } + +func (c *CgroupControl) createCgroupDirectory(controller string) (bool, error) { + cPath := c.getCgroupv1Path(controller) + _, err := os.Stat(cPath) + if err == nil { + return false, nil + } + + if !errors.Is(err, os.ErrNotExist) { + return false, err + } + + if err := os.MkdirAll(cPath, 0o755); err != nil { + return false, fmt.Errorf("creating cgroup for %s: %w", controller, err) + } + return true, nil +} + +var TestMode bool + +func createCgroupv2Path(path string) (deferredError error) { + if !strings.HasPrefix(path, cgroupRoot+"/") { + return fmt.Errorf("invalid cgroup path %s", path) + } + content, err := os.ReadFile(cgroupRoot + "/cgroup.controllers") + if err != nil { + return err + } + ctrs := bytes.Fields(content) + res := append([]byte("+"), bytes.Join(ctrs, []byte(" +"))...) + + current := "/sys/fs" + elements := strings.Split(path, "/") + for i, e := range elements[3:] { + current = filepath.Join(current, e) + if i > 0 { + if err := os.Mkdir(current, 0o755); err != nil { + if !os.IsExist(err) { + return err + } + } else { + // If the directory was created, be sure it is not left around on errors. + defer func() { + if deferredError != nil { + os.Remove(current) + } + }() + } + } + // We enable the controllers for all the path components except the last one. It is not allowed to add + // PIDs if there are already enabled controllers. + if i < len(elements[3:])-1 { + if err := os.WriteFile(filepath.Join(current, "cgroup.subtree_control"), res, 0o755); err != nil { + return err + } + } + } + return nil +} + +func cleanString(s string) string { + return strings.Trim(s, "\n") +} + +func readAcct(ctr *CgroupControl, name string) (uint64, error) { + p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) + return readFileAsUint64(p) +} + +func readAcctList(ctr *CgroupControl, name string) ([]uint64, error) { + p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) + data, err := os.ReadFile(p) + if err != nil { + return nil, err + } + r := []uint64{} + for _, s := range strings.Split(string(data), " ") { + s = cleanString(s) + if s == "" { + break + } + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return nil, fmt.Errorf("parsing %s: %w", s, err) + } + r = append(r, v) + } + return r, nil +} + +func cpusetCopyFromParent(path string, cgroupv2 bool) error { + for _, file := range []string{"cpuset.cpus", "cpuset.mems"} { + if _, err := cpusetCopyFileFromParent(path, file, cgroupv2); err != nil { + return err + } + } + return nil +} + +func cpusetCopyFileFromParent(dir, file string, cgroupv2 bool) ([]byte, error) { + if dir == cgroupRoot { + return nil, fmt.Errorf("could not find parent to initialize cpuset %s", file) + } + path := filepath.Join(dir, file) + parentPath := path + if cgroupv2 { + parentPath = fmt.Sprintf("%s.effective", parentPath) + } + data, err := os.ReadFile(parentPath) + if err != nil { + // if the file doesn't exist, it is likely that the cpuset controller + // is not enabled in the kernel. + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if strings.Trim(string(data), "\n") != "" { + return data, nil + } + data, err = cpusetCopyFileFromParent(filepath.Dir(dir), file, cgroupv2) + if err != nil { + return nil, err + } + if err := os.WriteFile(path, data, 0o644); err != nil { + return nil, fmt.Errorf("write %s: %w", path, err) + } + return data, nil +} diff --git a/pkg/cgroups/cgroups_test.go b/pkg/cgroups/cgroups_test.go deleted file mode 100644 index ef5138587..000000000 --- a/pkg/cgroups/cgroups_test.go +++ /dev/null @@ -1,35 +0,0 @@ -//go:build !linux -// +build !linux - -package cgroups - -import ( - "testing" - - "github.com/containers/storage/pkg/unshare" - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -func TestCreated(t *testing.T) { - // tests only works in rootless mode - if unshare.IsRootless() { - return - } - - var resources spec.LinuxResources - cgr, err := New("machine.slice", &resources) - if err != nil { - t.Error(err) - } - if err := cgr.Delete(); err != nil { - t.Error(err) - } - - cgr, err = NewSystemd("machine.slice") - if err != nil { - t.Error(err) - } - if err := cgr.Delete(); err != nil { - t.Error(err) - } -} diff --git a/pkg/cgroups/cpu.go b/pkg/cgroups/cpu.go deleted file mode 100644 index 16293e74c..000000000 --- a/pkg/cgroups/cpu.go +++ /dev/null @@ -1,91 +0,0 @@ -//go:build !linux -// +build !linux - -package cgroups - -import ( - "errors" - "fmt" - "os" - "strconv" - - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -type cpuHandler struct{} - -func getCPUHandler() *cpuHandler { - return &cpuHandler{} -} - -// Apply set the specified constraints -func (c *cpuHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.CPU == nil { - return nil - } - return fmt.Errorf("cpu apply not implemented yet") -} - -// Create the cgroup -func (c *cpuHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - return false, nil - } - return ctr.createCgroupDirectory(CPU) -} - -// Destroy the cgroup -func (c *cpuHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(CPU)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *cpuHandler) Stat(ctr *CgroupControl, m *Metrics) error { - var err error - usage := CPUUsage{} - if ctr.cgroup2 { - values, err := readCgroup2MapFile(ctr, "cpu.stat") - if err != nil { - return err - } - if val, found := values["usage_usec"]; found { - usage.Total, err = strconv.ParseUint(cleanString(val[0]), 10, 64) - if err != nil { - return err - } - usage.Kernel *= 1000 - } - if val, found := values["system_usec"]; found { - usage.Kernel, err = strconv.ParseUint(cleanString(val[0]), 10, 64) - if err != nil { - return err - } - usage.Total *= 1000 - } - // FIXME: How to read usage.PerCPU? - } else { - usage.Total, err = readAcct(ctr, "cpuacct.usage") - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return err - } - usage.Total = 0 - } - usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return err - } - usage.Kernel = 0 - } - usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return err - } - usage.PerCPU = nil - } - } - m.CPU = CPUMetrics{Usage: usage} - return nil -} diff --git a/pkg/cgroups/cpuset.go b/pkg/cgroups/cpuset.go deleted file mode 100644 index f7ec9a33b..000000000 --- a/pkg/cgroups/cpuset.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build !linux -// +build !linux - -package cgroups - -import ( - "fmt" - "path/filepath" - - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -type cpusetHandler struct{} - -func getCpusetHandler() *cpusetHandler { - return &cpusetHandler{} -} - -// Apply set the specified constraints -func (c *cpusetHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.CPU == nil { - return nil - } - return fmt.Errorf("cpuset apply not implemented yet") -} - -// Create the cgroup -func (c *cpusetHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - path := filepath.Join(cgroupRoot, ctr.path) - return true, cpusetCopyFromParent(path, true) - } - - created, err := ctr.createCgroupDirectory(CPUset) - if !created || err != nil { - return created, err - } - return true, cpusetCopyFromParent(ctr.getCgroupv1Path(CPUset), false) -} - -// Destroy the cgroup -func (c *cpusetHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(CPUset)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *cpusetHandler) Stat(ctr *CgroupControl, m *Metrics) error { - return nil -} diff --git a/pkg/cgroups/memory.go b/pkg/cgroups/memory.go deleted file mode 100644 index b597b85bf..000000000 --- a/pkg/cgroups/memory.go +++ /dev/null @@ -1,69 +0,0 @@ -//go:build !linux -// +build !linux - -package cgroups - -import ( - "fmt" - "path/filepath" - - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -type memHandler struct{} - -func getMemoryHandler() *memHandler { - return &memHandler{} -} - -// Apply set the specified constraints -func (c *memHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.Memory == nil { - return nil - } - return fmt.Errorf("memory apply not implemented yet") -} - -// Create the cgroup -func (c *memHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - return false, nil - } - return ctr.createCgroupDirectory(Memory) -} - -// Destroy the cgroup -func (c *memHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(Memory)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *memHandler) Stat(ctr *CgroupControl, m *Metrics) error { - var err error - usage := MemoryUsage{} - - var memoryRoot string - var limitFilename string - - if ctr.cgroup2 { - memoryRoot = filepath.Join(cgroupRoot, ctr.path) - limitFilename = "memory.max" - if usage.Usage, err = readFileByKeyAsUint64(filepath.Join(memoryRoot, "memory.stat"), "anon"); err != nil { - return err - } - } else { - memoryRoot = ctr.getCgroupv1Path(Memory) - limitFilename = "memory.limit_in_bytes" - if usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, "memory.usage_in_bytes")); err != nil { - return err - } - } - - usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, limitFilename)) - if err != nil { - return err - } - - m.Memory = MemoryMetrics{Usage: usage} - return nil -} diff --git a/pkg/cgroups/pids.go b/pkg/cgroups/pids.go deleted file mode 100644 index 76e983ea9..000000000 --- a/pkg/cgroups/pids.go +++ /dev/null @@ -1,71 +0,0 @@ -//go:build !linux -// +build !linux - -package cgroups - -import ( - "fmt" - "os" - "path/filepath" - - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -type pidHandler struct{} - -func getPidsHandler() *pidHandler { - return &pidHandler{} -} - -// Apply set the specified constraints -func (c *pidHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.Pids == nil { - return nil - } - var PIDRoot string - - if ctr.cgroup2 { - PIDRoot = filepath.Join(cgroupRoot, ctr.path) - } else { - PIDRoot = ctr.getCgroupv1Path(Pids) - } - - p := filepath.Join(PIDRoot, "pids.max") - return os.WriteFile(p, []byte(fmt.Sprintf("%d\n", res.Pids.Limit)), 0o644) -} - -// Create the cgroup -func (c *pidHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - return false, nil - } - return ctr.createCgroupDirectory(Pids) -} - -// Destroy the cgroup -func (c *pidHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(Pids)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *pidHandler) Stat(ctr *CgroupControl, m *Metrics) error { - if ctr.path == "" { - // nothing we can do to retrieve the pids.current path - return nil - } - - var PIDRoot string - if ctr.cgroup2 { - PIDRoot = filepath.Join(cgroupRoot, ctr.path) - } else { - PIDRoot = ctr.getCgroupv1Path(Pids) - } - - current, err := readFileAsUint64(filepath.Join(PIDRoot, "pids.current")) - if err != nil { - return err - } - - m.Pids = PidsMetrics{Current: current} - return nil -} diff --git a/pkg/cgroups/utils.go b/pkg/cgroups/utils.go deleted file mode 100644 index a77ba4a27..000000000 --- a/pkg/cgroups/utils.go +++ /dev/null @@ -1,179 +0,0 @@ -package cgroups - -import ( - "bytes" - "errors" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" -) - -var TestMode bool - -func cleanString(s string) string { - return strings.Trim(s, "\n") -} - -func readAcct(ctr *CgroupControl, name string) (uint64, error) { - p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) - return readFileAsUint64(p) -} - -func readAcctList(ctr *CgroupControl, name string) ([]uint64, error) { - p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) - data, err := os.ReadFile(p) - if err != nil { - return nil, err - } - r := []uint64{} - for _, s := range strings.Split(string(data), " ") { - s = cleanString(s) - if s == "" { - break - } - v, err := strconv.ParseUint(s, 10, 64) - if err != nil { - return nil, fmt.Errorf("parsing %s: %w", s, err) - } - r = append(r, v) - } - return r, nil -} - -// GetSystemCPUUsage returns the system usage for all the cgroups -func GetSystemCPUUsage() (uint64, error) { - cgroupv2, err := IsCgroup2UnifiedMode() - if err != nil { - return 0, err - } - if !cgroupv2 { - p := filepath.Join(cgroupRoot, CPUAcct, "cpuacct.usage") - return readFileAsUint64(p) - } - - files, err := os.ReadDir(cgroupRoot) - if err != nil { - return 0, err - } - var total uint64 - for _, file := range files { - if !file.IsDir() { - continue - } - p := filepath.Join(cgroupRoot, file.Name(), "cpu.stat") - - values, err := readCgroupMapPath(p) - if err != nil { - return 0, err - } - - if val, found := values["usage_usec"]; found { - v, err := strconv.ParseUint(cleanString(val[0]), 10, 64) - if err != nil { - return 0, err - } - total += v * 1000 - } - } - return total, nil -} - -func cpusetCopyFileFromParent(dir, file string, cgroupv2 bool) ([]byte, error) { - if dir == cgroupRoot { - return nil, fmt.Errorf("could not find parent to initialize cpuset %s", file) - } - path := filepath.Join(dir, file) - parentPath := path - if cgroupv2 { - parentPath = fmt.Sprintf("%s.effective", parentPath) - } - data, err := os.ReadFile(parentPath) - if err != nil { - // if the file doesn't exist, it is likely that the cpuset controller - // is not enabled in the kernel. - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - if strings.Trim(string(data), "\n") != "" { - return data, nil - } - data, err = cpusetCopyFileFromParent(filepath.Dir(dir), file, cgroupv2) - if err != nil { - return nil, err - } - if err := os.WriteFile(path, data, 0o644); err != nil { - return nil, fmt.Errorf("write %s: %w", path, err) - } - return data, nil -} - -func cpusetCopyFromParent(path string, cgroupv2 bool) error { - for _, file := range []string{"cpuset.cpus", "cpuset.mems"} { - if _, err := cpusetCopyFileFromParent(path, file, cgroupv2); err != nil { - return err - } - } - return nil -} - -// createCgroupv2Path creates the cgroupv2 path and enables all the available controllers -func createCgroupv2Path(path string) (deferredError error) { - if !strings.HasPrefix(path, cgroupRoot+"/") { - return fmt.Errorf("invalid cgroup path %s", path) - } - content, err := os.ReadFile(cgroupRoot + "/cgroup.controllers") - if err != nil { - return err - } - ctrs := bytes.Fields(content) - res := append([]byte("+"), bytes.Join(ctrs, []byte(" +"))...) - - current := "/sys/fs" - elements := strings.Split(path, "/") - for i, e := range elements[3:] { - current = filepath.Join(current, e) - if i > 0 { - if err := os.Mkdir(current, 0o755); err != nil { - if !os.IsExist(err) { - return err - } - } else { - // If the directory was created, be sure it is not left around on errors. - defer func() { - if deferredError != nil { - os.Remove(current) - } - }() - } - } - // We enable the controllers for all the path components except the last one. It is not allowed to add - // PIDs if there are already enabled controllers. - if i < len(elements[3:])-1 { - if err := os.WriteFile(filepath.Join(current, "cgroup.subtree_control"), res, 0o755); err != nil { - return err - } - } - } - return nil -} - -func (c *CgroupControl) createCgroupDirectory(controller string) (bool, error) { - cPath := c.getCgroupv1Path(controller) - _, err := os.Stat(cPath) - if err == nil { - return false, nil - } - - if !errors.Is(err, os.ErrNotExist) { - return false, err - } - - if err := os.MkdirAll(cPath, 0o755); err != nil { - return false, fmt.Errorf("creating cgroup for %s: %w", controller, err) - } - return true, nil -}