Bump github.com/cilium/cilium from 1.17.2 to 1.17.3

CONTRIBUTING.md

@@ -5,7 +5,7 @@ Below are some basic guidelines.
 
 
 ## Requirements
-* Linux ≥v4.16 (amd64, arm64)
+* Linux ≥v5.1 (amd64, arm64)
 * Go v1.23
 
 

Dockerfile

@@ -6,7 +6,7 @@ COPY go.sum .
 RUN go mod download
 COPY . .
 ARG VERSION=unknown
-RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
+RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
 
 FROM registry.access.redhat.com/ubi9/ubi
 

README.md

@@ -5,7 +5,7 @@
 
 The agent gathers metrics related to a node and the containers running on it, and it exposes them in the Prometheus format.
 
-It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 4.16.
+It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 5.1.
 
 <img src="https://coroot.com/static/img/blog/ebpf.svg" width="800" />
 

cgroup/cgroup.go

@@ -25,6 +25,7 @@ var (
 	lxcIdRegexp         = regexp.MustCompile(`/lxc/([^/]+)`)
 	systemSliceIdRegexp = regexp.MustCompile(`(/(system|runtime|reserved)\.slice/([^/]+))`)
 	talosIdRegexp       = regexp.MustCompile(`/(system|podruntime)/([^/]+)`)
+	lxcPayloadRegexp    = regexp.MustCompile(`/lxc\.payload\.([^/]+)`)
 )
 
 type ContainerType uint8
@@ -129,7 +130,14 @@ func NewFromProcessCgroupFile(filePath string) (*Cgroup, error) {
 			continue
 		}
 		for _, cgType := range strings.Split(parts[1], ",") {
-			p := path.Join(baseCgroupPath, parts[2])
+			cgPath := parts[2]
+			if strings.HasPrefix(parts[2], "/lxc.payload.") {
+				pp := strings.Split(cgPath, "/")
+				if len(parts) > 2 {
+					cgPath = "/" + pp[1]
+				}
+			}
+			p := path.Join(baseCgroupPath, cgPath)
 			switch p {
 			case "/", "/init.scope":
 				continue
@@ -147,24 +155,22 @@ func NewFromProcessCgroupFile(filePath string) (*Cgroup, error) {
 
 func containerByCgroup(cgroupPath string) (ContainerType, string, error) {
 	parts := strings.Split(strings.TrimLeft(cgroupPath, "/"), "/")
-	if cgroupPath == "/init" {
-		return ContainerTypeTalosRuntime, "/talos/init", nil
-	}
-	if len(parts) < 2 {
+	if len(parts) == 0 {
 		return ContainerTypeStandaloneProcess, "", nil
 	}
 	prefix := parts[0]
-	if prefix == "user.slice" || prefix == "init.scope" {
+	switch {
+	case cgroupPath == "/init":
+		return ContainerTypeTalosRuntime, "/talos/init", nil
+	case prefix == "user.slice" || prefix == "init.scope":
 		return ContainerTypeStandaloneProcess, "", nil
-	}
-	if prefix == "docker" || (prefix == "system.slice" && strings.HasPrefix(parts[1], "docker-")) {
+	case prefix == "docker" || (prefix == "system.slice" && len(parts) > 1 && strings.HasPrefix(parts[1], "docker-")):
 		matches := dockerIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid docker cgroup %s", cgroupPath)
 		}
 		return ContainerTypeDocker, matches[1], nil
-	}
-	if strings.Contains(cgroupPath, "kubepods") {
+	case strings.Contains(cgroupPath, "kubepods"):
 		crioMatches := crioIdRegexp.FindStringSubmatch(cgroupPath)
 		if crioMatches != nil {
 			return ContainerTypeCrio, crioMatches[1], nil
@@ -181,27 +187,33 @@ func containerByCgroup(cgroupPath string) (ContainerType, string, error) {
 			return ContainerTypeSandbox, "", nil
 		}
 		return ContainerTypeDocker, matches[1], nil
-	}
-	if prefix == "lxc" {
-		matches := lxcIdRegexp.FindStringSubmatch(cgroupPath)
-		if matches == nil {
-			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc cgroup %s", cgroupPath)
-		}
-		return ContainerTypeLxc, matches[1], nil
-	}
-	if prefix == "system" || prefix == "podruntime" {
+	case prefix == "system" || prefix == "podruntime":
 		matches := talosIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid talos runtime cgroup %s", cgroupPath)
 		}
 		return ContainerTypeTalosRuntime, path.Join("/talos/", matches[2]), nil
-	}
-	if prefix == "system.slice" || prefix == "runtime.slice" || prefix == "reserved.slice" {
+	case prefix == "system.slice" || prefix == "runtime.slice" || prefix == "reserved.slice":
 		matches := systemSliceIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid systemd cgroup %s", cgroupPath)
 		}
 		return ContainerTypeSystemdService, strings.Replace(matches[1], "\\x2d", "-", -1), nil
+	case prefix == "lxc":
+		matches := lxcIdRegexp.FindStringSubmatch(cgroupPath)
+		if matches == nil {
+			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc cgroup %s", cgroupPath)
+		}
+		return ContainerTypeLxc, matches[1], nil
+	case strings.HasPrefix(prefix, "lxc.payload."):
+		matches := lxcPayloadRegexp.FindStringSubmatch(cgroupPath)
+		if matches == nil {
+			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc payload cgroup %s", cgroupPath)
+		}
+		return ContainerTypeLxc, "/lxc/" + matches[1], nil
+	case len(parts) < 2:
+		return ContainerTypeStandaloneProcess, "", nil
 	}
+
 	return ContainerTypeUnknown, "", fmt.Errorf("unknown container: %s", cgroupPath)
 }

cgroup/cgroup_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestNewFromProcessCgroupFile(t *testing.T) {
@@ -75,6 +76,12 @@ func TestNewFromProcessCgroupFile(t *testing.T) {
 	assert.Equal(t, "95cbe853416f52d927dec41f1406dd75015ea131244a1ca875a7cd4ebe927ac8", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
 
+	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/3000/cgroup"))
+	require.Nil(t, err)
+	assert.Equal(t, "/lxc.payload.first", cg.Id)
+	assert.Equal(t, "/lxc/first", cg.ContainerId)
+	assert.Equal(t, ContainerTypeLxc, cg.ContainerType)
+
 	baseCgroupPath = "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podc83d0428_58af_41eb_8dba_b9e6eddffe7b.slice/docker-0e612005fd07e7f47e2cd07df99a2b4e909446814d71d0b5e4efc7159dd51252.scope"
 	defer func() {
 		baseCgroupPath = ""
@@ -178,4 +185,14 @@ func TestContainerByCgroup(t *testing.T) {
 	as.Equal(typ, ContainerTypeTalosRuntime)
 	as.Equal("/talos/init", id)
 	as.Nil(err)
+
+	typ, id, err = containerByCgroup("/lxc.payload.first")
+	as.Equal(typ, ContainerTypeLxc)
+	as.Equal("/lxc/first", id)
+	as.Nil(err)
+
+	typ, id, err = containerByCgroup("/lxc.monitor.first")
+	as.Equal(ContainerTypeStandaloneProcess, typ)
+	as.Equal("", id)
+	as.Nil(err)
 }

cgroup/fixtures/proc/3000/cgroup

@@ -0,0 +1 @@
+0::/lxc.payload.first/system.slice/systemd-logind.service

containers/app.go

@@ -120,6 +120,8 @@ func guessApplicationTypeByCmdline(cmdline []byte) string {
 		return "nats"
 	case bytes.HasSuffix(cmd, []byte("java")):
 		return "java"
+	case bytes.HasSuffix(cmd, []byte("ollama")):
+		return "ollama"
 	case bytes.Contains(cmd, []byte("victoria-metrics")) ||
 		bytes.Contains(cmd, []byte("vmstorage")) ||
 		bytes.Contains(cmd, []byte("vminsert")) ||

containers/container.go

@@ -29,6 +29,7 @@ var (
 	gcInterval                = 10 * time.Minute
 	pingTimeout               = 300 * time.Millisecond
 	multilineCollectorTimeout = time.Second
+	gpuStatsWindow            = 15 * time.Second
 )
 
 type ContainerID string
@@ -132,6 +133,8 @@ type Container struct {
 	l7Stats  L7Stats
 	dnsStats *L7Metrics
 
+	gpuStats map[string]*GpuUsage
+
 	oomKills                 int
 	pythonThreadLockWaitTime time.Duration
 
@@ -181,6 +184,8 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, pid
 		l7Stats:                  L7Stats{},
 		dnsStats:                 &L7Metrics{},
 
+		gpuStats: map[string]*GpuUsage{},
+
 		mounts:     map[string]proc.MountInfo{},
 		seenMounts: map[uint64]struct{}{},
 
@@ -370,7 +375,27 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 				process.dotNetMonitor.Collect(ch)
 			}
 		}
+
+		for _, usage := range c.gpuStats {
+			usage.Reset()
+		}
+		if usage := process.getGPUUsage(); usage != nil {
+			for uuid, u := range usage {
+				tu := c.gpuStats[uuid]
+				if tu == nil {
+					tu = &GpuUsage{}
+					c.gpuStats[uuid] = tu
+				}
+				tu.GPU += u.GPU
+				tu.Memory += u.Memory
+			}
+		}
 	}
+	for uuid, usage := range c.gpuStats {
+		ch <- gauge(metrics.GpuUsagePercent, usage.GPU, uuid)
+		ch <- gauge(metrics.GpuMemoryUsagePercent, usage.Memory, uuid)
+	}
+
 	for appType := range appTypes {
 		ch <- gauge(metrics.ApplicationType, 1, appType)
 	}

containers/metrics.go

@@ -51,6 +51,9 @@ var metrics = struct {
 
 	PythonThreadLockWaitTime *prometheus.Desc
 
+	GpuUsagePercent       *prometheus.Desc
+	GpuMemoryUsagePercent *prometheus.Desc
+
 	Ip2Fqdn *prometheus.Desc
 }{
 	ContainerInfo: metric("container_info", "Meta information about the container", "image", "systemd_triggered_by"),
@@ -100,6 +103,9 @@ var metrics = struct {
 	Ip2Fqdn: metric("ip_to_fqdn", "Mapping IP addresses to FQDNs based on DNS requests initiated by containers", "ip", "fqdn"),
 
 	PythonThreadLockWaitTime: metric("container_python_thread_lock_wait_time_seconds", "Time spent waiting acquiring GIL in seconds"),
+
+	GpuUsagePercent:       metric("container_resources_gpu_usage_percent", "Percent of GPU compute resources used by the container", "gpu_uuid"),
+	GpuMemoryUsagePercent: metric("container_resources_gpu_memory_usage_percent", "Percent of GPU memory used by the container", "gpu_uuid"),
 }
 
 var (

containers/process.go

@@ -8,11 +8,22 @@ import (
 
 	"github.com/cilium/ebpf/link"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
+	"github.com/coroot/coroot-node-agent/gpu"
 	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/jpillora/backoff"
 	"github.com/mdlayher/taskstats"
 )
 
+type GpuUsage struct {
+	GPU    float64
+	Memory float64
+}
+
+func (gu *GpuUsage) Reset() {
+	gu.Memory = 0
+	gu.GPU = 0
+}
+
 type Process struct {
 	Pid       uint32
 	StartedAt time.Time
@@ -29,6 +40,8 @@ type Process struct {
 	goTlsUprobesChecked   bool
 	openSslUprobesChecked bool
 	pythonGilChecked      bool
+
+	gpuUsageSamples []gpu.ProcessUsageSample
 }
 
 func NewProcess(pid uint32, stats *taskstats.Stats, tracer *ebpftracer.Tracer) *Process {
@@ -97,6 +110,43 @@ func (p *Process) instrumentPython(cmdline []byte, tracer *ebpftracer.Tracer) {
 	p.uprobes = append(p.uprobes, tracer.AttachPythonThreadLockProbes(p.Pid)...)
 }
 
+func (p *Process) addGpuUsageSample(sample gpu.ProcessUsageSample) {
+	p.removeOldGpuUsageSamples(sample.Timestamp.Add(-gpuStatsWindow))
+	p.gpuUsageSamples = append(p.gpuUsageSamples, sample)
+}
+
+func (p *Process) getGPUUsage() map[string]*GpuUsage {
+	p.removeOldGpuUsageSamples(time.Now().Add(-gpuStatsWindow))
+	if len(p.gpuUsageSamples) == 0 {
+		return nil
+	}
+	gpuStatsWindowSeconds := gpuStatsWindow.Seconds()
+	res := make(map[string]*GpuUsage)
+	for _, sample := range p.gpuUsageSamples {
+		u := res[sample.UUID]
+		if u == nil {
+			u = &GpuUsage{}
+			res[sample.UUID] = u
+		}
+		u.GPU += float64(sample.GPUPercent) / gpuStatsWindowSeconds
+		u.Memory += float64(sample.MemoryPercent) / gpuStatsWindowSeconds
+	}
+	return res
+}
+
+func (p *Process) removeOldGpuUsageSamples(cutoff time.Time) {
+	i := 0
+	for ; i < len(p.gpuUsageSamples); i++ {
+		if p.gpuUsageSamples[i].Timestamp.After(cutoff) {
+			break
+		}
+	}
+	if i > 0 {
+		copy(p.gpuUsageSamples, p.gpuUsageSamples[i:])
+		p.gpuUsageSamples = p.gpuUsageSamples[:len(p.gpuUsageSamples)-i]
+	}
+}
+
 func (p *Process) Close() {
 	p.cancelFunc()
 	for _, u := range p.uprobes {

containers/registry.go

@@ -14,6 +14,7 @@ import (
 	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/coroot/coroot-node-agent/flags"
+	"github.com/coroot/coroot-node-agent/gpu"
 	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/vishvananda/netns"
@@ -59,9 +60,11 @@ type Registry struct {
 	trafficStatsLastUpdated time.Time
 	trafficStatsLock        sync.Mutex
 	trafficStatsUpdateCh    chan *TrafficStatsUpdate
+
+	gpuProcessUsageSampleChan chan gpu.ProcessUsageSample
 }
 
-func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo) (*Registry, error) {
+func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, gpuProcessUsageSampleChan chan gpu.ProcessUsageSample) (*Registry, error) {
 	ns, err := proc.GetSelfNetNs()
 	if err != nil {
 		return nil, err
@@ -113,6 +116,8 @@ func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo) (*
 		tracer: ebpftracer.NewTracer(hostNetNs, selfNetNs, *flags.DisableL7Tracing),
 
 		trafficStatsUpdateCh: make(chan *TrafficStatsUpdate),
+
+		gpuProcessUsageSampleChan: gpuProcessUsageSampleChan,
 	}
 	if err = reg.Register(r); err != nil {
 		return nil, err
@@ -205,6 +210,12 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 			if c := r.containersByPid[u.Pid]; c != nil {
 				c.updateTrafficStats(u)
 			}
+		case sample := <-r.gpuProcessUsageSampleChan:
+			if c := r.containersByPid[sample.Pid]; c != nil {
+				if p := c.processes[sample.Pid]; p != nil {
+					p.addGpuUsageSample(sample)
+				}
+			}
 		case e, more := <-ch:
 			if !more {
 				return