From 794a6d63e1af2179bd750052378bcb086b5c3fa3 Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Tue, 10 Mar 2026 11:56:21 +0100 Subject: [PATCH] add new metadata to CPs Signed-off-by: Matthias Bertschy --- cmd/main.go | 19 ++++++------ configuration/config.json | 4 +-- docs/CONFIGURATION.md | 4 +-- go.mod | 4 +-- go.sum | 10 +++--- pkg/config/config.go | 2 ++ pkg/config/config_test.go | 4 +-- .../v1/container_data.go | 30 ++++++++++++------ .../v1/containerprofile_manager.go | 31 +++++++++++++++++++ .../v1/containerprofile_manager_test.go | 1 + pkg/containerprofilemanager/v1/lifecycle.go | 5 +-- pkg/containerprofilemanager/v1/monitoring.go | 6 ++-- pkg/objectcache/shared_container_data.go | 18 ++++++++++- pkg/objectcache/shared_container_data_test.go | 2 +- 14 files changed, 101 insertions(+), 39 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index c673a814f4..7960ed3dd6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -247,8 +247,17 @@ func main() { } var containerProfileManager containerprofilemanager.ContainerProfileManagerClient + var cloudMetadata *armotypes.CloudMetadata + + if cfg.EnableApplicationProfile || cfg.EnableRuntimeDetection || cfg.EnableMalwareDetection { + cloudMetadata, err = cloudmetadata.GetCloudMetadata(ctx, k8sClient, cfg.NodeName) + if err != nil { + logger.L().Ctx(ctx).Error("error getting cloud metadata", helpers.Error(err)) + } + } + if cfg.EnableApplicationProfile { - containerProfileManager, err = containerprofilemanagerv1.NewContainerProfileManager(ctx, cfg, k8sClient, k8sObjectCache, storageClient, dnsResolver, seccompManager, nil, ruleBindingCache) + containerProfileManager, err = containerprofilemanagerv1.NewContainerProfileManager(ctx, cfg, k8sClient, k8sObjectCache, storageClient, dnsResolver, seccompManager, nil, ruleBindingCache, cloudMetadata) if err != nil { logger.L().Ctx(ctx).Fatal("error creating the container profile manager", helpers.Error(err)) } @@ -260,7 +269,6 @@ func main() { var processTreeManager processtree.ProcessTreeManager var objCache objectcache.ObjectCache var ruleBindingNotify chan rulebinding.RuleBindingNotify - var cloudMetadata *armotypes.CloudMetadata // Create the container process tree containerProcessTree := containerprocesstree.NewContainerProcessTree() @@ -278,13 +286,6 @@ func main() { // Start the process tree manager to activate the exit cleanup manager processTreeManager.Start() - if cfg.EnableRuntimeDetection || cfg.EnableMalwareDetection { - cloudMetadata, err = cloudmetadata.GetCloudMetadata(ctx, k8sClient, cfg.NodeName) - if err != nil { - logger.L().Ctx(ctx).Error("error getting cloud metadata", helpers.Error(err)) - } - } - if cfg.EnableRuntimeDetection { // create exporter exporter := exporters.InitExporters(cfg.Exporters, clusterData.ClusterName, cfg.NodeName, cloudMetadata, clusterUID, armotypes.AlertSourcePlatformK8sAgent) diff --git a/configuration/config.json b/configuration/config.json index 8bc7b92aee..29b6770066 100644 --- a/configuration/config.json +++ b/configuration/config.json @@ -4,8 +4,8 @@ "fullPathTracingEnabled": false, "networkServiceEnabled": false, "relevantCVEServiceEnabled": false, - "maxSniffingTimePerContainer": "6h", - "updateDataPeriod": "1m", + "maxSniffingTimePerContainer": "24h", + "updateDataPeriod": "10m", "initialDelay": "2m", "prometheusExporterEnabled": "false", "runtimeDetectionEnabled": "false", diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index ef2a724704..b4b1d26d31 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -133,8 +133,8 @@ These environment variables are read directly (not through config file): | Key | Type | Default | Description | |-----|------|---------|-------------| | `initialDelay` | duration | `2m` | Delay before starting monitors | -| `maxSniffingTimePerContainer` | duration | - | Max time to monitor a container | -| `updateDataPeriod` | duration | - | How often to update storage | +| `maxSniffingTimePerContainer` | duration | `24h` | Max time to monitor a container | +| `updateDataPeriod` | duration | `10m` | How often to update storage | | `nodeProfileInterval` | duration | `10m` | Node profile update interval | | `networkStreamingInterval` | duration | `2m` | Network streaming batch interval | | `profilesCacheRefreshRate` | duration | `1m` | Profile cache refresh rate | diff --git a/go.mod b/go.mod index d1c94b405b..14a8bd0385 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 github.com/anchore/syft v1.32.0 github.com/aquilax/truncate v1.0.0 - github.com/armosec/armoapi-go v0.0.687 + github.com/armosec/armoapi-go v0.0.688 github.com/armosec/utils-k8s-go v0.0.35 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff/v4 v4.3.0 @@ -33,7 +33,7 @@ require ( github.com/joncrlsn/dque v0.0.0-20241024143830-7723fd131a64 github.com/kubescape/backend v0.0.37 github.com/kubescape/go-logger v0.0.24 - github.com/kubescape/k8s-interface v0.0.202 + github.com/kubescape/k8s-interface v0.0.204 github.com/kubescape/storage v0.0.247 github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf github.com/moby/sys/mountinfo v0.7.2 diff --git a/go.sum b/go.sum index add364f63c..b0b7e58046 100644 --- a/go.sum +++ b/go.sum @@ -761,10 +761,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/armosec/armoapi-go v0.0.682 h1:H/bMUS3ESNrcun16iS9ficCkE1mWyOIkZJXokauuI6U= -github.com/armosec/armoapi-go v0.0.682/go.mod h1:9jAH0g8ZsryhiBDd/aNMX4+n10bGwTx/doWCyyjSxts= -github.com/armosec/armoapi-go v0.0.687 h1:ZnHAgQVyK92KG1UlBEovzOVM3vFaw/nrnV48dmQKTbU= -github.com/armosec/armoapi-go v0.0.687/go.mod h1:9jAH0g8ZsryhiBDd/aNMX4+n10bGwTx/doWCyyjSxts= +github.com/armosec/armoapi-go v0.0.688 h1:YpDXRv2rJL3n45SZ64kvoHkKMeIe+bRfDhSLW6+CUxE= +github.com/armosec/armoapi-go v0.0.688/go.mod h1:9jAH0g8ZsryhiBDd/aNMX4+n10bGwTx/doWCyyjSxts= github.com/armosec/gojay v1.2.17 h1:VSkLBQzD1c2V+FMtlGFKqWXNsdNvIKygTKJI9ysY8eM= github.com/armosec/gojay v1.2.17/go.mod h1:vuvX3DlY0nbVrJ0qCklSS733AWMoQboq3cFyuQW9ybc= github.com/armosec/utils-go v0.0.58 h1:g9RnRkxZAmzTfPe2ruMo2OXSYLwVSegQSkSavOfmaIE= @@ -1503,8 +1501,8 @@ github.com/kubescape/backend v0.0.37 h1:aAMd5M0Ih4h+enD0LdKzVIDXYVFqEuFBkSyjiGto github.com/kubescape/backend v0.0.37/go.mod h1:4TjTNf9GSD2XxrnW6doB3ANSFzFEkXKYZHGFQX0BiKM= github.com/kubescape/go-logger v0.0.24 h1:JRNlblY16Ty7hD6MSYNPvWYDxNzVAufsDDX/sZJayL0= github.com/kubescape/go-logger v0.0.24/go.mod h1:sMPVCr3VpW/e+SeMaXig5kClGvmZbDXN8YktUeNU4nY= -github.com/kubescape/k8s-interface v0.0.202 h1:yu9x+07crFQAgrBatFFU2WuuxMJfHUMHVuCzuHE9Q4M= -github.com/kubescape/k8s-interface v0.0.202/go.mod h1:d4NVhL81bVXe8yEXlkT4ZHrt3iEppEIN39b8N1oXm5s= +github.com/kubescape/k8s-interface v0.0.204 h1:YkphM8aozocUazKpp0H37By/KZjUjnKeoYqP1b7uBWk= +github.com/kubescape/k8s-interface v0.0.204/go.mod h1:d4NVhL81bVXe8yEXlkT4ZHrt3iEppEIN39b8N1oXm5s= github.com/kubescape/storage v0.0.247 h1:Xf0ScExy7oT/NrZz9732tX/9V3/xudtIeHWKlNxXdxc= github.com/kubescape/storage v0.0.247/go.mod h1:huYJIFh7TUAlV0W3+cmOh7KoJnWRcbWtGw0kY9YIrjU= github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf h1:hI0jVwrB6fT4GJWvuUjzObfci1CUknrZdRHfnRVtKM0= diff --git a/pkg/config/config.go b/pkg/config/config.go index 4049cec772..3af844815a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -149,6 +149,7 @@ func LoadConfig(path string) (Config, error) { viper.SetDefault("maxImageSize", 5*1024*1024*1024) viper.SetDefault("maxSBOMSize", 20*1024*1024) viper.SetDefault("maxTsProfileSize", 2*1024*1024) + viper.SetDefault("maxSniffingTimePerContainer", 24*time.Hour) viper.SetDefault("namespaceName", os.Getenv(NamespaceEnvVar)) viper.SetDefault("nodeName", os.Getenv(NodeNameEnvVar)) viper.SetDefault("podName", os.Getenv(PodNameEnvVar)) @@ -175,6 +176,7 @@ func LoadConfig(path string) (Config, error) { viper.SetDefault("exitCleanup::maxPendingExits", 1000) viper.SetDefault("exitCleanup::cleanupInterval", 30*time.Second) viper.SetDefault("exitCleanup::cleanupDelay", 5*time.Minute) + viper.SetDefault("updateDataPeriod", 10*time.Minute) viper.SetDefault("workerChannelSize", 750000) viper.SetDefault("blockEvents", false) viper.SetDefault("celConfigCache::maxSize", 100000) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index c2afc72450..8acb04cf8b 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -44,8 +44,8 @@ func TestLoadConfig(t *testing.T) { KubernetesMode: true, NetworkStreamingInterval: 2 * time.Minute, InitialDelay: 2 * time.Minute, - MaxSniffingTime: 6 * time.Hour, - UpdateDataPeriod: 1 * time.Minute, + MaxSniffingTime: 24 * time.Hour, + UpdateDataPeriod: 10 * time.Minute, NodeProfileInterval: 1 * time.Minute, MaxDelaySeconds: 30, MaxJitterPercentage: 5, diff --git a/pkg/containerprofilemanager/v1/container_data.go b/pkg/containerprofilemanager/v1/container_data.go index 79b727ff11..9ddb1ed555 100644 --- a/pkg/containerprofilemanager/v1/container_data.go +++ b/pkg/containerprofilemanager/v1/container_data.go @@ -25,20 +25,30 @@ func (cd *containerData) emptyEvents() { cd.rulePolicies = nil cd.callStacks = nil cd.networks = nil - cd.lastReportedCompletion = string(cd.watchedContainerData.GetCompletionStatus()) - cd.lastReportedStatus = string(cd.watchedContainerData.GetStatus()) + if cd.watchedContainerData != nil { + cd.lastReportedCompletion = string(cd.watchedContainerData.GetCompletionStatus()) + cd.lastReportedStatus = string(cd.watchedContainerData.GetStatus()) + } } // isEmpty returns true if the container data is empty func (cd *containerData) isEmpty() bool { - return cd.capabilites == nil && - cd.endpoints == nil && - cd.execs == nil && - cd.opens == nil && - cd.rulePolicies == nil && - cd.callStacks == nil && - cd.networks == nil && - cd.lastReportedCompletion == string(cd.watchedContainerData.GetCompletionStatus()) && + if cd.capabilites != nil || + cd.syscalls != nil || + cd.endpoints != nil || + cd.execs != nil || + cd.opens != nil || + cd.rulePolicies != nil || + cd.callStacks != nil || + cd.networks != nil { + return false + } + + if cd.watchedContainerData == nil { + return true + } + + return cd.lastReportedCompletion == string(cd.watchedContainerData.GetCompletionStatus()) && cd.lastReportedStatus == string(cd.watchedContainerData.GetStatus()) } diff --git a/pkg/containerprofilemanager/v1/containerprofile_manager.go b/pkg/containerprofilemanager/v1/containerprofile_manager.go index 484ff4f9c9..f8455d61c1 100644 --- a/pkg/containerprofilemanager/v1/containerprofile_manager.go +++ b/pkg/containerprofilemanager/v1/containerprofile_manager.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/armosec/armoapi-go/armotypes" mapset "github.com/deckarep/golang-set/v2" "github.com/goradd/maps" containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" @@ -76,6 +77,9 @@ type ContainerProfileManager struct { ruleBindingCache rulebindingmanager.RuleBindingCache queueData *queue.QueueData + // Cloud metadata for annotation population + cloudMetadata *armotypes.CloudMetadata + // Container storage with embedded locking containers map[string]*ContainerEntry containersMu sync.RWMutex @@ -83,6 +87,11 @@ type ContainerProfileManager struct { // Notification channels for container end of life maxSniffTimeNotificationChan []chan *containercollection.Container notificationMu sync.RWMutex + + // Host profile support + hostProfile *v1beta1.ContainerProfile + hostProfileMu sync.RWMutex + hostID string } // NewContainerProfileManager creates a new container profile manager @@ -96,6 +105,7 @@ func NewContainerProfileManager( seccompManager seccompmanager.SeccompManagerClient, enricher containerprofilemanager.Enricher, ruleBindingCache rulebindingmanager.RuleBindingCache, + cloudMetadata *armotypes.CloudMetadata, ) (*ContainerProfileManager, error) { containerProfileManager := &ContainerProfileManager{ ctx: ctx, @@ -109,6 +119,7 @@ func NewContainerProfileManager( ruleBindingCache: ruleBindingCache, containers: make(map[string]*ContainerEntry), maxSniffTimeNotificationChan: make([]chan *containercollection.Container, 0), + cloudMetadata: cloudMetadata, } // Initialize queue @@ -152,5 +163,25 @@ func NewContainerProfileManager( return containerProfileManager, nil } +// Stop stops the container profile manager +func (cpm *ContainerProfileManager) Close() { + // Stop all container timers and clear container map + cpm.containersMu.Lock() + for containerID, entry := range cpm.containers { + entry.mu.Lock() + if entry.data != nil && entry.data.timer != nil { + entry.data.timer.Stop() + entry.data.timer = nil + } + entry.mu.Unlock() + delete(cpm.containers, containerID) + } + cpm.containersMu.Unlock() + + if cpm.queueData != nil { + _ = cpm.queueData.Close() + } +} + var _ containerprofilemanager.ContainerProfileManagerClient = (*ContainerProfileManager)(nil) var _ queue.ErrorCallback = (*ContainerProfileManager)(nil) diff --git a/pkg/containerprofilemanager/v1/containerprofile_manager_test.go b/pkg/containerprofilemanager/v1/containerprofile_manager_test.go index dbdcc8c704..5e7dda1696 100644 --- a/pkg/containerprofilemanager/v1/containerprofile_manager_test.go +++ b/pkg/containerprofilemanager/v1/containerprofile_manager_test.go @@ -517,6 +517,7 @@ func TestContainerProfileManagerCreation(t *testing.T) { seccompManagerMock, nil, nil, + nil, ) assert.NoError(t, err) diff --git a/pkg/containerprofilemanager/v1/lifecycle.go b/pkg/containerprofilemanager/v1/lifecycle.go index 68e0682e10..8e40fd8702 100644 --- a/pkg/containerprofilemanager/v1/lifecycle.go +++ b/pkg/containerprofilemanager/v1/lifecycle.go @@ -18,7 +18,8 @@ func (cpm *ContainerProfileManager) ContainerCallback(notif containercollection. switch notif.Type { case containercollection.EventTypeAddContainer: if utils.IsHostContainer(notif.Container) { - return + logger.L().Debug("adding host container to the container profile manager", + helpers.String("containerID", notif.Container.Runtime.ContainerID)) } if cpm.cfg.IgnoreContainer(notif.Container.K8s.Namespace, notif.Container.K8s.PodName, notif.Container.K8s.PodLabels) { return @@ -235,8 +236,8 @@ func (cpm *ContainerProfileManager) handleContainerMaxTime(container *containerc } if err == nil { - cpm.deleteContainer(container) cpm.notifyContainerEndOfLife(container) + cpm.deleteContainer(container) } } diff --git a/pkg/containerprofilemanager/v1/monitoring.go b/pkg/containerprofilemanager/v1/monitoring.go index 77dd828c2c..4832327db1 100644 --- a/pkg/containerprofilemanager/v1/monitoring.go +++ b/pkg/containerprofilemanager/v1/monitoring.go @@ -24,7 +24,9 @@ func (cpm *ContainerProfileManager) monitorContainer(container *containercollect // Adjust ticker after first tick for faster initial updates if !watchedContainer.InitialDelayExpired { watchedContainer.InitialDelayExpired = true - watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(cpm.cfg.UpdateDataPeriod, cpm.cfg.MaxJitterPercentage)) + if cpm.cfg.UpdateDataPeriod > 0 { + watchedContainer.UpdateDataTicker.Reset(utils.AddJitter(cpm.cfg.UpdateDataPeriod, cpm.cfg.MaxJitterPercentage)) + } } watchedContainer.SetStatus(objectcache.WatchedContainerStatusReady) @@ -166,7 +168,7 @@ func (cpm *ContainerProfileManager) saveContainerProfile(watchedContainer *objec helpersv1.PreviousReportTimestampMetadataKey: watchedContainer.PreviousReportTimestamp.String(), helpersv1.ReportTimestampMetadataKey: watchedContainer.CurrentReportTimestamp.String(), }, - Labels: objectcache.GetLabels(watchedContainer, false), + Labels: objectcache.GetLabels(cpm.cloudMetadata, watchedContainer, false), }, Spec: v1beta1.ContainerProfileSpec{ Architectures: []string{runtime.GOARCH}, diff --git a/pkg/objectcache/shared_container_data.go b/pkg/objectcache/shared_container_data.go index c852c608d0..cf0e7d2b4d 100644 --- a/pkg/objectcache/shared_container_data.go +++ b/pkg/objectcache/shared_container_data.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/armosec/armoapi-go/armotypes" "github.com/armosec/utils-k8s-go/wlid" "github.com/google/go-containerregistry/pkg/name" "github.com/kubescape/go-logger" @@ -89,7 +90,7 @@ type ContainerInfo struct { ImageID string } -func GetLabels(watchedContainer *WatchedContainerData, stripContainer bool) map[string]string { +func GetLabels(cloudMetadata *armotypes.CloudMetadata, watchedContainer *WatchedContainerData, stripContainer bool) map[string]string { labels := watchedContainer.InstanceID.GetLabels() for i := range labels { if labels[i] == "" { @@ -116,6 +117,21 @@ func GetLabels(watchedContainer *WatchedContainerData, stripContainer bool) map[ if watchedContainer.ParentResourceVersion != "" { labels[helpersv1.ResourceVersionMetadataKey] = watchedContainer.ParentResourceVersion } + if cloudMetadata != nil { + labels[helpersv1.HostTypeMetadataKey] = string(cloudMetadata.HostType) + if machineID := cloudMetadata.MachineID; machineID != "" { + labels[helpersv1.HostIDMetadataKey] = machineID + } + if clusterName := cloudMetadata.ClusterName; clusterName != "" { + labels[helpersv1.ClusterMetadataKey] = clusterName + } + if accountID := cloudMetadata.AccountID; accountID != "" { + labels[helpersv1.AWSAccountIDMetadataKey] = accountID + } + if region := cloudMetadata.Region; region != "" { + labels[helpersv1.RegionMetadataKey] = region + } + } return labels } diff --git a/pkg/objectcache/shared_container_data_test.go b/pkg/objectcache/shared_container_data_test.go index 20dccdada5..5b287b4ed5 100644 --- a/pkg/objectcache/shared_container_data_test.go +++ b/pkg/objectcache/shared_container_data_test.go @@ -74,7 +74,7 @@ func Test_GetLabels(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := GetLabels(tt.args.watchedContainer, tt.args.stripContainer) + got := GetLabels(nil, tt.args.watchedContainer, tt.args.stripContainer) assert.Equal(t, tt.want, got) }) }