From 655816d690ad70a9fb9711f83a86dec2649bedb6 Mon Sep 17 00:00:00 2001 From: tzvonimir Date: Thu, 19 Feb 2026 14:39:35 +0100 Subject: [PATCH] Add pmax treshold check --- gen/api/v1/mpa.pb.go | 79 ++++++++++++------- .../collector/historical_metrics_collector.go | 27 +++++++ internal/proto/mpa.proto | 3 + proto/api/v1/mpa.proto | 3 + 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/gen/api/v1/mpa.pb.go b/gen/api/v1/mpa.pb.go index eb6c0f25..36957213 100644 --- a/gen/api/v1/mpa.pb.go +++ b/gen/api/v1/mpa.pb.go @@ -490,6 +490,9 @@ type ContainerHistoricalMetrics struct { DiskP90 int64 `protobuf:"varint,21,opt,name=disk_p90,json=diskP90,proto3" json:"disk_p90,omitempty"` DiskP95 int64 `protobuf:"varint,22,opt,name=disk_p95,json=diskP95,proto3" json:"disk_p95,omitempty"` DiskP99 int64 `protobuf:"varint,23,opt,name=disk_p99,json=diskP99,proto3" json:"disk_p99,omitempty"` + // Pmax (max observed over 24h window) for spike protection + CpuPmax int64 `protobuf:"varint,24,opt,name=cpu_pmax,json=cpuPmax,proto3" json:"cpu_pmax,omitempty"` // max CPU over 24h window (millicores) + MemPmax int64 `protobuf:"varint,25,opt,name=mem_pmax,json=memPmax,proto3" json:"mem_pmax,omitempty"` // max memory over 24h window (bytes) } func (x *ContainerHistoricalMetrics) Reset() { @@ -685,6 +688,20 @@ func (x *ContainerHistoricalMetrics) GetDiskP99() int64 { return 0 } +func (x *ContainerHistoricalMetrics) GetCpuPmax() int64 { + if x != nil { + return x.CpuPmax + } + return 0 +} + +func (x *ContainerHistoricalMetrics) GetMemPmax() int64 { + if x != nil { + return x.MemPmax + } + return 0 +} + // MpaStreamResponse wraps both real-time and historical data on the same stream type MpaStreamResponse struct { state protoimpl.MessageState @@ -875,7 +892,7 @@ var file_api_v1_mpa_proto_rawDesc = []byte{ 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x45, 0x6e, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xf3, 0x04, 0x0a, 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xa9, 0x05, 0x0a, 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, @@ -914,34 +931,38 @@ var file_api_v1_mpa_proto_rawDesc = []byte{ 0x73, 0x6b, 0x50, 0x39, 0x30, 0x12, 0x19, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x70, 0x39, 0x35, 0x18, 0x16, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x64, 0x69, 0x73, 0x6b, 0x50, 0x39, 0x35, 0x12, 0x19, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x70, 0x39, 0x39, 0x18, 0x17, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x07, 0x64, 0x69, 0x73, 0x6b, 0x50, 0x39, 0x39, 0x22, 0xbd, 0x01, 0x0a, 0x11, - 0x4d, 0x70, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x4a, 0x0a, 0x10, 0x72, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, - 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x48, 0x00, 0x52, 0x0f, 0x72, 0x65, - 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x51, 0x0a, - 0x12, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x75, 0x6d, 0x6d, - 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x76, 0x31, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x48, 0x00, 0x52, 0x11, 0x68, - 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, - 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x32, 0x65, 0x0a, 0x0a, 0x4d, - 0x70, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x57, 0x0a, 0x15, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x70, 0x61, 0x57, - 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x70, 0x61, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, - 0x30, 0x01, 0x42, 0x81, 0x01, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, - 0x31, 0x42, 0x08, 0x4d, 0x70, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x30, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x65, 0x76, 0x7a, 0x65, 0x72, - 0x6f, 0x2d, 0x69, 0x6e, 0x63, 0x2f, 0x7a, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x2f, 0x67, - 0x65, 0x6e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0xa2, - 0x02, 0x03, 0x41, 0x58, 0x58, 0xaa, 0x02, 0x06, 0x41, 0x70, 0x69, 0x2e, 0x56, 0x31, 0xca, 0x02, - 0x06, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x12, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, - 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x07, 0x41, - 0x70, 0x69, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x28, 0x03, 0x52, 0x07, 0x64, 0x69, 0x73, 0x6b, 0x50, 0x39, 0x39, 0x12, 0x19, 0x0a, 0x08, 0x63, + 0x70, 0x75, 0x5f, 0x70, 0x6d, 0x61, 0x78, 0x18, 0x18, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, + 0x70, 0x75, 0x50, 0x6d, 0x61, 0x78, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x65, 0x6d, 0x5f, 0x70, 0x6d, + 0x61, 0x78, 0x18, 0x19, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, 0x65, 0x6d, 0x50, 0x6d, 0x61, + 0x78, 0x22, 0xbd, 0x01, 0x0a, 0x11, 0x4d, 0x70, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x10, 0x72, 0x65, 0x61, 0x6c, 0x74, + 0x69, 0x6d, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, + 0x48, 0x00, 0x52, 0x0f, 0x72, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x12, 0x51, 0x0a, 0x12, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, + 0x6c, 0x5f, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, + 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, + 0x79, 0x48, 0x00, 0x52, 0x11, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x53, + 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x32, 0x65, 0x0a, 0x0a, 0x4d, 0x70, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x57, 0x0a, 0x15, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, + 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x2e, 0x4d, 0x70, 0x61, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x75, 0x62, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x76, 0x31, 0x2e, 0x4d, 0x70, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x81, 0x01, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x42, 0x08, 0x4d, 0x70, 0x61, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x01, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x64, 0x65, 0x76, 0x7a, 0x65, 0x72, 0x6f, 0x2d, 0x69, 0x6e, 0x63, 0x2f, 0x7a, 0x78, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x72, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x3b, + 0x61, 0x70, 0x69, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x41, 0x58, 0x58, 0xaa, 0x02, 0x06, 0x41, 0x70, + 0x69, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x06, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x12, + 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0xea, 0x02, 0x07, 0x41, 0x70, 0x69, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/collector/historical_metrics_collector.go b/internal/collector/historical_metrics_collector.go index b613174a..5d990b21 100644 --- a/internal/collector/historical_metrics_collector.go +++ b/internal/collector/historical_metrics_collector.go @@ -187,6 +187,31 @@ func (c *HistoricalMetricsCollector) fetchContainerPercentiles(ctx context.Conte sampleCount = int32(countVal) } + // Pmax queries: max observed value over 24h for spike protection + var cpuPmax, memPmax int64 + + cpuPmaxQuery := fmt.Sprintf( + `max_over_time(rate(container_cpu_usage_seconds_total{namespace="%s", pod=~"%s", container="%s"}[5m])[24h:%s])`, + workload.Namespace, workload.PodRegex, containerName, historicalStepInterval, + ) + cpuPmaxVal, err := c.queryScalar(ctx, cpuPmaxQuery, now) + if err != nil { + c.logger.V(1).Info("CPU pmax query failed", "error", err) + } else { + cpuPmax = int64(cpuPmaxVal * 1000) // Convert cores to millicores + } + + memPmaxQuery := fmt.Sprintf( + `max_over_time(container_memory_working_set_bytes{namespace="%s", pod=~"%s", container="%s"}[24h])`, + workload.Namespace, workload.PodRegex, containerName, + ) + memPmaxVal, err := c.queryScalar(ctx, memPmaxQuery, now) + if err != nil { + c.logger.V(1).Info("Memory pmax query failed", "error", err) + } else { + memPmax = int64(memPmaxVal) + } + return &gen.ContainerHistoricalMetrics{ ContainerName: containerName, CpuP50: cpuValues[0.50], @@ -199,6 +224,8 @@ func (c *HistoricalMetricsCollector) fetchContainerPercentiles(ctx context.Conte MemP80: memValues[0.80], MemP90: memValues[0.90], MemP99: memValues[0.99], + CpuPmax: cpuPmax, + MemPmax: memPmax, }, sampleCount } diff --git a/internal/proto/mpa.proto b/internal/proto/mpa.proto index 402cd689..d634ec89 100644 --- a/internal/proto/mpa.proto +++ b/internal/proto/mpa.proto @@ -102,6 +102,9 @@ message ContainerHistoricalMetrics { int64 disk_p90 = 21; int64 disk_p95 = 22; int64 disk_p99 = 23; + // Pmax (max observed over 24h window) for spike protection + int64 cpu_pmax = 24; // max CPU over 24h window (millicores) + int64 mem_pmax = 25; // max memory over 24h window (bytes) } // MpaStreamResponse wraps both real-time and historical data on the same stream diff --git a/proto/api/v1/mpa.proto b/proto/api/v1/mpa.proto index 402cd689..d634ec89 100644 --- a/proto/api/v1/mpa.proto +++ b/proto/api/v1/mpa.proto @@ -102,6 +102,9 @@ message ContainerHistoricalMetrics { int64 disk_p90 = 21; int64 disk_p95 = 22; int64 disk_p99 = 23; + // Pmax (max observed over 24h window) for spike protection + int64 cpu_pmax = 24; // max CPU over 24h window (millicores) + int64 mem_pmax = 25; // max memory over 24h window (bytes) } // MpaStreamResponse wraps both real-time and historical data on the same stream