From 7e8ec1e2769520588034c2841a70ed92653f38bd Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 24 Mar 2026 15:51:54 +0300 Subject: [PATCH] refactor(hooks): discover kube feature gates via metrics Signed-off-by: Yaroslav Borbat --- .../hook.go | 128 +++++------ .../hook_test.go | 198 ++++++++++++++++++ .../pkg/kubeapi/kubeapi.go | 15 +- openapi/values.yaml | 3 - .../virtualization-controller/_helpers.tpl | 2 - templates/virtualization-dra/_helper.tpl | 6 +- tools/kubeconform/fixtures/module-values.yaml | 7 + 7 files changed, 273 insertions(+), 86 deletions(-) create mode 100644 images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook_test.go diff --git a/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go index 6540d6de65..4aaa571b47 100644 --- a/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go +++ b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go @@ -20,123 +20,99 @@ import ( "context" "fmt" "hooks/pkg/settings" - "slices" "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" + "k8s.io/client-go/kubernetes" "github.com/deckhouse/module-sdk/pkg" "github.com/deckhouse/module-sdk/pkg/registry" ) const ( - snapshotKubeAPIServerPod = "kube-apiserver-pod" + featureGatesPath = "virtualization.internal.kubeAPIServerFeatureGates" - featureGatesPath = "virtualization.internal.kubeAPIServerFeatureGates" - draFeatureGatesPath = "virtualization.internal.hasDraFeatureGates" + metricPrefix = "kubernetes_feature_enabled{" ) -var _ = registry.RegisterFunc(config, Reconcile) +var _ = registry.RegisterFunc(config, reconcile) var config = &pkg.HookConfig{ OnBeforeHelm: &pkg.OrderedConfig{Order: 5}, - Kubernetes: []pkg.KubernetesConfig{ - { - Name: snapshotKubeAPIServerPod, - APIVersion: "v1", - Kind: "Pod", - NamespaceSelector: &pkg.NamespaceSelector{ - NameSelector: &pkg.NameSelector{ - MatchNames: []string{"kube-system"}, - }, - }, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"component": "kube-apiserver"}, - }, - JqFilter: `{ - "name": .metadata.name, - "command": (.spec.containers[0].command // []), - "args": (.spec.containers[0].args // []) - }`, - ExecuteHookOnSynchronization: ptr.To(false), - }, - }, - Queue: fmt.Sprintf("modules/%s", settings.ModuleName), + Queue: fmt.Sprintf("modules/%s", settings.ModuleName), } -func Reconcile(_ context.Context, input *pkg.HookInput) error { - featureGates, err := discoverFeatureGates(input) +func reconcile(ctx context.Context, input *pkg.HookInput) error { + metricsData, err := fetchMetrics(ctx, input.DC) if err != nil { - return fmt.Errorf("failed to discover feature gates: %w", err) + return fmt.Errorf("failed to fetch kube-apiserver metrics: %w", err) } - input.Values.Set(featureGatesPath, featureGates) + featureGates := parseEnabledFeatureGates(metricsData) - if slices.Contains(featureGates, "DRADeviceBindingConditions") && - slices.Contains(featureGates, "DRAResourceClaimDeviceStatus") && - slices.Contains(featureGates, "DRAConsumableCapacity") { - input.Values.Set(draFeatureGatesPath, "true") - } + input.Values.Set(featureGatesPath, featureGates) return nil } -// discoverFeatureGates extracts enabled feature gates from kube-apiserver pod command/args. -// Returns a list of enabled feature gate names (those set to "true"). -func discoverFeatureGates(input *pkg.HookInput) ([]string, error) { - pods := input.Snapshots.Get(snapshotKubeAPIServerPod) - if len(pods) == 0 { - return nil, fmt.Errorf("no kube-apiserver pods found") +func fetchMetrics(ctx context.Context, dc pkg.DependencyContainer) ([]byte, error) { + cfg, err := dc.GetClientConfig() + if err != nil { + return nil, fmt.Errorf("get client config: %w", err) } - // Use the first pod - all kube-apiserver pods should have the same feature gates - pod := pods[0] + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("create kubernetes clientset: %w", err) + } - var podInfo struct { - Command []string `json:"command"` - Args []string `json:"args"` + result := clientset.RESTClient().Get().AbsPath("/metrics").Do(ctx) + if err := result.Error(); err != nil { + return nil, fmt.Errorf("request /metrics: %w", err) } - err := pod.UnmarshalTo(&podInfo) + raw, err := result.Raw() if err != nil { - return nil, fmt.Errorf("failed to unmarshal kube-apiserver pod: %w", err) + return nil, fmt.Errorf("read metrics response: %w", err) } - allArgs := make([]string, 0, len(podInfo.Command)+len(podInfo.Args)) - allArgs = append(allArgs, podInfo.Command...) - allArgs = append(allArgs, podInfo.Args...) + return raw, nil +} - var enabledGates []string +// parseEnabledFeatureGates extracts feature gate names from Prometheus metrics +// where kubernetes_feature_enabled gauge value equals 1. +func parseEnabledFeatureGates(data []byte) []string { + var enabled []string - for _, arg := range allArgs { - if !strings.HasPrefix(arg, "--feature-gates=") { + for line := range strings.SplitSeq(string(data), "\n") { + if !strings.HasPrefix(line, metricPrefix) { continue } - // Parse feature-gates value: "Gate1=true,Gate2=false,Gate3=true" - gatesStr := strings.TrimPrefix(arg, "--feature-gates=") - gates := strings.SplitSeq(gatesStr, ",") + if !strings.HasSuffix(strings.TrimSpace(line), " 1") { + continue + } - for gate := range gates { - gate = strings.TrimSpace(gate) - if gate == "" { - continue - } + name := extractLabel(line, "name") + if name != "" { + enabled = append(enabled, name) + } + } - parts := strings.SplitN(gate, "=", 2) - if len(parts) != 2 { - continue - } + return enabled +} - gateName := parts[0] - gateValue := strings.ToLower(parts[1]) +func extractLabel(metric, label string) string { + key := label + `="` + idx := strings.Index(metric, key) + if idx < 0 { + return "" + } - if gateValue == "true" { - enabledGates = append(enabledGates, gateName) - } - } + start := idx + len(key) + end := strings.Index(metric[start:], `"`) + if end < 0 { + return "" } - return enabledGates, nil + return metric[start : start+end] } diff --git a/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook_test.go b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook_test.go new file mode 100644 index 0000000000..ebb3dc4590 --- /dev/null +++ b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook_test.go @@ -0,0 +1,198 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discover_kube_apiserver_feature_gates + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/rest" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/module-sdk/pkg" + "github.com/deckhouse/module-sdk/testing/mock" +) + +func TestDiscoverKubeAPIServerFeatureGates(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DiscoverKubeAPIServerFeatureGates Suite") +} + +func metricsLine(name, stage string, value int) string { + return fmt.Sprintf(`kubernetes_feature_enabled{name="%s",stage="%s"} %d`, name, stage, value) +} + +var _ = Describe("parseEnabledFeatureGates", func() { + It("should return enabled feature gates only", func() { + data := joinLines( + metricsLine("FeatureA", "BETA", 1), + metricsLine("FeatureB", "ALPHA", 0), + metricsLine("FeatureC", "", 1), + metricsLine("FeatureD", "DEPRECATED", 0), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(ConsistOf("FeatureA", "FeatureC")) + }) + + It("should skip comment and type lines", func() { + data := joinLines( + "# HELP kubernetes_feature_enabled [BETA] This metric records the data about the stage and enablement of a k8s feature.", + "# TYPE kubernetes_feature_enabled gauge", + metricsLine("FeatureA", "BETA", 1), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(ConsistOf("FeatureA")) + }) + + It("should return nil for empty input", func() { + result := parseEnabledFeatureGates([]byte("")) + Expect(result).To(BeNil()) + }) + + It("should return nil when no feature gate metrics present", func() { + data := joinLines( + "# HELP apiserver_request_total", + "# TYPE apiserver_request_total counter", + `apiserver_request_total{verb="GET"} 42`, + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(BeNil()) + }) + + It("should return nil when all feature gates are disabled", func() { + data := joinLines( + metricsLine("FeatureA", "ALPHA", 0), + metricsLine("FeatureB", "ALPHA", 0), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(BeNil()) + }) + + It("should handle mixed metrics output", func() { + data := joinLines( + `apiserver_request_total{verb="GET"} 100`, + metricsLine("DRADeviceBindingConditions", "BETA", 1), + `apiserver_request_duration_seconds_bucket{le="0.1"} 50`, + metricsLine("DRAConsumableCapacity", "BETA", 1), + metricsLine("SomeDisabledFeature", "ALPHA", 0), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(ConsistOf("DRADeviceBindingConditions", "DRAConsumableCapacity")) + }) +}) + +var _ = Describe("extractLabel", func() { + It("should extract name label", func() { + line := `kubernetes_feature_enabled{name="FeatureA",stage="BETA"} 1` + Expect(extractLabel(line, "name")).To(Equal("FeatureA")) + }) + + It("should extract stage label", func() { + line := `kubernetes_feature_enabled{name="FeatureA",stage="BETA"} 1` + Expect(extractLabel(line, "stage")).To(Equal("BETA")) + }) + + It("should return empty for missing label", func() { + line := `kubernetes_feature_enabled{name="FeatureA"} 1` + Expect(extractLabel(line, "stage")).To(BeEmpty()) + }) + + It("should handle empty label value", func() { + line := `kubernetes_feature_enabled{name="FeatureA",stage=""} 1` + Expect(extractLabel(line, "stage")).To(BeEmpty()) + }) +}) + +var _ = Describe("reconcile", func() { + var ( + dc *mock.DependencyContainerMock + values *mock.OutputPatchableValuesCollectorMock + server *httptest.Server + ) + + newInput := func() *pkg.HookInput { + return &pkg.HookInput{ + Values: values, + DC: dc, + Logger: log.NewNop(), + } + } + + setupServer := func(metricsBody string) { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, metricsBody) + })) + + dc.GetClientConfigMock.Return(&rest.Config{Host: server.URL}, nil) + } + + BeforeEach(func() { + dc = mock.NewDependencyContainerMock(GinkgoT()) + values = mock.NewPatchableValuesCollectorMock(GinkgoT()) + }) + + AfterEach(func() { + if server != nil { + server.Close() + } + }) + + It("should set feature gates from metrics", func() { + setupServer(joinLines( + metricsLine("FeatureA", "BETA", 1), + metricsLine("FeatureB", "ALPHA", 0), + metricsLine("FeatureC", "", 1), + )) + + setValues := make(map[string]any) + values.SetMock.Set(func(path string, v any) { + setValues[path] = v + }) + + Expect(reconcile(context.Background(), newInput())).To(Succeed()) + + Expect(setValues).To(HaveKeyWithValue(featureGatesPath, ConsistOf("FeatureA", "FeatureC"))) + }) + + It("should return error when client config fails", func() { + dc.GetClientConfigMock.Return(nil, fmt.Errorf("no kubeconfig")) + + err := reconcile(context.Background(), newInput()) + Expect(err).To(MatchError(ContainSubstring("no kubeconfig"))) + }) +}) + +func joinLines(lines ...string) string { + result := "" + for i, line := range lines { + if i > 0 { + result += "\n" + } + result += line + } + return result +} diff --git a/images/virtualization-artifact/pkg/kubeapi/kubeapi.go b/images/virtualization-artifact/pkg/kubeapi/kubeapi.go index a4c47d29f4..40eb1505d7 100644 --- a/images/virtualization-artifact/pkg/kubeapi/kubeapi.go +++ b/images/virtualization-artifact/pkg/kubeapi/kubeapi.go @@ -19,6 +19,7 @@ package kubeapi import ( "log/slog" "os" + "strings" "sync" resourcev1 "k8s.io/api/resource/v1" @@ -78,10 +79,16 @@ func isResourceV1Enabled(clientset kubernetes.Interface) (bool, error) { } func HasDRAFeatureGates() bool { - envValue := os.Getenv("HAS_DRA_FEATURE_GATES") - if envValue == "" { - return false + count := 0 + for feature := range strings.SplitSeq(os.Getenv("KUBE_APISERVER_FEATURE_GATES"), ",") { + switch feature { + case "DRAResourceClaimDeviceStatus", "DRADeviceBindingConditions", "DRAConsumableCapacity": + count++ + } } + return count == 3 +} - return envValue == "true" +func HasDRAPartitionableDevices() bool { + return strings.Contains(os.Getenv("KUBE_APISERVER_FEATURE_GATES"), "DRAPartitionableDevices") } diff --git a/openapi/values.yaml b/openapi/values.yaml index c896917cdd..7fb7b2cc7f 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -147,9 +147,6 @@ properties: type: object default: {} additionalProperties: true - hasDraFeatureGates: - type: string - default: "" kubeAPIServerFeatureGates: type: array default: [] diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index f88f153591..94c0345afa 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -112,6 +112,4 @@ true value: {{ .Values.global.clusterConfiguration.serviceSubnetCIDR }} - name: KUBE_APISERVER_FEATURE_GATES value: {{ .Values.virtualization.internal.kubeAPIServerFeatureGates | toJson | quote }} -- name: HAS_DRA_FEATURE_GATES - value: {{ .Values.virtualization.internal.hasDraFeatureGates | quote }} {{- end }} diff --git a/templates/virtualization-dra/_helper.tpl b/templates/virtualization-dra/_helper.tpl index 749beb41c6..3432b1e264 100644 --- a/templates/virtualization-dra/_helper.tpl +++ b/templates/virtualization-dra/_helper.tpl @@ -1,9 +1,13 @@ {{- define "virtualization-dra.isEnabled" -}} {{- if eq (include "hasValidModuleConfig" .) "true" -}} {{- if semverCompare ">=1.34" .Values.global.discovery.kubernetesVersion -}} -{{- if eq "true" .Values.virtualization.internal.hasDraFeatureGates -}} +{{- if has "DRAResourceClaimDeviceStatus" .Values.virtualization.internal.kubeAPIServerFeatureGates -}} +{{- if has "DRADeviceBindingConditions" .Values.virtualization.internal.kubeAPIServerFeatureGates -}} +{{- if has "DRAConsumableCapacity" .Values.virtualization.internal.kubeAPIServerFeatureGates -}} true {{- end -}} {{- end -}} {{- end -}} {{- end -}} +{{- end -}} +{{- end -}} diff --git a/tools/kubeconform/fixtures/module-values.yaml b/tools/kubeconform/fixtures/module-values.yaml index 5b5454d09c..5c49d78f2f 100644 --- a/tools/kubeconform/fixtures/module-values.yaml +++ b/tools/kubeconform/fixtures/module-values.yaml @@ -404,6 +404,13 @@ virtualization: parallelOutboundMigrationsPerNode: 10 virtHandler: nodeCount: 1 + kubeAPIServerFeatureGates: + - TopologyAwareHints + - RotateKubeletServerCertificate + - DRADeviceBindingConditions + - DRAResourceClaimDeviceStatus + - DRAConsumableCapacity + - DRAPartitionableDevices logLevel: debug registry: base: some-registry.io/sys/deckhouse-oss/modules