diff --git a/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go index d053670ecf..96341fcbfa 100644 --- a/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go +++ b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook.go @@ -19,11 +19,9 @@ package discover_kube_apiserver_feature_gates import ( "context" "fmt" - "slices" "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" + "k8s.io/client-go/kubernetes" "github.com/deckhouse/module-sdk/pkg" "github.com/deckhouse/module-sdk/pkg/registry" @@ -31,112 +29,90 @@ import ( ) const ( - snapshotKubeAPIServerPod = "kube-apiserver-pod" + featureGatesPath = "virtualization.internal.kubeAPIServerFeatureGates" - featureGatesPath = "virtualization.internal.kubeAPIServerFeatureGates" - draFeatureGatesPath = "virtualization.internal.hasDraFeatureGates" + metricPrefix = "kubernetes_feature_enabled{" ) -var _ = registry.RegisterFunc(config, Reconcile) +var _ = registry.RegisterFunc(config, reconcile) var config = &pkg.HookConfig{ OnBeforeHelm: &pkg.OrderedConfig{Order: 5}, - Kubernetes: []pkg.KubernetesConfig{ - { - Name: snapshotKubeAPIServerPod, - APIVersion: "v1", - Kind: "Pod", - NamespaceSelector: &pkg.NamespaceSelector{ - NameSelector: &pkg.NameSelector{ - MatchNames: []string{"kube-system"}, - }, - }, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"component": "kube-apiserver"}, - }, - JqFilter: `{ - "name": .metadata.name, - "command": (.spec.containers[0].command // []), - "args": (.spec.containers[0].args // []) - }`, - ExecuteHookOnSynchronization: ptr.To(false), - }, - }, - Queue: fmt.Sprintf("modules/%s", settings.ModuleName), + Queue: fmt.Sprintf("modules/%s", settings.ModuleName), } -func Reconcile(_ context.Context, input *pkg.HookInput) error { - featureGates, err := discoverFeatureGates(input) +func reconcile(ctx context.Context, input *pkg.HookInput) error { + metricsData, err := fetchMetrics(ctx, input.DC) if err != nil { - return fmt.Errorf("failed to discover feature gates: %w", err) + return fmt.Errorf("failed to fetch kube-apiserver metrics: %w", err) } - input.Values.Set(featureGatesPath, featureGates) + featureGates := parseEnabledFeatureGates(metricsData) - // DRAResourceClaimDeviceStatus enabled by default - if slices.Contains(featureGates, "DRADeviceBindingConditions") && - slices.Contains(featureGates, "DRAConsumableCapacity") { - input.Values.Set(draFeatureGatesPath, "true") - } + input.Values.Set(featureGatesPath, featureGates) return nil } -// discoverFeatureGates extracts enabled feature gates from kube-apiserver pod command/args. -// Returns a list of enabled feature gate names (those set to "true"). -func discoverFeatureGates(input *pkg.HookInput) ([]string, error) { - pods := input.Snapshots.Get(snapshotKubeAPIServerPod) - if len(pods) == 0 { - return nil, fmt.Errorf("no kube-apiserver pods found") +func fetchMetrics(ctx context.Context, dc pkg.DependencyContainer) ([]byte, error) { + cfg, err := dc.GetClientConfig() + if err != nil { + return nil, fmt.Errorf("get client config: %w", err) } - // Use the first pod - all kube-apiserver pods should have the same feature gates - pod := pods[0] + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("create kubernetes clientset: %w", err) + } - var podInfo struct { - Command []string `json:"command"` - Args []string `json:"args"` + result := clientset.RESTClient().Get().AbsPath("/metrics").Do(ctx) + if err := result.Error(); err != nil { + return nil, fmt.Errorf("request /metrics: %w", err) } - err := pod.UnmarshalTo(&podInfo) + raw, err := result.Raw() if err != nil { - return nil, fmt.Errorf("failed to unmarshal kube-apiserver pod: %w", err) + return nil, fmt.Errorf("read metrics response: %w", err) } - allArgs := make([]string, 0, len(podInfo.Command)+len(podInfo.Args)) - allArgs = append(allArgs, podInfo.Command...) - allArgs = append(allArgs, podInfo.Args...) + return raw, nil +} - var enabledGates []string +// parseEnabledFeatureGates extracts feature gate names from Prometheus metrics +// where kubernetes_feature_enabled gauge value equals 1. +func parseEnabledFeatureGates(data []byte) []string { + var enabled []string - for _, arg := range allArgs { - if !strings.HasPrefix(arg, "--feature-gates=") { + for line := range strings.SplitSeq(string(data), "\n") { + if !strings.HasPrefix(line, metricPrefix) { continue } - // Parse feature-gates value: "Gate1=true,Gate2=false,Gate3=true" - gatesStr := strings.TrimPrefix(arg, "--feature-gates=") - gates := strings.SplitSeq(gatesStr, ",") + if !strings.HasSuffix(strings.TrimSpace(line), " 1") { + continue + } - for gate := range gates { - gate = strings.TrimSpace(gate) - if gate == "" { - continue - } + name := extractLabel(line, "name") + if name != "" { + enabled = append(enabled, name) + } + } - parts := strings.SplitN(gate, "=", 2) - if len(parts) != 2 { - continue - } + return enabled +} - gateName := parts[0] - gateValue := strings.ToLower(parts[1]) +func extractLabel(metric, label string) string { + key := label + `="` + idx := strings.Index(metric, key) + if idx < 0 { + return "" + } - if gateValue == "true" { - enabledGates = append(enabledGates, gateName) - } - } + start := idx + len(key) + end := strings.Index(metric[start:], `"`) + if end < 0 { + return "" } - return enabledGates, nil + return metric[start : start+end] } diff --git a/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook_test.go b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook_test.go new file mode 100644 index 0000000000..ebb3dc4590 --- /dev/null +++ b/images/hooks/pkg/hooks/discover-kube-apiserver-feature-gates/hook_test.go @@ -0,0 +1,198 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discover_kube_apiserver_feature_gates + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/rest" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/module-sdk/pkg" + "github.com/deckhouse/module-sdk/testing/mock" +) + +func TestDiscoverKubeAPIServerFeatureGates(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DiscoverKubeAPIServerFeatureGates Suite") +} + +func metricsLine(name, stage string, value int) string { + return fmt.Sprintf(`kubernetes_feature_enabled{name="%s",stage="%s"} %d`, name, stage, value) +} + +var _ = Describe("parseEnabledFeatureGates", func() { + It("should return enabled feature gates only", func() { + data := joinLines( + metricsLine("FeatureA", "BETA", 1), + metricsLine("FeatureB", "ALPHA", 0), + metricsLine("FeatureC", "", 1), + metricsLine("FeatureD", "DEPRECATED", 0), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(ConsistOf("FeatureA", "FeatureC")) + }) + + It("should skip comment and type lines", func() { + data := joinLines( + "# HELP kubernetes_feature_enabled [BETA] This metric records the data about the stage and enablement of a k8s feature.", + "# TYPE kubernetes_feature_enabled gauge", + metricsLine("FeatureA", "BETA", 1), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(ConsistOf("FeatureA")) + }) + + It("should return nil for empty input", func() { + result := parseEnabledFeatureGates([]byte("")) + Expect(result).To(BeNil()) + }) + + It("should return nil when no feature gate metrics present", func() { + data := joinLines( + "# HELP apiserver_request_total", + "# TYPE apiserver_request_total counter", + `apiserver_request_total{verb="GET"} 42`, + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(BeNil()) + }) + + It("should return nil when all feature gates are disabled", func() { + data := joinLines( + metricsLine("FeatureA", "ALPHA", 0), + metricsLine("FeatureB", "ALPHA", 0), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(BeNil()) + }) + + It("should handle mixed metrics output", func() { + data := joinLines( + `apiserver_request_total{verb="GET"} 100`, + metricsLine("DRADeviceBindingConditions", "BETA", 1), + `apiserver_request_duration_seconds_bucket{le="0.1"} 50`, + metricsLine("DRAConsumableCapacity", "BETA", 1), + metricsLine("SomeDisabledFeature", "ALPHA", 0), + ) + + result := parseEnabledFeatureGates([]byte(data)) + Expect(result).To(ConsistOf("DRADeviceBindingConditions", "DRAConsumableCapacity")) + }) +}) + +var _ = Describe("extractLabel", func() { + It("should extract name label", func() { + line := `kubernetes_feature_enabled{name="FeatureA",stage="BETA"} 1` + Expect(extractLabel(line, "name")).To(Equal("FeatureA")) + }) + + It("should extract stage label", func() { + line := `kubernetes_feature_enabled{name="FeatureA",stage="BETA"} 1` + Expect(extractLabel(line, "stage")).To(Equal("BETA")) + }) + + It("should return empty for missing label", func() { + line := `kubernetes_feature_enabled{name="FeatureA"} 1` + Expect(extractLabel(line, "stage")).To(BeEmpty()) + }) + + It("should handle empty label value", func() { + line := `kubernetes_feature_enabled{name="FeatureA",stage=""} 1` + Expect(extractLabel(line, "stage")).To(BeEmpty()) + }) +}) + +var _ = Describe("reconcile", func() { + var ( + dc *mock.DependencyContainerMock + values *mock.OutputPatchableValuesCollectorMock + server *httptest.Server + ) + + newInput := func() *pkg.HookInput { + return &pkg.HookInput{ + Values: values, + DC: dc, + Logger: log.NewNop(), + } + } + + setupServer := func(metricsBody string) { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, metricsBody) + })) + + dc.GetClientConfigMock.Return(&rest.Config{Host: server.URL}, nil) + } + + BeforeEach(func() { + dc = mock.NewDependencyContainerMock(GinkgoT()) + values = mock.NewPatchableValuesCollectorMock(GinkgoT()) + }) + + AfterEach(func() { + if server != nil { + server.Close() + } + }) + + It("should set feature gates from metrics", func() { + setupServer(joinLines( + metricsLine("FeatureA", "BETA", 1), + metricsLine("FeatureB", "ALPHA", 0), + metricsLine("FeatureC", "", 1), + )) + + setValues := make(map[string]any) + values.SetMock.Set(func(path string, v any) { + setValues[path] = v + }) + + Expect(reconcile(context.Background(), newInput())).To(Succeed()) + + Expect(setValues).To(HaveKeyWithValue(featureGatesPath, ConsistOf("FeatureA", "FeatureC"))) + }) + + It("should return error when client config fails", func() { + dc.GetClientConfigMock.Return(nil, fmt.Errorf("no kubeconfig")) + + err := reconcile(context.Background(), newInput()) + Expect(err).To(MatchError(ContainSubstring("no kubeconfig"))) + }) +}) + +func joinLines(lines ...string) string { + result := "" + for i, line := range lines { + if i > 0 { + result += "\n" + } + result += line + } + return result +} diff --git a/images/virtualization-artifact/pkg/kubeapi/kubeapi.go b/images/virtualization-artifact/pkg/kubeapi/kubeapi.go index 21c41e34d6..40eb1505d7 100644 --- a/images/virtualization-artifact/pkg/kubeapi/kubeapi.go +++ b/images/virtualization-artifact/pkg/kubeapi/kubeapi.go @@ -79,7 +79,14 @@ func isResourceV1Enabled(clientset kubernetes.Interface) (bool, error) { } func HasDRAFeatureGates() bool { - return os.Getenv("HAS_DRA_FEATURE_GATES") == "true" + count := 0 + for feature := range strings.SplitSeq(os.Getenv("KUBE_APISERVER_FEATURE_GATES"), ",") { + switch feature { + case "DRAResourceClaimDeviceStatus", "DRADeviceBindingConditions", "DRAConsumableCapacity": + count++ + } + } + return count == 3 } func HasDRAPartitionableDevices() bool { diff --git a/openapi/values.yaml b/openapi/values.yaml index c896917cdd..7fb7b2cc7f 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -147,9 +147,6 @@ properties: type: object default: {} additionalProperties: true - hasDraFeatureGates: - type: string - default: "" kubeAPIServerFeatureGates: type: array default: [] diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index f63cddcd54..8db602dd1e 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -116,6 +116,4 @@ true value: {{ .Values.global.clusterConfiguration.serviceSubnetCIDR }} - name: KUBE_APISERVER_FEATURE_GATES value: {{ .Values.virtualization.internal.kubeAPIServerFeatureGates | toJson | quote }} -- name: HAS_DRA_FEATURE_GATES - value: {{ .Values.virtualization.internal.hasDraFeatureGates | quote }} {{- end }} diff --git a/templates/virtualization-dra/_helper.tpl b/templates/virtualization-dra/_helper.tpl index fa1681e470..e27215d57e 100644 --- a/templates/virtualization-dra/_helper.tpl +++ b/templates/virtualization-dra/_helper.tpl @@ -1,12 +1,16 @@ {{- define "virtualization-dra.isEnabled" -}} {{- if eq (include "hasValidModuleConfig" .) "true" -}} {{- if semverCompare ">=1.34" .Values.global.discovery.kubernetesVersion -}} -{{- if eq "true" .Values.virtualization.internal.hasDraFeatureGates -}} +{{- if has "DRAResourceClaimDeviceStatus" .Values.virtualization.internal.kubeAPIServerFeatureGates -}} +{{- if has "DRADeviceBindingConditions" .Values.virtualization.internal.kubeAPIServerFeatureGates -}} +{{- if has "DRAConsumableCapacity" .Values.virtualization.internal.kubeAPIServerFeatureGates -}} true {{- end -}} {{- end -}} {{- end -}} {{- end -}} +{{- end -}} +{{- end -}} {{- define "virtualization-dra.featureGates" -}} {{- $base := "--feature-gates=USBGateway=true,USBNodeLocalMultiAllocation=true" -}} diff --git a/tools/kubeconform/fixtures/module-values.yaml b/tools/kubeconform/fixtures/module-values.yaml index 2877b39f4c..5c49d78f2f 100644 --- a/tools/kubeconform/fixtures/module-values.yaml +++ b/tools/kubeconform/fixtures/module-values.yaml @@ -404,7 +404,6 @@ virtualization: parallelOutboundMigrationsPerNode: 10 virtHandler: nodeCount: 1 - hasDraFeatureGates: "true" kubeAPIServerFeatureGates: - TopologyAwareHints - RotateKubeletServerCertificate