diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 36e72248..a08d3269 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,6 +27,6 @@ jobs: with: go-version: ${{ matrix.go-version }} - uses: actions/checkout@v4 - - run: make test + - run: sudo make test - run: make lint diff --git a/examples/demo_gke_multinetwork/resourceclaim.yaml b/examples/demo_gke_multinetwork/resourceclaim.yaml index b602bfe3..78a2ca6e 100644 --- a/examples/demo_gke_multinetwork/resourceclaim.yaml +++ b/examples/demo_gke_multinetwork/resourceclaim.yaml @@ -34,7 +34,7 @@ metadata: spec: containers: - name: ctr1 - image: registry.k8s.io/e2e-test-images/agnhost:2.39 + image: registry.k8s.io/e2e-test-images/agnhost:2.54 resourceClaims: - name: dranet-network resourceClaimName: dranet-network diff --git a/examples/demo_gke_multinetwork/resourceclaimtemplate.yaml b/examples/demo_gke_multinetwork/resourceclaimtemplate.yaml index 9f6e4c73..5b1415e4 100644 --- a/examples/demo_gke_multinetwork/resourceclaimtemplate.yaml +++ b/examples/demo_gke_multinetwork/resourceclaimtemplate.yaml @@ -48,7 +48,7 @@ spec: resourceClaimTemplateName: phy-interfaces-template containers: - name: agnhost - image: registry.k8s.io/e2e-test-images/agnhost:2.39 + image: registry.k8s.io/e2e-test-images/agnhost:2.54 args: - netexec - --http-port=80 diff --git a/examples/resourceclaim.yaml b/examples/resourceclaim.yaml index fead792e..6ac38acc 100644 --- a/examples/resourceclaim.yaml +++ b/examples/resourceclaim.yaml @@ -42,7 +42,7 @@ metadata: spec: containers: - name: ctr1 - image: registry.k8s.io/e2e-test-images/agnhost:2.39 + image: registry.k8s.io/e2e-test-images/agnhost:2.54 resourceClaims: - name: dummy1 resourceClaimName: dummy-interface-static-ip diff --git a/examples/resourceclaim_advanced.yaml b/examples/resourceclaim_advanced.yaml new file mode 100644 index 00000000..97becee1 --- /dev/null +++ b/examples/resourceclaim_advanced.yaml @@ -0,0 +1,57 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaim +metadata: + name: dummy-interface-advanced +spec: + devices: + requests: + - name: req-dummy-advanced + deviceClassName: dra.net + selectors: + - cel: + expression: device.attributes["dra.net"].ifName == "dummy3" + config: + - opaque: + driver: dra.net + parameters: + interface: + name: "dranet0" + addresses: + - "169.254.169.14/24" + mtu: 4321 + hardwareAddr: "00:11:22:33:44:55" + ethtool: + features: + tcp-segmentation-offload: false + generic-receive-offload: false + large-receive-offload: false +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-advanced-cfg + labels: + app: pod +spec: + containers: + - name: ctr1 + image: registry.k8s.io/e2e-test-images/agnhost:2.54 + # Keep the container running + command: ["sleep", "infinity"] + resourceClaims: + - name: dummy1 + resourceClaimName: dummy-interface-advanced diff --git a/examples/resourceclaim_bigtcp.yaml b/examples/resourceclaim_bigtcp.yaml new file mode 100644 index 00000000..10ea3a60 --- /dev/null +++ b/examples/resourceclaim_bigtcp.yaml @@ -0,0 +1,59 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaim +metadata: + name: dummy-interface-bigtcp +spec: + devices: + requests: + - name: req-dummy-bigtcp + deviceClassName: dra.net + selectors: + - cel: + expression: device.attributes["dra.net"].ifName == "dummy4" + config: + - opaque: + driver: dra.net + parameters: + interface: + name: "dranet1" # Name of the interface inside the pod + addresses: + - "192.168.200.1/24" + mtu: 8896 + gsoMaxSize: 65536 + groMaxSize: 65536 + gsoIPv4MaxSize: 65536 + groIPv4MaxSize: 65536 + ethtool: + features: + tcp-segmentation-offload: true + generic-receive-offload: true + large-receive-offload: false +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-bigtcp-test + labels: + app: bigtcp +spec: + containers: + - name: agnhost-bigtcp + image: registry.k8s.io/e2e-test-images/agnhost:2.54 + command: ["sleep", "infinity"] + resourceClaims: + - name: bigtcp-net + resourceClaimName: dummy-interface-bigtcp diff --git a/examples/resourceclaim_route.yaml b/examples/resourceclaim_route.yaml index c91d8c2d..b5cec2a3 100644 --- a/examples/resourceclaim_route.yaml +++ b/examples/resourceclaim_route.yaml @@ -47,7 +47,7 @@ metadata: spec: containers: - name: ctr1 - image: registry.k8s.io/e2e-test-images/agnhost:2.39 + image: registry.k8s.io/e2e-test-images/agnhost:2.54 resourceClaims: - name: dummy1 resourceClaimName: dummy-interface-static-ip-route diff --git a/examples/resourceclaimtemplate.yaml b/examples/resourceclaimtemplate.yaml index e5886329..a2504050 100644 --- a/examples/resourceclaimtemplate.yaml +++ b/examples/resourceclaimtemplate.yaml @@ -56,7 +56,7 @@ spec: resourceClaimTemplateName: phy-interfaces-template containers: - name: agnhost - image: registry.k8s.io/e2e-test-images/agnhost:2.39 + image: registry.k8s.io/e2e-test-images/agnhost:2.54 args: - netexec - --http-port=80 diff --git a/go.mod b/go.mod index 48cf56b8..30b21c76 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/google/cel-go v0.25.0 github.com/google/go-cmp v0.7.0 github.com/insomniacslk/dhcp v0.0.0-20250417080101-5f8cf70e8c5f + github.com/mdlayher/genetlink v1.3.2 + github.com/mdlayher/netlink v1.7.2 github.com/prometheus/client_golang v1.22.0 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 diff --git a/go.sum b/go.sum index 873f2cdc..792f1127 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,10 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= +github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY= github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= diff --git a/pkg/apis/types.go b/pkg/apis/types.go index 69a09088..f3adba88 100644 --- a/pkg/apis/types.go +++ b/pkg/apis/types.go @@ -16,24 +16,75 @@ limitations under the License. package apis -// NetworkConfig represents the desired state of all network interfaces and their associated routes. +// NetworkConfig represents the desired state of all network interfaces and their associated routes, +// along with ethtool and sysctl configurations to be applied within the Pod's network namespace. type NetworkConfig struct { - Interface InterfaceConfig `json:"interface"` // Changed to a slice to support multiple interfaces - Routes []RouteConfig `json:"routes"` + // Interface defines core properties of the network interface. + // Settings here are typically managed by `ip link` commands. + Interface InterfaceConfig `json:"interface"` + + // Routes defines static routes to be configured for this interface. + Routes []RouteConfig `json:"routes,omitempty"` + + // Ethtool defines hardware offload features and other settings managed by `ethtool`. + Ethtool *EthtoolConfig `json:"ethtool,omitempty"` } // InterfaceConfig represents the configuration for a single network interface. +// These are fundamental properties, often managed using `ip link` commands. type InterfaceConfig struct { - Name string `json:"name,omitempty"` // Logical name of the interface (e.g., "eth0", "enp0s3") - Addresses []string `json:"addresses,omitempty"` // IP addresses and their CIDR masks - MTU int32 `json:"mtu,omitempty"` // Maximum Transmission Unit, optional - HardwareAddr string `json:"hardwareAddr,omitempty"` // Read-only: Current hardware address (might be useful for GET) + // Name is the desired logical name of the interface inside the Pod (e.g., "net0", "eth_app"). + // If not specified, DraNet may use or derive a name from the original interface. + Name string `json:"name,omitempty"` + + // Addresses is a list of IP addresses in CIDR format (e.g., "192.168.1.10/24") + // to be assigned to the interface. + Addresses []string `json:"addresses,omitempty"` + + // MTU is the Maximum Transmission Unit for the interface. + MTU *int32 `json:"mtu,omitempty"` + + // HardwareAddr is the MAC address of the interface. + HardwareAddr *string `json:"hardwareAddr,omitempty"` + + // GSOMaxSize sets the maximum Generic Segmentation Offload size for IPv6. + // Managed by `ip link set gso_max_size `. For enabling Big TCP. + GSOMaxSize *int32 `json:"gsoMaxSize,omitempty"` + + // GROMaxSize sets the maximum Generic Receive Offload size for IPv6. + // Managed by `ip link set gro_max_size `. For enabling Big TCP. + GROMaxSize *int32 `json:"groMaxSize,omitempty"` + + // GSOv4MaxSize sets the maximum Generic Segmentation Offload size. + // Managed by `ip link set gso_ipv4_max_size `. For enabling Big TCP. + GSOIPv4MaxSize *int32 `json:"gsoIPv4MaxSize,omitempty"` + + // GROv4MaxSize sets the maximum Generic Receive Offload size. + // Managed by `ip link set gro_ipv4_max_size `. For enabling Big TCP. + GROIPv4MaxSize *int32 `json:"groIPv4MaxSize,omitempty"` } // RouteConfig represents a network route configuration. type RouteConfig struct { - Destination string `json:"destination,omitempty"` // e.g., "0.0.0.0/0" for default, "10.0.0.0/8" - Gateway string `json:"gateway,omitempty"` // The "gateway" address, e.g., "192.168.1.1" - Source string `json:"source,omitempty"` // Optional source address for policy routing - Scope uint8 `json:"scope,omitempty"` // Optional scope of the route, only Link (253) or Universe (0) allowed + // Destination is the target network in CIDR format (e.g., "0.0.0.0/0", "10.0.0.0/8"). + Destination string `json:"destination,omitempty"` + // Gateway is the IP address of the gateway for this route. + Gateway string `json:"gateway,omitempty"` + // Source is an optional source IP address for policy routing. + Source string `json:"source,omitempty"` + // Scope is the scope of the route (e.g., link, host, global). + // Refers to Linux route scopes (e.g., 0 for RT_SCOPE_UNIVERSE, 253 for RT_SCOPE_LINK). + Scope uint8 `json:"scope,omitempty"` +} + +// EthtoolConfig defines ethtool-based optimizations for a network interface. +// These settings correspond to features typically toggled using `ethtool -K on|off`. +type EthtoolConfig struct { + // Features is a map of ethtool feature names to their desired state (true for on, false for off). + // Example: {"tcp-segmentation-offload": true, "rx-checksum": true} + Features map[string]bool `json:"features,omitempty"` + + // PrivateFlags is a map of device-specific private flag names to their desired state. + // Example: {"my-custom-flag": true} + PrivateFlags map[string]bool `json:"privateFlags,omitempty"` } diff --git a/pkg/apis/validation.go b/pkg/apis/validation.go index 965597cc..970742b7 100644 --- a/pkg/apis/validation.go +++ b/pkg/apis/validation.go @@ -17,67 +17,189 @@ limitations under the License. package apis import ( - "errors" "fmt" "net" "net/netip" + "strings" + "unicode" "golang.org/x/sys/unix" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/json" ) -// ValidateConfig validates the data in a runtime.RawExtension against the OpenAPI schema. -func ValidateConfig(raw *runtime.RawExtension) (*NetworkConfig, error) { - if raw == nil || raw.Raw == nil { - return nil, nil - } - // Check if raw.Raw is empty - if len(raw.Raw) == 0 { - return nil, nil +const ( + // MinMTU is the minimum practical MTU (e.g., for IPv4). + MinMTU = 68 + // MaxInterfaceNameLen is typically IFNAMSIZ-1 (usually 15 on Linux). + MaxInterfaceNameLen = 15 +) + +// ValidateConfig unmarshals and validates the NetworkConfig from a runtime.RawExtension. +// It performs strict unmarshalling and then calls specific validation functions for each part of the config. +// Returns the parsed NetworkConfig and a slice of errors if any validation fails. +func ValidateConfig(raw *runtime.RawExtension) (*NetworkConfig, []error) { + if raw == nil || raw.Raw == nil || len(raw.Raw) == 0 { + return nil, nil // No configuration provided, so no validation errors. } - var errorsList []error + var config NetworkConfig + var allErrors []error + + // Strict unmarshalling to catch unknown fields. strictErrs, err := json.UnmarshalStrict(raw.Raw, &config) if err != nil { - return nil, fmt.Errorf("failed to unmarshal JSON data: %w", err) + allErrors = append(allErrors, fmt.Errorf("failed to unmarshal JSON data: %w", err)) + // If basic unmarshalling fails, we can't proceed with further validation. + return nil, allErrors } if len(strictErrs) > 0 { - return nil, fmt.Errorf("failed to unmarshal strict JSON data: %w", errors.Join(strictErrs...)) + for _, e := range strictErrs { + allErrors = append(allErrors, fmt.Errorf("failed to unmarshal strict JSON data: %w", e)) + } + } + + // Validate InterfaceConfig + allErrors = append(allErrors, validateInterfaceConfig(&config.Interface, "interface")...) + + // Validate Routes + if len(config.Routes) > 0 { + allErrors = append(allErrors, validateRoutes(config.Routes, "routes")...) + } + + // Validate EthtoolConfig if present + if config.Ethtool != nil { + allErrors = append(allErrors, validateEthtoolConfig(config.Ethtool, "ethtool")...) + } + + if len(allErrors) > 0 { + return &config, allErrors // Return partially parsed config with errors } - for _, ip := range config.Interface.Addresses { - if _, err := netip.ParsePrefix(ip); err != nil { - errorsList = append(errorsList, fmt.Errorf("invalid IP in CIDR format %s", ip)) + return &config, nil +} + +// isValidLinuxInterfaceName checks if the provided name is a valid Linux interface name. +// Basic checks: length, no '/', no whitespace, not '.' or '..'. +func isValidLinuxInterfaceName(name string, fieldPath string) (allErrors []error) { + if name == "" { + // Allow empty name, as DraNet might derive it. If a name is provided, it must be valid. + return nil + } + if len(name) > MaxInterfaceNameLen { + allErrors = append(allErrors, fmt.Errorf("%s: name '%s' exceeds maximum length of %d characters", fieldPath, name, MaxInterfaceNameLen)) + } + if strings.Contains(name, "/") { + allErrors = append(allErrors, fmt.Errorf("%s: name '%s' cannot contain '/'", fieldPath, name)) + } + if strings.ContainsAny(name, " \t\n\v\f\r") { // Check for any whitespace + allErrors = append(allErrors, fmt.Errorf("%s: name '%s' cannot contain whitespace", fieldPath, name)) + } + if name == "." || name == ".." { + allErrors = append(allErrors, fmt.Errorf("%s: name '%s' cannot be '.' or '..'", fieldPath, name)) + } + for _, r := range name { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' && r != '_' && r != '.' { + // If '/' or whitespace, it's already covered by more specific checks above. + // Avoid adding a generic "invalid character" error for these specific cases. + if r == '/' || unicode.IsSpace(r) { + continue + } + // This is a more restrictive set for safety, Linux itself might allow more. + // Common practice avoids special characters other than '-', '_', '.' + // The kernel function dev_valid_name also disallows non-printable chars. + allErrors = append(allErrors, fmt.Errorf("%s: name '%s' contains invalid character '%c'. Only letters, digits, '-', '_', '.' are recommended", fieldPath, name, r)) } } + return allErrors +} + +// validateInterfaceConfig validates the InterfaceConfig part of the NetworkConfig. +func validateInterfaceConfig(cfg *InterfaceConfig, fieldPath string) (allErrors []error) { + if cfg == nil { + return + } + + allErrors = append(allErrors, isValidLinuxInterfaceName(cfg.Name, fieldPath+".name")...) + + for i, addr := range cfg.Addresses { + if _, err := netip.ParsePrefix(addr); err != nil { + allErrors = append(allErrors, fmt.Errorf("%s.addresses[%d]: invalid IP CIDR format '%s': %w", fieldPath, i, addr, err)) + } + } + + if cfg.MTU != nil { + if *cfg.MTU < MinMTU { + allErrors = append(allErrors, fmt.Errorf("%s.mtu: must be at least %d, got %d", fieldPath, MinMTU, *cfg.MTU)) + } + } + + if cfg.HardwareAddr != nil { + if _, err := net.ParseMAC(*cfg.HardwareAddr); err != nil { + allErrors = append(allErrors, fmt.Errorf("%s.hardwareAddress: invalid Hardware Address format '%s': %w", fieldPath, *cfg.HardwareAddr, err)) + } + } + + if cfg.GSOMaxSize != nil && *cfg.GSOMaxSize <= 0 { + allErrors = append(allErrors, fmt.Errorf("%s.gsoMaxSize: must be positive, got %d", fieldPath, *cfg.GSOMaxSize)) + } + + if cfg.GROMaxSize != nil && *cfg.GROMaxSize <= 0 { + allErrors = append(allErrors, fmt.Errorf("%s.groMaxSize: must be positive, got %d", fieldPath, *cfg.GROMaxSize)) + } + + if cfg.GSOIPv4MaxSize != nil && *cfg.GSOIPv4MaxSize <= 0 { + allErrors = append(allErrors, fmt.Errorf("%s.gsov4MaxSize: must be positive, got %d", fieldPath, *cfg.GSOIPv4MaxSize)) + } + + if cfg.GROIPv4MaxSize != nil && *cfg.GROIPv4MaxSize <= 0 { + allErrors = append(allErrors, fmt.Errorf("%s.grov4MaxSize: must be positive, got %d", fieldPath, *cfg.GROIPv4MaxSize)) + } + + return allErrors +} + +// validateRoutes validates a slice of RouteConfig. +func validateRoutes(routes []RouteConfig, fieldPath string) (allErrors []error) { + for i, route := range routes { + currentFieldPath := fmt.Sprintf("%s[%d]", fieldPath, i) - // Validate routes - for i, route := range config.Routes { if route.Destination == "" { - errorsList = append(errorsList, fmt.Errorf("route %d: destination cannot be empty", i)) + allErrors = append(allErrors, fmt.Errorf("%s.destination: cannot be empty", currentFieldPath)) } else { - // Validate Destination as CIDR or IP if _, _, err := net.ParseCIDR(route.Destination); err != nil { if net.ParseIP(route.Destination) == nil { - errorsList = append(errorsList, fmt.Errorf("route %d: invalid destination IP or CIDR '%s'", i, route.Destination)) + allErrors = append(allErrors, fmt.Errorf("%s.destination: invalid IP or CIDR format '%s'", currentFieldPath, route.Destination)) } } } - // only Link or Univer scope allowed + scopeIsLink := false if route.Scope != unix.RT_SCOPE_UNIVERSE && route.Scope != unix.RT_SCOPE_LINK { - errorsList = append(errorsList, fmt.Errorf("route %d: invalid scope '%d' only Link (253) or Universe (0) allowed", i, route.Scope)) + allErrors = append(allErrors, fmt.Errorf("%s.scope: invalid scope '%d', only Link (%d) or Universe (%d) allowed", currentFieldPath, route.Scope, unix.RT_SCOPE_LINK, unix.RT_SCOPE_UNIVERSE)) + } + if route.Scope == unix.RT_SCOPE_LINK { + scopeIsLink = true } - // Link scoped routes do not need gateway if route.Gateway != "" { if net.ParseIP(route.Gateway) == nil { - errorsList = append(errorsList, fmt.Errorf("route %d: invalid gateway IP '%s'", i, route.Gateway)) + allErrors = append(allErrors, fmt.Errorf("%s.gateway: invalid IP address format '%s'", currentFieldPath, route.Gateway)) + } + } else if !scopeIsLink { // Gateway is required if scope is Universe + allErrors = append(allErrors, fmt.Errorf("%s.gateway: must be specified for Universe scope routes", currentFieldPath)) + } + + if route.Source != "" { + if net.ParseIP(route.Source) == nil { + allErrors = append(allErrors, fmt.Errorf("%s.source: invalid IP address format '%s'", currentFieldPath, route.Source)) } - } else if route.Scope != unix.RT_SCOPE_LINK { - errorsList = append(errorsList, fmt.Errorf("route %d: for destination '%s' must have a gateway", i, route.Destination)) } } - return &config, errors.Join(errorsList...) + return allErrors +} + +// validateEthtoolConfig validates the EthtoolConfig part of the NetworkConfig. +func validateEthtoolConfig(cfg *EthtoolConfig, fieldPath string) (allErrors []error) { + return allErrors } diff --git a/pkg/apis/validation_test.go b/pkg/apis/validation_test.go index 2837d470..ae718365 100644 --- a/pkg/apis/validation_test.go +++ b/pkg/apis/validation_test.go @@ -13,217 +13,357 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package apis import ( + "encoding/json" + "reflect" "strings" "testing" + "golang.org/x/sys/unix" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ) +// Helper to create a *runtime.RawExtension from a struct +func newRawExtension(t *testing.T, data interface{}) *runtime.RawExtension { + t.Helper() + rawJSON, err := json.Marshal(data) + if err != nil { + t.Fatalf("Failed to marshal test data: %v", err) + } + return &runtime.RawExtension{Raw: rawJSON} +} + +// Helper to create a *runtime.RawExtension from a JSON string +func newRawExtensionFromString(t *testing.T, jsonStr string) *runtime.RawExtension { + t.Helper() + return &runtime.RawExtension{Raw: []byte(jsonStr)} +} + func TestValidateConfig(t *testing.T) { + validConfig := NetworkConfig{ + Interface: InterfaceConfig{Name: "eth0", Addresses: []string{"192.168.1.1/24"}, MTU: ptr.To[int32](1500)}, + Routes: []RouteConfig{ + {Destination: "0.0.0.0/0", Gateway: "192.168.1.254", Scope: unix.RT_SCOPE_UNIVERSE}, + }, + Ethtool: &EthtoolConfig{Features: map[string]bool{"tso": true}}, + } + invalidInterfaceConf := NetworkConfig{Interface: InterfaceConfig{Name: "eth/0"}} + invalidRouteConf := NetworkConfig{Interface: InterfaceConfig{Name: "eth0"}, Routes: []RouteConfig{{Destination: "invalid-cidr"}}} + tests := []struct { - name string - raw *runtime.RawExtension - wantErr bool - errMsgs []string + name string + raw *runtime.RawExtension + expectErr bool + expectedCfg *NetworkConfig + errContains []string }{ { - name: "valid config", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": { - "name": "eth0", - "addresses": ["192.168.1.10/24", "2001:db8::1/64"], - "mtu": 1500 - }, - "routes": [ - { - "destination": "0.0.0.0/0", - "gateway": "192.168.1.1" - }, - { - "destination": "2001:db8:abcd::/48", - "gateway": "2001:db::1" - } - ] - }`)}, - wantErr: false, + name: "valid full config", + raw: newRawExtension(t, validConfig), + expectErr: false, + expectedCfg: &validConfig, }, { - name: "nil raw extension", - raw: nil, - wantErr: false, + name: "nil raw extension", + raw: nil, + expectErr: false, + expectedCfg: nil, }, { - name: "nil raw field in raw extension", - raw: &runtime.RawExtension{Raw: nil}, - wantErr: false, + name: "empty raw data in extension", + raw: &runtime.RawExtension{Raw: []byte{}}, + expectErr: false, + expectedCfg: nil, }, { - name: "empty raw field in raw extension", - raw: &runtime.RawExtension{Raw: []byte{}}, - wantErr: false, + name: "malformed json", + raw: newRawExtensionFromString(t, `{"interface": {"name": "eth0"`), + expectErr: true, + expectedCfg: nil, // Unmarshal itself fails, cfg should be nil or zero + errContains: []string{"failed to unmarshal JSON data"}, }, { - name: "malformed json", - raw: &runtime.RawExtension{Raw: []byte(`{"interface": {"name": "eth0"`)}, // Missing closing brace - wantErr: true, - errMsgs: []string{"failed to unmarshal JSON data: unexpected end of JSON"}, + name: "unknown field (strict unmarshal error)", + raw: newRawExtensionFromString(t, `{"interface": {"name": "eth0", "unknownField": "test"}}`), + expectErr: true, + expectedCfg: &NetworkConfig{Interface: InterfaceConfig{Name: "eth0"}}, // sigs.k8s.io/json unmarshals known fields first + errContains: []string{"failed to unmarshal strict JSON data", "unknownField"}, }, { - name: "unknown fields", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"gateways": "192.168.1.1"}] - }`)}, // use gateways instead gateway - wantErr: true, - errMsgs: []string{`failed to unmarshal strict JSON data: unknown field "routes[0].gateways"`}, + name: "config with interface validation error", + raw: newRawExtension(t, invalidInterfaceConf), + expectErr: true, + expectedCfg: &invalidInterfaceConf, + errContains: []string{"interface.name: name 'eth/0' cannot contain '/'"}, }, { - name: "invalid interface IP CIDR", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": { - "name": "eth0", - "addresses": ["192.168.1.10/240"] - } - }`)}, - wantErr: true, - errMsgs: []string{"invalid IP in CIDR format 192.168.1.10/240"}, - }, - { - name: "route with empty destination", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"gateway": "192.168.1.1"}] - }`)}, - wantErr: true, - errMsgs: []string{"route 0: destination cannot be empty"}, - }, - { - name: "route with invalid destination", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "not-an-ip", "gateway": "192.168.1.1"}] - }`)}, - wantErr: true, - errMsgs: []string{"route 0: invalid destination IP or CIDR 'not-an-ip'"}, - }, - { - name: "route with no gateway", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "10.0.0.0/8"}] - }`)}, - wantErr: true, - errMsgs: []string{"route 0: for destination '10.0.0.0/8' must have a gateway"}, - }, - { - name: "route with invalid gateway IP", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "10.0.0.0/8", "gateway": "not-a-gateway"}] - }`)}, - wantErr: true, - errMsgs: []string{"route 0: invalid gateway IP 'not-a-gateway'"}, - }, - { - name: "multiple errors", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": { - "name": "eth0", - "addresses": ["192.168.1.10/240", "10.0.0.1/invalid"] - }, - "routes": [ - {"destination": "", "gateway": "192.168.1.1"}, - {"destination": "not-an-ip", "gateway": "192.168.1.1"}, - {"destination": "10.0.0.0/8"}, - {"destination": "10.0.1.0/24", "gateway": "not-a-gateway"} - ] - }`)}, - wantErr: true, - errMsgs: []string{ - "invalid IP in CIDR format 192.168.1.10/240", - "invalid IP in CIDR format 10.0.0.1/invalid", - "route 0: destination cannot be empty", - "route 1: invalid destination IP or CIDR 'not-an-ip'", - "route 3: invalid gateway IP 'not-a-gateway'", - }, - }, - { - name: "route with valid scope universe (0)", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "10.0.0.0/8", "gateway": "192.168.1.1", "scope": 0}] - }`)}, - wantErr: false, - }, - { - name: "route with valid scope link (253)", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "10.0.0.0/8", "scope": 253}] - }`)}, - wantErr: false, - }, - { - name: "route with invalid scope", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "10.0.0.0/8", "gateway": "192.168.1.1", "scope": 100}] - }`)}, - wantErr: true, - errMsgs: []string{"route 0: invalid scope '100' only Link (253) or Universe (0) allowed"}, - }, - { - name: "route with link scope and no gateway (valid)", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "10.0.0.0/8", "scope": 253}] - }`)}, - wantErr: false, - }, - { - name: "route with universe scope and no gateway (invalid)", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"name": "eth0", "addresses": ["192.168.1.10/24"]}, - "routes": [{"destination": "10.0.0.0/8", "scope": 0}] - }`)}, - wantErr: true, - errMsgs: []string{"route 0: for destination '10.0.0.0/8' must have a gateway"}, - }, - { - name: "multiple errors including scope", - raw: &runtime.RawExtension{Raw: []byte(`{ - "interface": {"addresses": ["192.168.1.10/240"]}, - "routes": [ - {"destination": "10.0.0.0/8", "gateway": "192.168.1.1", "scope": 100}, - {"destination": "10.0.1.0/24", "scope": 0} - ] - }`)}, - wantErr: true, - errMsgs: []string{ - "invalid IP in CIDR format 192.168.1.10/240", - "route 0: invalid scope '100' only Link (253) or Universe (0) allowed", - "route 1: for destination '10.0.1.0/24' must have a gateway"}, + name: "config with route validation error", + raw: newRawExtension(t, invalidRouteConf), + expectErr: true, + expectedCfg: &invalidRouteConf, + errContains: []string{"routes[0].destination: invalid IP or CIDR format 'invalid-cidr'"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := ValidateConfig(tt.raw) - if (err != nil) != tt.wantErr { - t.Errorf("ValidateConfig() error = %v, wantErr %v", err, tt.wantErr) - return + cfg, errs := ValidateConfig(tt.raw) + hasErrs := len(errs) > 0 + + if hasErrs != tt.expectErr { + t.Errorf("ValidateConfig() error = %v, expectErr %v. Errors: %v", hasErrs, tt.expectErr, errs) } - if tt.wantErr { - for _, errMsg := range tt.errMsgs { - if !strings.Contains(err.Error(), errMsg) { - t.Errorf("ValidateConfig() error = %v, want to contain %v", err, errMsg) + + if tt.expectErr { + for _, substr := range tt.errContains { + found := false + for _, err := range errs { + if strings.Contains(err.Error(), substr) { + found = true + break + } + } + if !found { + t.Errorf("ValidateConfig() expected error to contain '%s', but it didn't. Errors: %v", substr, errs) } } } + + if !reflect.DeepEqual(cfg, tt.expectedCfg) { + t.Errorf("ValidateConfig() cfg = %+v, expectedCfg %+v", cfg, tt.expectedCfg) + } + }) + } +} + +func TestIsValidLinuxInterfaceName(t *testing.T) { + tests := []struct { + name string + ifName string + fieldPath string + expectErr bool + }{ + {"valid short", "eth0", "iface.name", false}, + {"valid with hyphen", "my-nic", "iface.name", false}, + {"valid with underscore", "my_nic", "iface.name", false}, + {"valid with period", "my.nic", "iface.name", false}, + {"valid max length", strings.Repeat("a", MaxInterfaceNameLen), "iface.name", false}, + {"empty name (allowed)", "", "iface.name", false}, + {"too long", strings.Repeat("a", MaxInterfaceNameLen+1), "iface.name", true}, + {"contains slash", "eth/0", "iface.name", true}, + {"contains space", "eth 0", "iface.name", true}, + {"is dot", ".", "iface.name", true}, + {"is dotdot", "..", "iface.name", true}, + {"contains invalid char", "eth!", "iface.name", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + errs := isValidLinuxInterfaceName(tt.ifName, tt.fieldPath) + if (len(errs) > 0) != tt.expectErr { + t.Errorf("isValidLinuxInterfaceName(%s) expectErr %v, got errors: %v", tt.ifName, tt.expectErr, errs) + } + }) + } +} + +func TestValidateInterfaceConfig(t *testing.T) { + tests := []struct { + name string + cfg *InterfaceConfig + fieldPath string + expectErr bool + errCount int // Expected number of errors + }{ + { + name: "valid", + cfg: &InterfaceConfig{Name: "eth0", Addresses: []string{"10.0.0.1/24"}, MTU: ptr.To[int32](1500), GSOMaxSize: ptr.To[int32](65536)}, + fieldPath: "iface", + expectErr: false, + }, + { + name: "invalid name", + cfg: &InterfaceConfig{Name: "eth/"}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "invalid address", + cfg: &InterfaceConfig{Name: "eth0", Addresses: []string{"10.0.0/24"}}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "invalid MTU (zero)", + cfg: &InterfaceConfig{Name: "eth0", MTU: ptr.To[int32](0)}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "invalid MTU (too small)", + cfg: &InterfaceConfig{Name: "eth0", MTU: ptr.To[int32](MinMTU - 1)}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "invalid GSO MaxSize", + cfg: &InterfaceConfig{Name: "eth0", GSOMaxSize: ptr.To[int32](0)}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "invalid GRO MaxSize", + cfg: &InterfaceConfig{Name: "eth0", GROMaxSize: ptr.To[int32](-1)}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "invalid GSO V4 MaxSize", + cfg: &InterfaceConfig{Name: "eth0", GSOIPv4MaxSize: ptr.To[int32](0)}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "invalid GRO V4 MaxSize", + cfg: &InterfaceConfig{Name: "eth0", GROIPv4MaxSize: ptr.To[int32](-1)}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "valid hardware address", + cfg: &InterfaceConfig{Name: "eth0", HardwareAddr: ptr.To("00:1A:2B:3C:4D:5E")}, + fieldPath: "iface", + expectErr: false, + }, + { + name: "invalid hardware address", + cfg: &InterfaceConfig{Name: "eth0", HardwareAddr: ptr.To("00-1A-2B-3C-4D-5E-XX")}, + fieldPath: "iface", + expectErr: true, + errCount: 1, + }, + { + name: "multiple errors", + cfg: &InterfaceConfig{Name: "eth/0", Addresses: []string{"badip"}, MTU: ptr.To[int32](0)}, + fieldPath: "iface", + expectErr: true, + errCount: 3, + }, + { + name: "nil config", + cfg: nil, + fieldPath: "iface", + expectErr: false, // Function should handle nil gracefully + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + errs := validateInterfaceConfig(tt.cfg, tt.fieldPath) + if (len(errs) > 0) != tt.expectErr { + t.Errorf("validateInterfaceConfig() expectErr %v, got errors: %v", tt.expectErr, errs) + } + if tt.expectErr && len(errs) != tt.errCount { + t.Errorf("validateInterfaceConfig() expected %d errors, got %d: %v", tt.errCount, len(errs), errs) + } + }) + } +} + +func TestValidateRoutes(t *testing.T) { + scopeLink := uint8(unix.RT_SCOPE_LINK) + scopeUniverse := uint8(unix.RT_SCOPE_UNIVERSE) + invalidScope := uint8(100) + + tests := []struct { + name string + routes []RouteConfig + fieldPath string + expectErr bool + errCount int + }{ + { + name: "valid default route", + routes: []RouteConfig{{Destination: "0.0.0.0/0", Gateway: "192.168.1.1", Scope: scopeUniverse}}, + fieldPath: "routes", + expectErr: false, + }, + { + name: "valid link-local route", + routes: []RouteConfig{{Destination: "169.254.0.0/16", Scope: scopeLink}}, + fieldPath: "routes", + expectErr: false, + }, + { + name: "empty destination", + routes: []RouteConfig{{Gateway: "192.168.1.1"}}, + fieldPath: "routes", + expectErr: true, + errCount: 1, + }, + { + name: "invalid destination CIDR", + routes: []RouteConfig{{Destination: "10.0.0/24", Gateway: "192.168.1.1"}}, + fieldPath: "routes", + expectErr: true, + errCount: 1, + }, + { + name: "universe scope no gateway", + routes: []RouteConfig{{Destination: "10.0.0.0/8", Scope: scopeUniverse}}, + fieldPath: "routes", + expectErr: true, + errCount: 1, + }, + { + name: "default scope (universe) no gateway", + routes: []RouteConfig{{Destination: "10.0.0.0/8"}}, // Scope defaults to Universe + fieldPath: "routes", + expectErr: true, + errCount: 1, + }, + { + name: "invalid gateway IP", + routes: []RouteConfig{{Destination: "0.0.0.0/0", Gateway: "not-an-ip"}}, + fieldPath: "routes", + expectErr: true, + errCount: 1, + }, + { + name: "invalid scope value", + routes: []RouteConfig{{Destination: "10.0.0.0/8", Scope: invalidScope, Gateway: "192.168.1.1"}}, + fieldPath: "routes", + expectErr: true, + errCount: 1, + }, + { + name: "invalid source IP", + routes: []RouteConfig{{Destination: "0.0.0.0/0", Gateway: "192.168.1.1", Source: "not-an-ip"}}, + fieldPath: "routes", + expectErr: true, + errCount: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + errs := validateRoutes(tt.routes, tt.fieldPath) + if (len(errs) > 0) != tt.expectErr { + t.Errorf("validateRoutes() expectErr %v, got errors: %v", tt.expectErr, errs) + } + if tt.expectErr && len(errs) != tt.errCount { + t.Errorf("validateRoutes() expected %d errors, got %d: %v", tt.errCount, len(errs), errs) + } }) } } diff --git a/pkg/driver/dra_hooks.go b/pkg/driver/dra_hooks.go index 40517844..a653983e 100644 --- a/pkg/driver/dra_hooks.go +++ b/pkg/driver/dra_hooks.go @@ -135,9 +135,9 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour continue } // Check if there is a custom configuration - conf, err := apis.ValidateConfig(&config.Opaque.Parameters) - if err != nil { - errorList = append(errorList, err) + conf, errs := apis.ValidateConfig(&config.Opaque.Parameters) + if len(errs) > 0 { + errorList = append(errorList, errs...) continue } // TODO: define a strategy for multiple configs @@ -152,8 +152,7 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour Namespace: claim.Namespace, Name: claim.Name, }, - NetDevice: netconf.Interface, - NetNamespaceRoutes: netconf.Routes, + Network: netconf, } ifName := names.GetOriginalName(result.Device) // Get Network configuration and merge it @@ -163,12 +162,12 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour continue } - if podCfg.NetDevice.Name == "" { - podCfg.NetDevice.Name = ifName + if podCfg.Network.Interface.Name == "" { + podCfg.Network.Interface.Name = ifName } // If there is no custom addresses then use the existing ones - if len(podCfg.NetDevice.Addresses) == 0 { + if len(podCfg.Network.Interface.Addresses) == 0 { // get the existing IP addresses nlAddresses, err := nlHandle.AddrList(link, netlink.FAMILY_ALL) if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) { @@ -181,7 +180,7 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour if address.Scope != unix.RT_SCOPE_UNIVERSE { continue } - podCfg.NetDevice.Addresses = append(podCfg.NetDevice.Addresses, address.IPNet.String()) + podCfg.Network.Interface.Addresses = append(podCfg.Network.Interface.Addresses, address.IPNet.String()) } } } @@ -190,15 +189,45 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour // this may be an interface that uses DHCP, so we bring it up if necessary and do a DHCP // request to gather the network parameters (IPs and Routes) ... but we DO NOT apply them // in the root namespace - if len(podCfg.NetDevice.Addresses) == 0 { + if len(podCfg.Network.Interface.Addresses) == 0 { klog.V(2).Infof("trying to get network configuration via DHCP") ip, routes, err := getDHCP(ifName) if err != nil { klog.Infof("fail to get configuration via DHCP: %v", err) } else { - podCfg.NetDevice.Addresses = []string{ip} - podCfg.NetNamespaceRoutes = append(podCfg.NetNamespaceRoutes, routes...) + podCfg.Network.Interface.Addresses = []string{ip} + podCfg.Network.Routes = append(podCfg.Network.Routes, routes...) + } + } + + // Obtain the existing supported ethtool features and validate the config + if podCfg.Network.Ethtool != nil { + client, err := newEthtoolClient(0) + if err != nil { + errorList = append(errorList, fmt.Errorf("fail to create ethtool client %v", err)) + continue + } + defer client.Close() + + ifFeatures, err := client.GetFeatures(ifName) + if err != nil { + errorList = append(errorList, fmt.Errorf("fail to get ethtool features %v", err)) + continue + } + + // translate features to the actual kernel names + ethtoolFeatures := map[string]bool{} + for feature, value := range podCfg.Network.Ethtool.Features { + aliases := ifFeatures.Get(feature) + if len(aliases) == 0 { + errorList = append(errorList, fmt.Errorf("feature %s not supported by interface", feature)) + continue + } + for _, alias := range aliases { + ethtoolFeatures[alias] = value + } } + podCfg.Network.Ethtool.Features = ethtoolFeatures } // Obtain the routes associated to the interface @@ -229,7 +258,7 @@ func (np *NetworkDriver) prepareResourceClaim(ctx context.Context, claim *resour routeCfg.Source = route.Src.String() } routeCfg.Scope = uint8(route.Scope) - podCfg.NetNamespaceRoutes = append(podCfg.NetNamespaceRoutes, routeCfg) + podCfg.Network.Routes = append(podCfg.Network.Routes, routeCfg) } // Get RDMA configuration: link and char devices diff --git a/pkg/driver/ethtool.go b/pkg/driver/ethtool.go new file mode 100644 index 00000000..3bb0dafb --- /dev/null +++ b/pkg/driver/ethtool.go @@ -0,0 +1,520 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/google/dranet/pkg/apis" + + "github.com/mdlayher/genetlink" + "github.com/mdlayher/netlink" + + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" + "k8s.io/klog/v2" +) + +// off_flag_def +// https://git.kernel.org/pub/scm/network/ethtool/ethtool.git/tree/common.c#n51 +// OffloadFlagDefinition is the Go equivalent of the C struct off_flag_def. +// We keep this struct as the source of truth for building the map. +type OffloadFlagDefinition struct { + ShortName string + LongName string + KernelPattern string +} + +// offloadFlagDefs is the translated slice of legacy feature definitions. +var offloadFlagDefs = []OffloadFlagDefinition{ + {"rx", "rx-checksumming", "rx-checksum"}, + {"tx", "tx-checksumming", "tx-checksum-*"}, + {"sg", "scatter-gather", "tx-scatter-gather*"}, + {"tso", "tcp-segmentation-offload", "tx-tcp*-segmentation"}, + {"ufo", "udp-fragmentation-offload", "tx-udp-fragmentation"}, + {"gso", "generic-segmentation-offload", "tx-generic-segmentation"}, + {"gro", "generic-receive-offload", "rx-gro"}, + {"lro", "large-receive-offload", "rx-lro"}, + {"rxvlan", "rx-vlan-offload", "rx-vlan-hw-parse"}, + {"txvlan", "tx-vlan-offload", "tx-vlan-hw-insert"}, + {"ntuple", "ntuple-filters", "rx-ntuple-filter"}, + {"rxhash", "receive-hashing", "rx-hashing"}, +} + +// It maps both short and long aliases to their corresponding match pattern. +var legacyFeaturePatterns map[string]string + +// build the map +func init() { + legacyFeaturePatterns = make(map[string]string) + for _, def := range offloadFlagDefs { + // Note: The pattern is a shell-style "glob", not a true regex. + legacyFeaturePatterns[def.ShortName] = def.KernelPattern + legacyFeaturePatterns[def.LongName] = def.KernelPattern + } +} + +// https://docs.kernel.org/networking/ethtool-netlink.html#features-get +// ETHTOOL_A_FEATURES_HW +// ETHTOOL_A_FEATURES_WANTED +// ETHTOOL_A_FEATURES_ACTIVE +// ETHTOOL_A_FEATURES_NOCHANGE +type ethtoolFeatures struct { + hardware map[string]bool + wanted map[string]bool + active map[string]bool + nochange map[string]bool +} + +func (e ethtoolFeatures) Get(name string) []string { + // check if it exists and is not an alias + if _, ok := e.hardware[name]; ok { + return []string{name} + } + // it can be an alias or multiple features + matchedFeatures := []string{} + pattern, ok := legacyFeaturePatterns[name] + if !ok { + return matchedFeatures + } + for featureName := range e.hardware { + matched, _ := filepath.Match(pattern, featureName) + if matched { + matchedFeatures = append(matchedFeatures, featureName) + } + } + return matchedFeatures +} + +// String provides a pretty-printed, sorted list of all feature maps. +func (e ethtoolFeatures) String() string { + var output strings.Builder + + // This helper function formats and appends a single map to the output string. + appendMap := func(title string, features map[string]bool) { + if len(features) == 0 { + return + } + + // Get and sort the keys for consistent output order. + keys := make([]string, 0, len(features)) + for k := range features { + keys = append(keys, k) + } + sort.Strings(keys) + + // Write the formatted output for this map. + fmt.Fprintf(&output, "%s:\n", title) + for _, key := range keys { + msg := fmt.Sprintf(" %s: %t", key, features[key]) + + // Annotate if the feature is marked as fixed. + if _, ok := e.nochange[key]; ok { + msg += " [fixed]" + } + fmt.Fprintln(&output, msg) + } + fmt.Fprintln(&output) // Add a blank line for spacing between maps. + } + + // Print each map in a structured and sorted way. + appendMap("Hardware-supported features", e.hardware) + appendMap("Wanted features", e.wanted) + appendMap("Active features", e.active) + appendMap("No change features", e.nochange) + + return output.String() +} + +type ethtoolClient struct { + conn *genetlink.Conn + familyID uint16 +} + +// newEthtoolClient handles the initial setup and validation. +func newEthtoolClient(netNS int) (*ethtoolClient, error) { + c, err := genetlink.Dial(&netlink.Config{ + Strict: true, + NetNS: netNS, + }) + if err != nil { + return nil, fmt.Errorf("failed to dial generic netlink: %w", err) + } + + family, err := c.GetFamily(unix.ETHTOOL_GENL_NAME) + if err != nil { + // Clean up the connection if family check fails. + _ = c.Close() + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%q family not available", unix.ETHTOOL_GENL_NAME) + } + return nil, fmt.Errorf("failed to query for family: %w", err) + } + + return ðtoolClient{ + conn: c, + familyID: family.ID, + }, nil +} + +// Close wraps the underlying connection's Close method. +func (c *ethtoolClient) Close() { + c.conn.Close() +} + +// GetFeatures retrieves the device features for a given interface. +func (c *ethtoolClient) GetFeatures(ifaceName string) (*ethtoolFeatures, error) { + msgs, err := c.execute( + unix.ETHTOOL_MSG_FEATURES_GET, + unix.ETHTOOL_A_FEATURES_HEADER, + ifaceName, + ) + if err != nil { + return nil, fmt.Errorf("failed to execute FEATURES_GET command: %w", err) + } + + ethFeatures := ðtoolFeatures{} + // The feature flags are nested inside ETHTOOL_A_FEATURES_HARDWARE. + // We need to parse the response to find it. + for _, msg := range msgs { + ad, err := netlink.NewAttributeDecoder(msg.Data) + if err != nil { + return nil, fmt.Errorf("failed to create attribute decoder: %w", err) + } + var parseErr error + // Iterate through top-level attributes. + for ad.Next() { + switch ad.Type() { + case unix.ETHTOOL_A_FEATURES_HW: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.hardware, parseErr = parseBitset(innerAd) + return parseErr + }) + case unix.ETHTOOL_A_FEATURES_WANTED: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.wanted, parseErr = parseBitset(innerAd) + return parseErr + }) + case unix.ETHTOOL_A_FEATURES_ACTIVE: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.active, parseErr = parseBitset(innerAd) + return parseErr + }) + case unix.ETHTOOL_A_FEATURES_NOCHANGE: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.nochange, parseErr = parseBitset(innerAd) + return parseErr + }) + default: + continue + } + } + if err := ad.Err(); err != nil { + return nil, fmt.Errorf("feature attribute decoder error: %w", err) + } + } + return ethFeatures, nil +} + +// GetPrivateFlags retrieves the device-specific private flags. +func (c *ethtoolClient) GetPrivateFlags(ifaceName string) (map[string]bool, error) { + // For private flags, the ETHTOOL_A_PRIVFLAGS_FLAGS attribute directly contains the bitset. + // The structure is: Message -> ETHTOOL_A_PRIVFLAGS_FLAGS (nested) -> Bitset attributes + msgs, err := c.execute( + unix.ETHTOOL_MSG_PRIVFLAGS_GET, + unix.ETHTOOL_A_PRIVFLAGS_HEADER, + ifaceName, + ) + if err != nil { + return nil, fmt.Errorf("failed to execute PRIVFLAGS_GET command: %w", err) + } + if len(msgs) == 0 { + return nil, errors.New("no private flag data in response") + } + + allFlags := make(map[string]bool) + + for _, msg := range msgs { + ad, err := netlink.NewAttributeDecoder(msg.Data) + if err != nil { + return nil, fmt.Errorf("failed to create attribute decoder for a message: %w", err) + } + + // Iterate through top-level attributes in the message. + for ad.Next() { + if ad.Type() == unix.ETHTOOL_A_PRIVFLAGS_FLAGS { + // The bitset is nested inside ETHTOOL_A_PRIVFLAGS_FLAGS. + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + var parseErr error + allFlags, parseErr = parseBitset(innerAd) + return parseErr + }) + return allFlags, nil + } + } + if err := ad.Err(); err != nil { + return nil, fmt.Errorf("private flags attribute decoder error: %w", err) + } + } + // do not fail if there are not private flags + return allFlags, nil +} + +// SetFeatures sets the device features for a given interface. +func (c *ethtoolClient) SetFeatures(ifaceName string, featuresToSet map[string]bool) error { + features, err := c.executeSet( + unix.ETHTOOL_MSG_FEATURES_SET, + unix.ETHTOOL_A_FEATURES_HEADER, + ifaceName, + unix.ETHTOOL_A_FEATURES_WANTED, + featuresToSet, + ) + if err != nil { + return err + } + klog.V(4).Infof("SetFeatures for %s result %s", ifaceName, features) + + // ETHTOOL_A_FEATURES_WANTED reports the difference between client request and actual result: mask consists of bits which differ between requested features and result (dev->features after the operation) + // value consists of values of these bits in the request (i.e. negated values from resulting features) + if len(features.wanted) > 0 { + return fmt.Errorf("could not set the following features: %#v", features.wanted) + } + // ETHTOOL_A_FEATURES_ACTIVE reports the difference between old and new dev->features: mask + // consists of bits which have changed, values are their values in new dev->features (after the operation). + if len(features.active) != len(featuresToSet) { + klog.V(2).Infof("not all features changed, desired: %#v active: %#v", featuresToSet, features.active) + } + return nil +} + +// SetPrivateFlags sets the device-specific private flags. +func (c *ethtoolClient) SetPrivateFlags(ifaceName string, flagsToSet map[string]bool) error { + _, err := c.executeSet( + unix.ETHTOOL_MSG_PRIVFLAGS_SET, + unix.ETHTOOL_A_PRIVFLAGS_HEADER, + ifaceName, + unix.ETHTOOL_A_PRIVFLAGS_FLAGS, + flagsToSet, + ) + return err +} + +// executeSet handles commands that set flags. +// It encodes a header with the interface name and a data payload containing the bitset of flags. +func (c *ethtoolClient) executeSet(cmd uint8, headerAttributeType uint16, ifaceName string, dataPayloadAttributeType uint16, flagsToSet map[string]bool) (*ethtoolFeatures, error) { + ae := netlink.NewAttributeEncoder() + + // Encode the header (e.g., ETHTOOL_A_FEATURES_HEADER or ETHTOOL_A_PRIVFLAGS_HEADER) + ae.Nested(headerAttributeType, func(nae *netlink.AttributeEncoder) error { + nae.String(unix.ETHTOOL_A_HEADER_DEV_NAME, ifaceName) + return nil + }) + + // Encode the data payload (e.g., ETHTOOL_A_FEATURES_WANTED or ETHTOOL_A_PRIVFLAGS_FLAGS) + ae.Nested(dataPayloadAttributeType, func(nae *netlink.AttributeEncoder) error { + nae.Flag(unix.ETHTOOL_A_BITSET_NOMASK, false) + nae.Nested(unix.ETHTOOL_A_BITSET_BITS, func(nnae *netlink.AttributeEncoder) error { + for name, active := range flagsToSet { + nnae.Nested(unix.ETHTOOL_A_BITSET_BITS_BIT, func(bitEncoder *netlink.AttributeEncoder) error { + bitEncoder.String(unix.ETHTOOL_A_BITSET_BIT_NAME, name) + bitEncoder.Flag(unix.ETHTOOL_A_BITSET_BIT_VALUE, active) + return nil + }) + } + return nil + }) + return nil + }) + + reqData, err := ae.Encode() + if err != nil { + return nil, fmt.Errorf("failed to encode attributes for set operation: %w", err) + } + + req := genetlink.Message{ + Header: genetlink.Header{Command: cmd, Version: unix.ETHTOOL_GENL_VERSION}, + Data: reqData, + } + + msgs, err := c.conn.Execute(req, c.familyID, netlink.Request|netlink.Acknowledge) + if err != nil { + return nil, fmt.Errorf("failed to execute set command %d: %w", cmd, err) + } + // ETHTOOL_MSG_PRIVFLAGS_SET does not return anything + if cmd == unix.ETHTOOL_MSG_PRIVFLAGS_SET { + return nil, nil + } + ethFeatures := ðtoolFeatures{} + // The feature flags are nested inside ETHTOOL_A_FEATURES_HARDWARE. + // We need to parse the response to find it. + for _, msg := range msgs { + ad, err := netlink.NewAttributeDecoder(msg.Data) + if err != nil { + return nil, fmt.Errorf("failed to create attribute decoder: %w", err) + } + var parseErr error + // Iterate through top-level attributes. + for ad.Next() { + switch ad.Type() { + case unix.ETHTOOL_A_FEATURES_HW: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.hardware, parseErr = parseBitset(innerAd) + return parseErr + }) + case unix.ETHTOOL_A_FEATURES_WANTED: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.wanted, parseErr = parseBitset(innerAd) + return parseErr + }) + case unix.ETHTOOL_A_FEATURES_ACTIVE: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.active, parseErr = parseBitset(innerAd) + return parseErr + }) + case unix.ETHTOOL_A_FEATURES_NOCHANGE: + ad.Nested(func(innerAd *netlink.AttributeDecoder) error { + ethFeatures.nochange, parseErr = parseBitset(innerAd) + return parseErr + }) + } + } + if err := ad.Err(); err != nil { + return nil, fmt.Errorf("feature attribute decoder error: %w", err) + } + } + return ethFeatures, nil +} + +// 4. A single, generic execute method to avoid code duplication. +// It builds and sends the request, returning the kernel's response. +func (c *ethtoolClient) execute(cmd uint8, headerType uint16, ifaceName string) ([]genetlink.Message, error) { + ae := netlink.NewAttributeEncoder() + ae.Nested(headerType, func(nae *netlink.AttributeEncoder) error { + nae.String(unix.ETHTOOL_A_HEADER_DEV_NAME, ifaceName) + return nil + }) + + reqData, err := ae.Encode() + if err != nil { + return nil, fmt.Errorf("failed to encode attributes: %w", err) + } + + req := genetlink.Message{ + Header: genetlink.Header{ + Command: cmd, + Version: unix.ETHTOOL_GENL_VERSION, + }, + Data: reqData, + } + + return c.conn.Execute(req, c.familyID, netlink.Request) +} + +// parseBitset decodes a complete set of ethtool bitset attributes. +func parseBitset(ad *netlink.AttributeDecoder) (map[string]bool, error) { + flags := make(map[string]bool) + for ad.Next() { + // The actual flags are nested inside the ETHTOOL_A_BITSET_BITS attribute. + if ad.Type() == unix.ETHTOOL_A_BITSET_BITS { + // Pass the nested decoder to the next level of parsing. + ad.Nested(func(nad *netlink.AttributeDecoder) error { + for nad.Next() { + if nad.Type() == unix.ETHTOOL_A_BITSET_BITS_BIT { + name, active, err := parseBit(nad) + if err != nil { + return err + } + if name != "" { + flags[name] = active + } + } + } + return nad.Err() + }) + return flags, nil + } + } + return flags, ad.Err() +} + +// parseBit decodes a single flag (a bit) from the bitset. +func parseBit(ad *netlink.AttributeDecoder) (name string, active bool, err error) { + ad.Nested(func(nnad *netlink.AttributeDecoder) error { + for nnad.Next() { + switch nnad.Type() { + case unix.ETHTOOL_A_BITSET_BIT_NAME: + name = nnad.String() + case unix.ETHTOOL_A_BITSET_BIT_VALUE: + // The presence of this attribute indicates the flag is active. + active = true + } + } + return nnad.Err() + }) + return name, active, err +} + +// applyEthtoolConfig applies ethtool configurations (features, private flags) to an interface +// within a specified network namespace. +func applyEthtoolConfig(containerNsPath string, ifName string, config *apis.EthtoolConfig) error { + if config == nil { + klog.V(2).Infof("No ethtool configuration to apply for %s in ns %s", ifName, containerNsPath) + return nil + } + + hasFeatures := len(config.Features) > 0 + hasPrivateFlags := len(config.PrivateFlags) > 0 + if !hasFeatures && !hasPrivateFlags { + klog.V(2).Infof("Ethtool configuration for %s in ns %s is empty (no features or private flags).", ifName, containerNsPath) + return nil + } + + targetNs, err := netns.GetFromPath(containerNsPath) + if err != nil { + return fmt.Errorf("failed to get target network namespace from path %s: %w", containerNsPath, err) + } + defer targetNs.Close() + + client, err := newEthtoolClient(int(targetNs)) + if err != nil { + return fmt.Errorf("failed to create ethtool client in namespace %s: %w", containerNsPath, err) + } + defer client.Close() + + var errorList []error + + if hasFeatures { + klog.V(2).Infof("Applying ethtool features for %s in ns %s: %v", ifName, containerNsPath, config.Features) + if err := client.SetFeatures(ifName, config.Features); err != nil { + errorList = append(errorList, fmt.Errorf("failed to set ethtool features for %s: %w", ifName, err)) + } + } + + if hasPrivateFlags { + klog.V(2).Infof("Applying ethtool private flags for %s in ns %s: %v", ifName, containerNsPath, config.PrivateFlags) + if err := client.SetPrivateFlags(ifName, config.PrivateFlags); err != nil { + errorList = append(errorList, fmt.Errorf("failed to set ethtool private flags for %s: %w", ifName, err)) + } + } + + return errors.Join(errorList...) +} diff --git a/pkg/driver/ethtool_test.go b/pkg/driver/ethtool_test.go new file mode 100644 index 00000000..be37e69d --- /dev/null +++ b/pkg/driver/ethtool_test.go @@ -0,0 +1,271 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "crypto/rand" + "fmt" + "os" + "os/exec" + "path" + "runtime" + "strings" + "testing" + + "github.com/google/dranet/pkg/apis" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" +) + +func Test_applyEthtoolConfig(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("Test requires root privileges.") + } + + origns, err := netns.Get() + if err != nil { + t.Fatalf("unexpected error trying to get namespace: %v", err) + } + defer origns.Close() + + rndString := make([]byte, 4) + _, err = rand.Read(rndString) + if err != nil { + t.Errorf("fail to generate random name: %v", err) + } + nsName := fmt.Sprintf("ns%x", rndString) + testNS, err := netns.NewNamed(nsName) + if err != nil { + t.Fatalf("Failed to create network namespace: %v", err) + } + defer netns.DeleteNamed(nsName) + defer testNS.Close() + + // Switch back to the original namespace + netns.Set(origns) + + // Create a dummy interface in the test namespace + nhNs, err := netlink.NewHandleAt(testNS) + if err != nil { + t.Fatalf("fail to open netlink handle: %v", err) + } + defer nhNs.Close() + + loLink, err := nhNs.LinkByName("lo") + if err != nil { + t.Fatalf("Failed to get loopback interface: %v", err) + } + if err := nhNs.LinkSetUp(loLink); err != nil { + t.Fatalf("Failed to set up loopback interface: %v", err) + } + + ifaceName := "dummy0" + // Create a veth pair + la := netlink.NewLinkAttrs() + la.Name = ifaceName + la.Namespace = netlink.NsFd(int(testNS)) + link := &netlink.Dummy{ + LinkAttrs: la, + } + if err := nhNs.LinkAdd(link); err != nil { + t.Fatalf("Failed to add dummy link %s in ns %s: %v", ifaceName, nsName, err) + } + + if err := nhNs.LinkSetUp(link); err != nil { + t.Fatalf("Failed to add veth link %s in ns %s: %v", ifaceName, nsName, err) + } + + client, err := newEthtoolClient(int(testNS)) + if err != nil { + t.Fatalf("failed to create ethtool client in namespace %s: %v", nsName, err) + } + defer client.Close() + + deviceFeatures, err := client.GetFeatures(ifaceName) + if err != nil { + t.Fatalf("can not get features: %v", err) + } + + // t.Logf("Features: %s", features) + + // check against ethtool -k + var ethtoolCmdFeatures map[string]bool + func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + err := netns.Set(testNS) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("ethtool", "-k", ifaceName) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("not able to use ethtool from namespace: %v", err) + } + ethtoolCmdFeatures = ParseEthtoolFeatures(string(output)) + + // Switch back to the original namespace + err = netns.Set(origns) + if err != nil { + t.Fatal(err) + } + }() + + /* + // does not work with dummy interface + privateFlags, err := client.GetPrivateFlags(ifaceName) + if err != nil { + t.Logf("can not get privateFlags: %v", err) + } + */ + + // flip the config for the following config + highdmaValue := ethtoolCmdFeatures["highdma"] + rxGroListValue := ethtoolCmdFeatures["rx-gro-list"] + + // Define the ethtool configuration to apply + config := &apis.EthtoolConfig{ + Features: map[string]bool{ + "highdma": !highdmaValue, + "rx-gro-list": !rxGroListValue, + "tcp-segmentation-offload": false, + "generic-receive-offload": false, + "large-receive-offload": false, + }, + } + + // translate features to the actual kernel names + ethtoolFeatures := map[string]bool{} + for feature, value := range config.Features { + aliases := deviceFeatures.Get(feature) + if len(aliases) == 0 { + t.Errorf("feature %s not supported by interface", feature) + continue + } + for _, alias := range aliases { + ethtoolFeatures[alias] = value + } + } + config.Features = ethtoolFeatures + + t.Logf("EthtoolConfig %#v", config.Features) + + // Apply the ethtool configuration + err = applyEthtoolConfig(path.Join("/run/netns", nsName), ifaceName, config) + if err != nil { + t.Fatalf("applyEthtoolConfig failed: %v", err) + } + + // Check features + _, err = client.GetFeatures(ifaceName) + if err != nil { + t.Fatalf("Failed to get features after applying config: %v", err) + } + + // check against ethtool -k + func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + err := netns.Set(testNS) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("ethtool", "-k", ifaceName) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("not able to use ethtool from namespace: %v", err) + } + ethtoolCmdFeatures = ParseEthtoolFeatures(string(output)) + // Switch back to the original namespace + err = netns.Set(origns) + if err != nil { + t.Fatal(err) + } + }() + + for name, expectedState := range config.Features { + actualState := ethtoolCmdFeatures[name] + if actualState != expectedState { + t.Errorf("Feature %s: expected %v, got %v", name, expectedState, actualState) + } + } + + /* + // does not work with dummy interface + appliedPrivateFlags, err := client.GetPrivateFlags(ifaceName) + if err != nil { + t.Errorf("Failed to get private flags after applying config: %v", err) + } + */ + + // Fail to update fixed features + config = &apis.EthtoolConfig{ + Features: map[string]bool{ + "rx-vlan-filter": true, + "hsr-dup-offload": true, + }, + } + + // Apply the ethtool configuration + err = applyEthtoolConfig(path.Join("/run/netns", nsName), ifaceName, config) + if err == nil { + t.Fatalf("applyEthtoolConfig expected to fail: %v", err) + } +} + +func ParseEthtoolFeatures(output string) map[string]bool { + features := make(map[string]bool) + lines := strings.Split(output, "\n") + + for _, line := range lines { + // Split the line at the colon to separate the feature name from its value. + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + // Skip lines that don't contain a colon, like the header or empty lines. + continue + } + + // Clean up the feature name by trimming whitespace. + name := strings.TrimSpace(parts[0]) + + // The value part may contain the state ("on"/"off") and a "[fixed]" tag. + // We can split by space and just take the first element. + valuePart := strings.TrimSpace(parts[1]) + valueFields := strings.Fields(valuePart) + if len(valueFields) == 0 { + continue + } + valueStr := valueFields[0] + + // Convert the string state "on" or "off" to a boolean. + var state bool + switch valueStr { + case "on": + state = true + case "off": + state = false + default: + continue + } + + features[name] = state + } + + return features +} diff --git a/pkg/driver/hostdevice.go b/pkg/driver/hostdevice.go index 905c6d7d..cb6b7b18 100644 --- a/pkg/driver/hostdevice.go +++ b/pkg/driver/hostdevice.go @@ -77,12 +77,43 @@ func nsAttachNetdev(hostIfName string, containerNsPAth string, interfaceConfig a nameData := nl.NewRtAttr(unix.IFLA_IFNAME, nl.ZeroTerminated(ifName)) req.AddData(nameData) - ifMtu := uint32(attrs.MTU) - if interfaceConfig.MTU > 0 { - ifMtu = uint32(interfaceConfig.MTU) + // Configuration values + if interfaceConfig.MTU != nil { + ifMtu := uint32(*interfaceConfig.MTU) + mtu := nl.NewRtAttr(unix.IFLA_MTU, nl.Uint32Attr(ifMtu)) + req.AddData(mtu) + } + + if interfaceConfig.HardwareAddr != nil { + if hardwareAddr, err := net.ParseMAC(*interfaceConfig.HardwareAddr); err == nil { + hwaddr := nl.NewRtAttr(unix.IFLA_ADDRESS, []byte(hardwareAddr)) + req.AddData(hwaddr) + } + } + + if interfaceConfig.GSOMaxSize != nil { + gsoMaxSize := uint32(*interfaceConfig.GSOMaxSize) + gsoAttr := nl.NewRtAttr(unix.IFLA_GSO_MAX_SIZE, nl.Uint32Attr(gsoMaxSize)) + req.AddData(gsoAttr) + } + + if interfaceConfig.GROMaxSize != nil { + groMaxSize := uint32(*interfaceConfig.GROMaxSize) + groAttr := nl.NewRtAttr(unix.IFLA_GRO_MAX_SIZE, nl.Uint32Attr(groMaxSize)) + req.AddData(groAttr) + } + + if interfaceConfig.GSOIPv4MaxSize != nil { + gsoMaxSize := uint32(*interfaceConfig.GSOIPv4MaxSize) + gsoV4Attr := nl.NewRtAttr(unix.IFLA_GSO_IPV4_MAX_SIZE, nl.Uint32Attr(gsoMaxSize)) + req.AddData(gsoV4Attr) + } + + if interfaceConfig.GROIPv4MaxSize != nil { + groMaxSize := uint32(*interfaceConfig.GROIPv4MaxSize) + groV4Attr := nl.NewRtAttr(unix.IFLA_GRO_IPV4_MAX_SIZE, nl.Uint32Attr(groMaxSize)) + req.AddData(groV4Attr) } - mtu := nl.NewRtAttr(unix.IFLA_MTU, nl.Uint32Attr(ifMtu)) - req.AddData(mtu) val := nl.Uint32Attr(uint32(containerNs)) attr := nl.NewRtAttr(unix.IFLA_NET_NS_FD, val) diff --git a/pkg/driver/hostdevice_test.go b/pkg/driver/hostdevice_test.go new file mode 100644 index 00000000..659c4d0d --- /dev/null +++ b/pkg/driver/hostdevice_test.go @@ -0,0 +1,180 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "crypto/rand" + "fmt" + "os" + "os/exec" + "path" + "runtime" + "strings" + "testing" + + "github.com/google/dranet/pkg/apis" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "k8s.io/utils/ptr" +) + +func Test_nhNetdev(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("Test requires root privileges.") + } + + origns, err := netns.Get() + if err != nil { + t.Fatalf("unexpected error trying to get namespace: %v", err) + } + defer origns.Close() + + rndString := make([]byte, 4) + _, err = rand.Read(rndString) + if err != nil { + t.Errorf("fail to generate random name: %v", err) + } + nsName := fmt.Sprintf("ns%x", rndString) + testNS, err := netns.NewNamed(nsName) + if err != nil { + t.Fatalf("Failed to create network namespace: %v", err) + } + defer netns.DeleteNamed(nsName) + defer testNS.Close() + + // Switch back to the original namespace + netns.Set(origns) + + // Create a dummy interface in the test namespace + nhNs, err := netlink.NewHandleAt(testNS) + if err != nil { + t.Fatalf("fail to open netlink handle: %v", err) + } + defer nhNs.Close() + + loLink, err := nhNs.LinkByName("lo") + if err != nil { + t.Fatalf("Failed to get loopback interface: %v", err) + } + if err := nhNs.LinkSetUp(loLink); err != nil { + t.Fatalf("Failed to set up loopback interface: %v", err) + } + + ifaceName := "testdummy-0" + // Create a veth pair + la := netlink.NewLinkAttrs() + la.Name = ifaceName + link := &netlink.Dummy{ + LinkAttrs: la, + } + if err := netlink.LinkAdd(link); err != nil { + t.Fatalf("Failed to add dummy link %s in ns %s: %v", ifaceName, nsName, err) + } + + t.Cleanup(func() { + link, err := netlink.LinkByName(ifaceName) + if err == nil { + _ = netlink.LinkDel(link) + } + }) + if err := netlink.LinkSetUp(link); err != nil { + t.Fatalf("Failed to add veth link %s in ns %s: %v", ifaceName, nsName, err) + } + config := apis.InterfaceConfig{ + Name: "dranet0", + Addresses: []string{"192.168.7.7/32"}, + MTU: ptr.To[int32](1234), + HardwareAddr: ptr.To("00:11:22:33:44:55"), + GSOMaxSize: ptr.To[int32](1024), + GROMaxSize: ptr.To[int32](1025), + GSOIPv4MaxSize: ptr.To[int32](1026), + GROIPv4MaxSize: ptr.To[int32](1027), + } + + deviceData, err := nsAttachNetdev(ifaceName, path.Join("/run/netns", nsName), config) + if err != nil { + t.Fatalf("fail to attach netdev to namespace: %v", err) + } + + // check against ip lin + func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + err := netns.Set(testNS) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("ip", "-d", "link", "show", config.Name) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("not able to use ethtool from namespace: %v", err) + } + outputStr := string(output) + + if !strings.Contains(outputStr, fmt.Sprintf("mtu %d", *config.MTU)) { + t.Errorf("mtu not changed %s", outputStr) + } + if !strings.Contains(outputStr, fmt.Sprintf("gso_max_size %d", *config.GSOMaxSize)) { + t.Errorf("GSOMaxSize not changed wanted %s got %s", fmt.Sprintf("gso_max_size %d", *config.GSOMaxSize), outputStr) + } + if !strings.Contains(outputStr, fmt.Sprintf("gro_max_size %d", *config.GROMaxSize)) { + t.Errorf("GROMaxSize not changed %s", outputStr) + } + // require iproute 6.3.0+ + // TODO: validate the ip version to check it + // https://github.com/iproute2/iproute2/commit/1dafe448c7a2f2be5dfddd8da250980708a48c41 + /* + if !strings.Contains(outputStr, fmt.Sprintf("gso_ipv4_max_size %d", *config.GSOIPv4MaxSize)) { + t.Errorf("GSOIPv4MaxSize not changed %s", outputStr) + } + if !strings.Contains(outputStr, fmt.Sprintf("gro_ipv4_max_size %d", *config.GROIPv4MaxSize)) { + t.Errorf("GROIPv4MaxSize not changed %s", outputStr) + } + */ + if !strings.Contains(outputStr, fmt.Sprintf("link/ether %s", *config.HardwareAddr)) { + t.Errorf("HardwareAddr not changed %s", outputStr) + } + if *config.HardwareAddr != deviceData.HardwareAddress { + t.Errorf("HardwareAddr not reported") + } + + cmd = exec.Command("ip", "addr", "show", config.Name) + output, err = cmd.CombinedOutput() + if err != nil { + t.Fatalf("not able to use ethtool from namespace: %v", err) + } + outputStr = string(output) + // TODO check reported state + for _, addr := range config.Addresses { + if !strings.Contains(outputStr, addr) { + t.Errorf("address %s not found", addr) + } + } + + // Switch back to the original namespace + err = netns.Set(origns) + if err != nil { + t.Fatal(err) + } + }() + + err = nsDetachNetdev(path.Join("/run/netns", nsName), config.Name, ifaceName) + if err != nil { + t.Fatalf("fail to attach netdev to namespace: %v", err) + } + +} diff --git a/pkg/driver/netnamespace.go b/pkg/driver/netnamespace.go index 0ec08605..278a9b30 100644 --- a/pkg/driver/netnamespace.go +++ b/pkg/driver/netnamespace.go @@ -28,7 +28,7 @@ import ( "github.com/vishvananda/netns" ) -func netnsRouting(containerNsPAth string, ifName string, routeConfig []apis.RouteConfig) error { +func applyRoutingConfig(containerNsPAth string, ifName string, routeConfig []apis.RouteConfig) error { containerNs, err := netns.GetFromPath(containerNsPAth) if err != nil { return err diff --git a/pkg/driver/netnamespace_test.go b/pkg/driver/netnamespace_test.go new file mode 100644 index 00000000..d8273b7e --- /dev/null +++ b/pkg/driver/netnamespace_test.go @@ -0,0 +1,25 @@ +/* +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "testing" +) + +func Test_applyRoutingConfig(t *testing.T) { + // TODO: see hostdevice_test.go and ethtool_test.go +} diff --git a/pkg/driver/nri_hooks.go b/pkg/driver/nri_hooks.go index f1f75521..12ed6f8a 100644 --- a/pkg/driver/nri_hooks.go +++ b/pkg/driver/nri_hooks.go @@ -117,7 +117,7 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) klog.V(2).Infof("RunPodSandbox processing Network device: %s", ifName) // TODO config options to rename the device and pass parameters // use https://github.com/opencontainers/runtime-spec/pull/1271 - networkData, err := nsAttachNetdev(ifName, ns, config.NetDevice) + networkData, err := nsAttachNetdev(ifName, ns, config.Network.Interface) if err != nil { klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", deviceName, ns, err) return fmt.Errorf("error moving network device %s to namespace %s: %v", deviceName, ns, err) @@ -133,9 +133,22 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) WithInterfaceName(networkData.InterfaceName). WithHardwareAddress(networkData.HardwareAddress). WithIPs(networkData.IPs...), - ) - // configure routes - err = netnsRouting(ns, config.NetDevice.Name, config.NetNamespaceRoutes) + ) // End of WithNetworkData + + // The interface name inside the container's namespace. + ifNameInNs := networkData.InterfaceName + + // Apply Ethtool configurations + if config.Network.Ethtool != nil { + err = applyEthtoolConfig(ns, ifNameInNs, config.Network.Ethtool) + if err != nil { + klog.Infof("RunPodSandbox error applying ethtool config for %s in ns %s: %v", ifNameInNs, ns, err) + return fmt.Errorf("error applying ethtool config for %s in ns %s: %v", ifNameInNs, ns, err) + } + } + + // Configure routes + err = applyRoutingConfig(ns, ifNameInNs, config.Network.Routes) if err != nil { klog.Infof("RunPodSandbox error configuring device %s namespace %s routing: %v", deviceName, ns, err) return fmt.Errorf("error configuring device %s routes on namespace %s: %v", deviceName, ns, err) @@ -217,7 +230,7 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox for deviceName, config := range podConfig { ifName := names.GetOriginalName(deviceName) - if err := nsDetachNetdev(ns, config.NetDevice.Name, ifName); err != nil { + if err := nsDetachNetdev(ns, config.Network.Interface.Name, ifName); err != nil { klog.Infof("fail to return network device %s : %v", deviceName, err) } diff --git a/pkg/driver/pod_device_config.go b/pkg/driver/pod_device_config.go index c0f8b369..890c88cc 100644 --- a/pkg/driver/pod_device_config.go +++ b/pkg/driver/pod_device_config.go @@ -28,13 +28,10 @@ import ( // routes for the Pod's network namespace, and RDMA configurations. type PodConfig struct { Claim types.NamespacedName - // NetDevice specifies the configuration for the network interface itself, - // such as its desired name within the Pod, IP addresses, and MTU. - NetDevice apis.InterfaceConfig - // NetNamespaceRoutes lists the routes to be configured within the Pod's - // network namespace, associated with this network device. - NetNamespaceRoutes []apis.RouteConfig + // Network contains all network-related configurations (interface, routes, + // ethtool, sysctl) to be applied for this device in the Pod's namespace. + Network apis.NetworkConfig // RDMADevice holds RDMA-specific configurations if the network device // has associated RDMA capabilities. diff --git a/pkg/driver/pod_device_config_test.go b/pkg/driver/pod_device_config_test.go index fe3474d7..ce8669c4 100644 --- a/pkg/driver/pod_device_config_test.go +++ b/pkg/driver/pod_device_config_test.go @@ -41,9 +41,14 @@ func TestPodConfigStore_SetAndGet(t *testing.T) { podUID := types.UID("test-pod-uid-1") deviceName := "eth0" config := PodConfig{ - NetDevice: apis.InterfaceConfig{Name: "eth0-pod"}, - NetNamespaceRoutes: []apis.RouteConfig{ - {Destination: "0.0.0.0/0", Gateway: "192.168.1.1"}, + Network: apis.NetworkConfig{ + Interface: apis.InterfaceConfig{Name: "eth0-pod"}, + Routes: []apis.RouteConfig{ + {Destination: "0.0.0.0/0", Gateway: "192.168.1.1"}, + }, + Ethtool: &apis.EthtoolConfig{ + Features: map[string]bool{"tx-checksumming": true}, + }, }, RDMADevice: RDMAConfig{LinkDev: "mlx5_0"}, } @@ -77,7 +82,12 @@ func TestPodConfigStore_SetAndGet(t *testing.T) { } // Test overwriting - newConfig := PodConfig{NetDevice: apis.InterfaceConfig{Name: "eth0-new"}} + newConfig := PodConfig{ + Network: apis.NetworkConfig{ + Interface: apis.InterfaceConfig{Name: "eth0-new"}, + Ethtool: &apis.EthtoolConfig{PrivateFlags: map[string]bool{"custom-flag": false}}, + }, + } store.Set(podUID, deviceName, newConfig) retrievedConfig, found = store.Get(podUID, deviceName) if !found { @@ -94,9 +104,9 @@ func TestPodConfigStore_DeletePod(t *testing.T) { podUID2 := types.UID("test-pod-uid-2") dev1 := "eth0" dev2 := "eth1" - config1 := PodConfig{NetDevice: apis.InterfaceConfig{Name: "p1eth0"}} - config2 := PodConfig{NetDevice: apis.InterfaceConfig{Name: "p1eth1"}} - config3 := PodConfig{NetDevice: apis.InterfaceConfig{Name: "p2eth0"}} + config1 := PodConfig{Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p1eth0"}}} + config2 := PodConfig{Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p1eth1"}}} + config3 := PodConfig{Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p2eth0"}}} store.Set(podUID1, dev1, config1) store.Set(podUID1, dev2, config2) @@ -131,9 +141,9 @@ func TestPodConfigStore_GetPodConfigs(t *testing.T) { podUID2 := types.UID("test-pod-uid-2") dev1 := "eth0" dev2 := "eth1" - config1 := PodConfig{NetDevice: apis.InterfaceConfig{Name: "p1eth0"}} - config2 := PodConfig{NetDevice: apis.InterfaceConfig{Name: "p1eth1"}} - config3 := PodConfig{NetDevice: apis.InterfaceConfig{Name: "p2eth0"}} + config1 := PodConfig{Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p1eth0"}}} + config2 := PodConfig{Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p1eth1"}}} + config3 := PodConfig{Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p2eth0"}}} store.Set(podUID1, dev1, config1) store.Set(podUID1, dev2, config2) @@ -177,7 +187,7 @@ func TestPodConfigStore_ThreadSafety(t *testing.T) { defer wg.Done() podUID := types.UID(fmt.Sprintf("pod-%d", i)) deviceName := fmt.Sprintf("eth%d", i%2) - config := PodConfig{NetDevice: apis.InterfaceConfig{Name: fmt.Sprintf("dev-%d", i)}} + config := PodConfig{Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: fmt.Sprintf("dev-%d", i)}}} store.Set(podUID, deviceName, config) retrieved, _ := store.Get(podUID, deviceName) if !reflect.DeepEqual(retrieved, config) { @@ -206,10 +216,10 @@ func TestPodConfigStore_DeleteClaim(t *testing.T) { dev1 := "eth0" dev2 := "eth1" - config1_1 := PodConfig{Claim: claim1, NetDevice: apis.InterfaceConfig{Name: "p1d1c1"}} // Pod1, Dev1, Claim1 - config1_2 := PodConfig{Claim: claim1, NetDevice: apis.InterfaceConfig{Name: "p1d2c1"}} // Pod1, Dev2, Claim1 - config2_1 := PodConfig{Claim: claim1, NetDevice: apis.InterfaceConfig{Name: "p2d1c1"}} // Pod2, Dev1, Claim1 - config3_1 := PodConfig{Claim: claim2, NetDevice: apis.InterfaceConfig{Name: "p3d1c2"}} // Pod3, Dev1, Claim2 + config1_1 := PodConfig{Claim: claim1, Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p1d1c1"}}} // Pod1, Dev1, Claim1 + config1_2 := PodConfig{Claim: claim1, Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p1d2c1"}}} // Pod1, Dev2, Claim1 + config2_1 := PodConfig{Claim: claim1, Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p2d1c1"}}} // Pod2, Dev1, Claim1 + config3_1 := PodConfig{Claim: claim2, Network: apis.NetworkConfig{Interface: apis.InterfaceConfig{Name: "p3d1c2"}}} // Pod3, Dev1, Claim2 tests := []struct { name string diff --git a/tests/e2e.bats b/tests/e2e.bats index eb87b3f0..56613c73 100644 --- a/tests/e2e.bats +++ b/tests/e2e.bats @@ -6,7 +6,7 @@ kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim.yaml - kubectl wait --timeout=2m --for=condition=ready pods -l app=pod + kubectl wait --timeout=30s --for=condition=ready pods -l app=pod run kubectl exec pod1 -- ip addr show eth99 [ "$status" -eq 0 ] [[ "$output" == *"169.254.169.13"* ]] @@ -25,7 +25,7 @@ kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim.yaml - kubectl wait --timeout=2m --for=condition=ready pods -l app=pod + kubectl wait --timeout=30s --for=condition=ready pods -l app=pod run kubectl exec pod1 -- ip addr show eth99 [ "$status" -eq 0 ] [[ "$output" == *"169.254.169.13"* ]] @@ -42,7 +42,7 @@ docker exec "$CLUSTER_NAME"-worker2 bash -c "ip addr add 169.254.169.14/32 dev dummy1" kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaimtemplate.yaml - kubectl wait --timeout=2m --for=condition=ready pods -l app=MyApp + kubectl wait --timeout=30s --for=condition=ready pods -l app=MyApp POD_NAME=$(kubectl get pods -l app=MyApp -o name) run kubectl exec $POD_NAME -- ip addr show dummy1 [ "$status" -eq 0 ] @@ -61,7 +61,7 @@ kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_route.yaml - kubectl wait --timeout=2m --for=condition=ready pods -l app=pod + kubectl wait --timeout=30s --for=condition=ready pods -l app=pod run kubectl exec pod3 -- ip addr show eth99 [ "$status" -eq 0 ] [[ "$output" == *"169.254.169.13"* ]] @@ -82,10 +82,68 @@ @test "test metric server is up and operating on host" { output=$(kubectl \ run -i test-metrics \ - --image registry.k8s.io/e2e-test-images/agnhost:2.39 \ + --image registry.k8s.io/e2e-test-images/agnhost:2.54 \ --overrides='{"spec": {"hostNetwork": true}}' \ --restart=Never \ --command \ -- sh -c "curl --silent localhost:9177/metrics | grep process_start_time_seconds >/dev/null && echo ok || echo fail") test "$output" = "ok" -} \ No newline at end of file +} + + +@test "validate advanced network configurations with dummy" { + docker exec "$CLUSTER_NAME"-worker bash -c "ip link add dummy3 type dummy" + docker exec "$CLUSTER_NAME"-worker bash -c "ip link set up dev dummy3" + + kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml + kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_advanced.yaml + + # Wait for the pod to become ready + kubectl wait --for=condition=ready pod/pod-advanced-cfg --timeout=30s + + # Validate mtu and hardware address + run kubectl exec pod-advanced-cfg -- ip addr show dranet0 + [ "$status" -eq 0 ] + [[ "$output" == *"169.254.169.14/24"* ]] + [[ "$output" == *"mtu 4321"* ]] + [[ "$output" == *"00:11:22:33:44:55"* ]] + + # Validate ethtool settings inside the pod for interface dranet0 + run kubectl exec pod-advanced-cfg -- ash -c "apk add ethtool && ethtool -k dranet0" + [ "$status" -eq 0 ] + [[ "$output" == *"tcp-segmentation-offload: off"* ]] + [[ "$output" == *"generic-receive-offload: off"* ]] + [[ "$output" == *"large-receive-offload: off"* ]] + + # Cleanup the resources for this test + kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_advanced.yaml + kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml +} + +# Test case for validating Big TCP configurations. +@test "validate big tcp network configurations on dummy interface" { + docker exec "$CLUSTER_NAME"-worker2 bash -c "ip link add dummy4 type dummy" + docker exec "$CLUSTER_NAME"-worker2 bash -c "ip link set up dev dummy4" + + kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml + kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_bigtcp.yaml + kubectl wait --for=condition=ready pod/pod-bigtcp-test --timeout=300s + + run kubectl exec pod-bigtcp-test -- ip -d link show dranet1 + [ "$status" -eq 0 ] + [[ "$output" == *"mtu 8896"* ]] + [[ "$output" == *"gso_max_size 65536"* ]] + [[ "$output" == *"gro_max_size 65536"* ]] + [[ "$output" == *"gso_ipv4_max_size 65536"* ]] + [[ "$output" == *"gro_ipv4_max_size 65536"* ]] + + run kubectl exec pod-bigtcp-test -- ash -c "apk add ethtool && ethtool -k dranet1" + [ "$status" -eq 0 ] + [[ "$output" == *"tcp-segmentation-offload: on"* ]] + [[ "$output" == *"generic-receive-offload: on"* ]] + [[ "$output" == *"large-receive-offload: off"* ]] + + # Cleanup the resources for this test + kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_bigtcp.yaml + kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml +}