From 2d444eae9524f84a743187861c89abcd09ecb9e2 Mon Sep 17 00:00:00 2001 From: "Huabing (Robin) Zhao" Date: Mon, 14 Apr 2025 04:20:42 +0000 Subject: [PATCH] impl dynamic resolver backend type Signed-off-by: Huabing (Robin) Zhao fix test Signed-off-by: Huabing (Robin) Zhao add comment Signed-off-by: Huabing (Robin) Zhao add e2e test Signed-off-by: Huabing (Robin) Zhao Update internal/gatewayapi/backend.go Co-authored-by: Arko Dasgupta Signed-off-by: Huabing (Robin) Zhao fix reason Signed-off-by: Huabing (Robin) Zhao fix tests Signed-off-by: Huabing (Robin) Zhao improve status messages Signed-off-by: Huabing (Robin) Zhao --- internal/gatewayapi/backend.go | 10 ++ internal/gatewayapi/route.go | 27 +++ ...ic-resolver-with-mutliple-backends.in.yaml | 54 ++++++ ...c-resolver-with-mutliple-backends.out.yaml | 167 ++++++++++++++++++ .../httproute-dynamic-resolver.in.yaml | 41 +++++ .../httproute-dynamic-resolver.out.yaml | 149 ++++++++++++++++ internal/ir/xds.go | 12 +- internal/xds/translator/cluster.go | 109 ++++++++---- internal/xds/translator/route.go | 2 +- .../xds-ir/http-route-dynamic-resolver.yaml | 34 ++++ .../http-route-dynamic-resolver.clusters.yaml | 18 ++ ...http-route-dynamic-resolver.endpoints.yaml | 1 + ...http-route-dynamic-resolver.listeners.yaml | 70 ++++++++ .../http-route-dynamic-resolver.routes.yaml | 29 +++ internal/xds/translator/translator.go | 10 +- ...tproute-with-dynamic-resolver-backend.yaml | 49 +++++ ...httproute_with_dynamic_resolver_backend.go | 65 +++++++ 17 files changed, 805 insertions(+), 42 deletions(-) create mode 100644 internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.out.yaml create mode 100644 internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.routes.yaml create mode 100644 test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml create mode 100644 test/e2e/tests/httproute_with_dynamic_resolver_backend.go diff --git a/internal/gatewayapi/backend.go b/internal/gatewayapi/backend.go index ae184ee70e..715504d495 100644 --- a/internal/gatewayapi/backend.go +++ b/internal/gatewayapi/backend.go @@ -40,6 +40,16 @@ func (t *Translator) ProcessBackends(backends []*egv1a1.Backend) []*egv1a1.Backe } func validateBackend(backend *egv1a1.Backend) status.Error { + if backend.Spec.Type != nil && + *backend.Spec.Type == egv1a1.BackendTypeDynamicResolver { + if len(backend.Spec.Endpoints) > 0 || len(backend.Spec.AppProtocols) > 0 { + return status.NewRouteStatusError( + fmt.Errorf("DynamicResolver type cannot have endpoints or appProtocols specified"), + status.RouteReasonInvalidBackendRef, + ) + } + } + for _, ep := range backend.Spec.Endpoints { if ep.FQDN != nil { hostname := ep.FQDN.Hostname diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index ee75ac8003..af47eae578 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -220,6 +220,7 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe destName := irRouteDestinationName(httpRoute, ruleIdx) allDs := []*ir.DestinationSetting{} failedProcessDestination := false + hasDynamicResolver := false // process each backendRef, and calculate the destination settings for this rule for i, backendRef := range rule.BackendRefs { @@ -240,6 +241,11 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe } allDs = append(allDs, ds) + // check if there is a dynamic resolver in the backendRefs + if ds.IsDynamicResolver { + hasDynamicResolver = true + } + if !t.IsEnvoyServiceRouting(envoyProxy) && len(ds.Endpoints) > 0 && ds.AddressType != nil { dstAddrTypeSet.Insert(*ds.AddressType) } @@ -266,6 +272,12 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe irRoute.DirectResponse = &ir.CustomResponse{ StatusCode: ptr.To(uint32(500)), } + // A route can only have one destination if this destination is a dynamic resolver, because the behavior of + // multiple destinations with one being a dynamic resolver just doesn't make sense. + case hasDynamicResolver && len(rule.BackendRefs) > 1: + irRoute.DirectResponse = &ir.CustomResponse{ + StatusCode: ptr.To(uint32(500)), + } default: destination.Name = destName destination.Settings = allDs @@ -273,6 +285,15 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe } } + if hasDynamicResolver && len(rule.BackendRefs) > 1 { + errs.Add(status.NewRouteStatusError( + fmt.Errorf( + "failed to process route rule %d: dynamic resolver is not supported for multiple backendRefs", + ruleIdx), + status.RouteReasonInvalidBackendRef, + )) + } + // TODO: support mixed endpointslice address type between backendRefs if !t.IsEnvoyServiceRouting(envoyProxy) && (dstAddrTypeSet.Len() > 1 || dstAddrTypeSet.Has(ir.MIXED)) { errs.Add(status.NewRouteStatusError( @@ -1823,6 +1844,12 @@ func (t *Translator) processBackendDestinationSetting( backend := resources.GetBackend(backendNamespace, string(backendRef.Name)) ds := &ir.DestinationSetting{Name: name} + // There is only one backend if it is a dynamic resolver + if backend.Spec.Type != nil && *backend.Spec.Type == egv1a1.BackendTypeDynamicResolver { + ds.IsDynamicResolver = true + return ds + } + for _, bep := range backend.Spec.Endpoints { var irde *ir.DestinationEndpoint switch { diff --git a/internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.in.yaml b/internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.in.yaml new file mode 100644 index 0000000000..cbb462e0a8 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.in.yaml @@ -0,0 +1,54 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + - group: gateway.envoyproxy.io + kind: Backend + name: backend-2 +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-1 + namespace: default + spec: + type: DynamicResolver +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-2 + namespace: default + spec: + endpoints: + - ip: + address: 1.1.1.1 + port: 3001 diff --git a/internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.out.yaml b/internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.out.yaml new file mode 100644 index 0000000000..3570d0b795 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-dynamic-resolver-with-mutliple-backends.out.yaml @@ -0,0 +1,167 @@ +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-1 + namespace: default + spec: + type: DynamicResolver + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-2 + namespace: default + spec: + endpoints: + - ip: + address: 1.1.1.1 + port: 3001 + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + - group: gateway.envoyproxy.io + kind: Backend + name: backend-2 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: 'Failed to process route rule 0: dynamic resolver is not supported + for multiple backendRefs.' + reason: InvalidBackendRef + status: "False" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - directResponse: + statusCode: 500 + hostname: gateway.envoyproxy.io + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/-1/gateway_envoyproxy_io + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml b/internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml new file mode 100644 index 0000000000..3cd1bbb8ef --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml @@ -0,0 +1,41 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + name: backend-1 + namespace: default + spec: + type: DynamicResolver diff --git a/internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml b/internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml new file mode 100644 index 0000000000..da6d594075 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml @@ -0,0 +1,149 @@ +backends: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: Backend + metadata: + creationTimestamp: null + name: backend-1 + namespace: default + spec: + type: DynamicResolver + status: + conditions: + - lastTransitionTime: null + message: The Backend was accepted + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-1 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/default/httproute-1/rule/0 + settings: + - isDynamicResolver: true + name: httproute/default/httproute-1/rule/0/backend/0 + weight: 1 + hostname: gateway.envoyproxy.io + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/-1/gateway_envoyproxy_io + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 7139d0dc5a..acf491ac1e 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1500,9 +1500,12 @@ func (r *RouteDestination) ToBackendWeights() *BackendWeights { continue } - if len(s.Endpoints) > 0 { + switch { + case s.IsDynamicResolver: // Dynamic resolver has no endpoints w.Valid += *s.Weight - } else { + case len(s.Endpoints) > 0: + w.Valid += *s.Weight + default: w.Invalid += *s.Weight } } @@ -1515,6 +1518,11 @@ func (r *RouteDestination) ToBackendWeights() *BackendWeights { type DestinationSetting struct { // Name of the setting Name string `json:"name" yaml:"name"` + + // IsDynamicResolver specifies whether the destination is a dynamic resolver. + // A dynamic resolver is a destination that is resolved dynamically using the request's host header. + IsDynamicResolver bool `json:"isDynamicResolver,omitempty" yaml:"isDynamicResolver,omitempty"` + // Weight associated with this destination, // invalid endpoints are represents with a // non-zero weight with an empty endpoints list diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index 4957ec2fd9..bfefee5f8e 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -14,6 +14,8 @@ import ( clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + dfpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3" + commondfpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/common/dynamic_forward_proxy/v3" codecv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/upstream_codec/v3" hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" preservecasev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/http/header_formatters/preserve_case/v3" @@ -68,6 +70,7 @@ type EndpointType int const ( EndpointTypeDNS EndpointType = iota EndpointTypeStatic + EndpointTypeDynamicResolver ) func buildEndpointType(settings []*ir.DestinationSetting) EndpointType { @@ -77,6 +80,10 @@ func buildEndpointType(settings []*ir.DestinationSetting) EndpointType { return EndpointTypeStatic } + if settings[0].IsDynamicResolver { + return EndpointTypeDynamicResolver + } + addrType := settings[0].AddressType if addrType != nil && *addrType == ir.FQDN { @@ -166,7 +173,6 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { } for i, ds := range args.settings { - // If zone aware routing is enabled we update the cluster lb config if ds.ZoneAwareRoutingEnabled { cluster.CommonLbConfig.LocalityConfigSpecifier = &clusterv3.Cluster_CommonLbConfig_ZoneAwareLbConfig_{ZoneAwareLbConfig: &clusterv3.Cluster_CommonLbConfig_ZoneAwareLbConfig{}} @@ -194,36 +200,6 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { } } - if args.endpointType == EndpointTypeStatic { - cluster.ClusterDiscoveryType = &clusterv3.Cluster_Type{Type: clusterv3.Cluster_EDS} - cluster.EdsClusterConfig = &clusterv3.Cluster_EdsClusterConfig{ - ServiceName: args.name, - EdsConfig: &corev3.ConfigSource{ - ResourceApiVersion: resource.DefaultAPIVersion, - ConfigSourceSpecifier: &corev3.ConfigSource_Ads{ - Ads: &corev3.AggregatedConfigSource{}, - }, - }, - } - // Dont wait for a health check to determine health and remove these endpoints - // if the endpoint has been removed via EDS by the control plane - cluster.IgnoreHealthOnHostRemoval = true - } else { - cluster.ClusterDiscoveryType = &clusterv3.Cluster_Type{Type: clusterv3.Cluster_STRICT_DNS} - cluster.DnsRefreshRate = durationpb.New(30 * time.Second) - cluster.RespectDnsTtl = true - if args.dns != nil { - if args.dns.DNSRefreshRate != nil { - if args.dns.DNSRefreshRate.Duration > 0 { - cluster.DnsRefreshRate = durationpb.New(args.dns.DNSRefreshRate.Duration) - } - } - if args.dns.RespectDNSTTL != nil { - cluster.RespectDnsTtl = ptr.Deref(args.dns.RespectDNSTTL, true) - } - } - } - // build common, HTTP/1 and HTTP/2 protocol options for cluster epo, secrets, err := buildTypedExtensionProtocolOptions(args) if err != nil { @@ -235,9 +211,10 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { // Set Load Balancer policy //nolint:gocritic - if args.loadBalancer == nil { + switch { + case args.loadBalancer == nil: cluster.LbPolicy = clusterv3.Cluster_LEAST_REQUEST - } else if args.loadBalancer.LeastRequest != nil { + case args.loadBalancer.LeastRequest != nil: cluster.LbPolicy = clusterv3.Cluster_LEAST_REQUEST if args.loadBalancer.LeastRequest.SlowStart != nil { if args.loadBalancer.LeastRequest.SlowStart.Window != nil { @@ -250,7 +227,7 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { } } } - } else if args.loadBalancer.RoundRobin != nil { + case args.loadBalancer.RoundRobin != nil: cluster.LbPolicy = clusterv3.Cluster_ROUND_ROBIN if args.loadBalancer.RoundRobin.SlowStart != nil && args.loadBalancer.RoundRobin.SlowStart.Window != nil { cluster.LbConfig = &clusterv3.Cluster_RoundRobinLbConfig_{ @@ -261,11 +238,10 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { }, } } - } else if args.loadBalancer.Random != nil { + case args.loadBalancer.Random != nil: cluster.LbPolicy = clusterv3.Cluster_RANDOM - } else if args.loadBalancer.ConsistentHash != nil { + case args.loadBalancer.ConsistentHash != nil: cluster.LbPolicy = clusterv3.Cluster_MAGLEV - if args.loadBalancer.ConsistentHash.TableSize != nil { cluster.LbConfig = &clusterv3.Cluster_MaglevLbConfig_{ MaglevLbConfig: &clusterv3.Cluster_MaglevLbConfig{ @@ -288,6 +264,65 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { if args.tcpkeepalive != nil { cluster.UpstreamConnectionOptions = buildXdsClusterUpstreamOptions(args.tcpkeepalive) } + + switch args.endpointType { + case EndpointTypeDynamicResolver: + dnsCacheConfig := &commondfpv3.DnsCacheConfig{ + Name: args.name, + DnsLookupFamily: dnsLookupFamily, + } + dnsCacheConfig.DnsRefreshRate = durationpb.New(30 * time.Second) + if args.dns != nil { + if args.dns.DNSRefreshRate != nil { + if args.dns.DNSRefreshRate.Duration > 0 { + dnsCacheConfig.DnsRefreshRate = durationpb.New(args.dns.DNSRefreshRate.Duration) + } + } + } + + dfp := &dfpv3.ClusterConfig{ + ClusterImplementationSpecifier: &dfpv3.ClusterConfig_DnsCacheConfig{ + DnsCacheConfig: dnsCacheConfig, + }, + } + dfpAny, err := proto.ToAnyWithValidation(dfp) + if err != nil { + return nil, err + } + cluster.ClusterDiscoveryType = &clusterv3.Cluster_ClusterType{ClusterType: &clusterv3.Cluster_CustomClusterType{ + Name: args.name, + TypedConfig: dfpAny, + }} + cluster.LbPolicy = clusterv3.Cluster_CLUSTER_PROVIDED + case EndpointTypeStatic: + cluster.ClusterDiscoveryType = &clusterv3.Cluster_Type{Type: clusterv3.Cluster_EDS} + cluster.EdsClusterConfig = &clusterv3.Cluster_EdsClusterConfig{ + ServiceName: args.name, + EdsConfig: &corev3.ConfigSource{ + ResourceApiVersion: resource.DefaultAPIVersion, + ConfigSourceSpecifier: &corev3.ConfigSource_Ads{ + Ads: &corev3.AggregatedConfigSource{}, + }, + }, + } + // Dont wait for a health check to determine health and remove these endpoints + // if the endpoint has been removed via EDS by the control plane + cluster.IgnoreHealthOnHostRemoval = true + default: + cluster.ClusterDiscoveryType = &clusterv3.Cluster_Type{Type: clusterv3.Cluster_STRICT_DNS} + cluster.DnsRefreshRate = durationpb.New(30 * time.Second) + cluster.RespectDnsTtl = true + if args.dns != nil { + if args.dns.DNSRefreshRate != nil { + if args.dns.DNSRefreshRate.Duration > 0 { + cluster.DnsRefreshRate = durationpb.New(args.dns.DNSRefreshRate.Duration) + } + } + if args.dns.RespectDNSTTL != nil { + cluster.RespectDnsTtl = ptr.Deref(args.dns.RespectDNSTTL, true) + } + } + } return &buildClusterResult{ cluster: cluster, secrets: secrets, diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index 033f8851bc..1021e2c5b7 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -270,7 +270,7 @@ func buildXdsWeightedRouteAction(backendWeights *ir.BackendWeights, settings []* } for _, destinationSetting := range settings { - if len(destinationSetting.Endpoints) > 0 { + if len(destinationSetting.Endpoints) > 0 || destinationSetting.IsDynamicResolver { // Dynamic resolver has no endpoints validCluster := &routev3.WeightedCluster_ClusterWeight{ Name: destinationSetting.Name, Weight: &wrapperspb.UInt32Value{Value: *destinationSetting.Weight}, diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml new file mode 100644 index 0000000000..5d7939120e --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml @@ -0,0 +1,34 @@ +http: +- address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/default/httproute-1/rule/0 + settings: + - isDynamicResolver: true + name: httproute/default/httproute-1/rule/0/backend/0 + weight: 1 + hostname: gateway.envoyproxy.io + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/-1/gateway_envoyproxy_io +readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml new file mode 100644 index 0000000000..29b8be2f44 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml @@ -0,0 +1,18 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + clusterType: + name: httproute/default/httproute-1/rule/0 + typedConfig: + '@type': type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dnsCacheConfig: + dnsLookupFamily: V4_PREFERRED + dnsRefreshRate: 30s + name: httproute/default/httproute-1/rule/0 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + lbPolicy: CLUSTER_PROVIDED + name: httproute/default/httproute-1/rule/0 + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.endpoints.yaml new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.endpoints.yaml @@ -0,0 +1 @@ +[] diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.listeners.yaml new file mode 100644 index 0000000000..9677ab7557 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.listeners.yaml @@ -0,0 +1,70 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 19003 + bypassOverloadManager: true + filterChains: + - filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.health_check + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + headers: + - name: :path + stringMatch: + exact: /ready + passThroughMode: false + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + routeConfig: + name: ready_route + virtualHosts: + - domains: + - '*' + name: ready_route + routes: + - directResponse: + status: 500 + match: + prefix: / + statPrefix: eg-ready-http + name: envoy-gateway-proxy-ready-0.0.0.0-19003 +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: envoy-gateway/gateway-1/http + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: envoy-gateway/gateway-1/http + name: envoy-gateway/gateway-1/http + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.routes.yaml new file mode 100644 index 0000000000..b4e7a9e655 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.routes.yaml @@ -0,0 +1,29 @@ +- ignorePortInHostMatching: true + name: envoy-gateway/gateway-1/http + virtualHosts: + - domains: + - gateway.envoyproxy.io + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http/gateway_envoyproxy_io + routes: + - match: + prefix: / + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/-1/gateway_envoyproxy_io + route: + cluster: httproute/default/httproute-1/rule/0 + upgradeConfigs: + - upgradeType: websocket diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index 4862a0a3eb..db2db5cbe2 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -940,13 +940,19 @@ func addXdsCluster(tCtx *types.ResourceVersionTable, args *xdsClusterArgs) error } } // Use EDS for static endpoints - if args.endpointType == EndpointTypeStatic { + switch args.endpointType { + case EndpointTypeStatic: if err := tCtx.AddXdsResource(resourcev3.EndpointType, xdsEndpoints); err != nil { return err } - } else { + case EndpointTypeDNS: xdsCluster.LoadAssignment = xdsEndpoints + case EndpointTypeDynamicResolver: + // Dynamic resolver has no endpoints + // This assignment is not necessary, but it is added for clarity. + xdsCluster.LoadAssignment = nil } + if err := tCtx.AddXdsResource(resourcev3.ClusterType, xdsCluster); err != nil { return err } diff --git a/test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml b/test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml new file mode 100644 index 0000000000..abb97d608c --- /dev/null +++ b/test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml @@ -0,0 +1,49 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute-with-dynamic-resolver-backend + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-dynamic-resolver +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: backend-dynamic-resolver + namespace: gateway-conformance-infra +spec: + type: DynamicResolver +--- +apiVersion: v1 +kind: Service +metadata: + name: test-service-foo + namespace: gateway-conformance-infra +spec: + selector: + app: infra-backend-v1 + ports: + - protocol: TCP + port: 80 + name: http11 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: test-service-bar + namespace: gateway-conformance-infra +spec: + selector: + app: infra-backend-v1 + ports: + - protocol: TCP + port: 80 + name: http11 + targetPort: 3000 diff --git a/test/e2e/tests/httproute_with_dynamic_resolver_backend.go b/test/e2e/tests/httproute_with_dynamic_resolver_backend.go new file mode 100644 index 0000000000..dfd1fe3468 --- /dev/null +++ b/test/e2e/tests/httproute_with_dynamic_resolver_backend.go @@ -0,0 +1,65 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e + +package tests + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +func init() { + ConformanceTests = append(ConformanceTests, EnvoyGatewayDynamicResolverBackendTest) +} + +var EnvoyGatewayDynamicResolverBackendTest = suite.ConformanceTest{ + ShortName: "EnvoyGatewayDynamicResolverBackend", + Description: "Routes with a backend ref to a dynamic resolver backend", + Manifests: []string{ + "testdata/httproute-with-dynamic-resolver-backend.yaml", + }, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "httproute-with-dynamic-resolver-backend", Namespace: ns} + gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + BackendMustBeAccepted(t, suite.Client, types.NamespacedName{Name: "backend-dynamic-resolver", Namespace: ns}) + + t.Run("route to service foo", func(t *testing.T) { + expectedResponse := http.ExpectedResponse{ + Request: http.Request{ + Host: "test-service-foo.gateway-conformance-infra", + Path: "/", + }, + Response: http.Response{ + StatusCode: 200, + }, + Namespace: ns, + } + + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + }) + t.Run("route to service bar", func(t *testing.T) { + expectedResponse := http.ExpectedResponse{ + Request: http.Request{ + Host: "test-service-bar.gateway-conformance-infra", + Path: "/", + }, + Response: http.Response{ + StatusCode: 200, + }, + Namespace: ns, + } + + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + }) + }, +}