From 0bfa4c0f5d1cda3cd4d29c984abb908f7f87de1e Mon Sep 17 00:00:00 2001 From: danehans Date: Mon, 6 Jun 2022 14:47:51 -0700 Subject: [PATCH 1/6] Adds Initial Config API Design Doc Signed-off-by: danehans --- docs/design/CONFIG_API.md | 231 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 docs/design/CONFIG_API.md diff --git a/docs/design/CONFIG_API.md b/docs/design/CONFIG_API.md new file mode 100644 index 0000000000..2a928495cc --- /dev/null +++ b/docs/design/CONFIG_API.md @@ -0,0 +1,231 @@ +Config API Design +=================== + +## Motivation + +[Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway. This configuration is referred +to as the "Bootstrap Config" in the Envoy Gateway high-level [design doc][design_doc]. + +## Goals + +* Define an API for configuring Envoy Gateway. +* Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. + +## Non-Goals + +* Implementation of the API. +* Define the `status` subresource of the configuration API. + +## Proposal + +Utilize the [Kubebuilder][kubebuilder] framework to build and manage Envoy Gateway APIs. Kubebuilder facilitates the +following developer workflow for building APIs: + +- Create one or more resource APIs as CRDs and then add fields to the resources. +- Implement reconcile loops in controllers and watch additional resources. +- Test by running against a cluster (self-installs CRDs and starts controllers automatically). +- Update bootstrapped integration tests to test new fields and business logic. +- Build and publish a container from a Dockerfile. + +After installing the latest Kubebuilder release, create the `BootstrapConfig` API: +```shell +kubebuilder create api --group envoygateway.io --version v1alpha1 --kind BootstrapConfig +``` + +The `v1alpha1` version and `envoygateway.io` API group get generated: +```go +// gateway/api/v1alpha1/doc.go + +// Package v1alpha1 contains API Schema definitions for the envoygateway.io API group. +// +// +groupName=config.envoygateway.io +package v1alpha1 +``` + +The initial `BootstrapConfig` API being proposed: +```go +// gateway/api/v1alpha1/bootstrapconfig_types.go + +// BootstrapConfigSpec defines the desired state of BootstrapConfig. +type BootstrapConfigSpec struct { + // EnabledServices defines the services to enable. If unspecified, + // defaults to enabling all supported services. + // + // +optional + // +kubebuilder:validation:MaxItems=12 + // +kubebuilder:validation:Enum=XdsServer;Provisioner + // +kubebuilder:default=XdsServer;Provisioner + EnabledServices []ServiceType `json:"enabledServices,omitempty"` + + // XDSServer contains parameters of the xDS server. If unspecified, + // the xDS server will use default configuration settings. + // + // +optional + XDSServer *XDSServerConfig `json:"xdsServer,omitempty"` + + // Provisioner contains parameters of the provisioner server. If + // unspecified, the provisioner server will use default configuration + // settings. + // + // +optional + Provisioner *ProvisionerConfig `json:"Provisioner,omitempty"` +} + +// ServiceType defines the type of services supported by Envoy Gateway. +// +// +kubebuilder:validation:Enum=XdsServer;Provisioner +type ServiceType string + +const ( + // XdsServerServiceType is the name of the "XdsServer" service type. + XdsServerServiceType ServiceType = "XdsServer" + + // ProvisionerServiceType is the name of the "Provisioner" service type. + ProvisionerServiceType ServiceType = "Provisioner" +) + +// XDSServerConfig contains the configuration of the xDS server. +type XDSServerConfig struct { + // Defines the IP address of the server to serve xDS over gRPC. + // + // If unspecified, defaults to "0.0.0.0", e.g. all IP addresses. + // + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:default=0.0.0.0 + Address *string `json:"address,omitempty"` + + // Defines the network port for the xDS server to serve xDS over gRPC. + // + // If unspecified, defaults is 18000. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:default=18000 + Port *int32 `json:"port,omitempty"` +} + +// ProvisionerConfig defines the provisioner server configuration. +type ProvisionerConfig struct { + // ControlPlane specifies deployment configuration of the control plane, + // e.g. the xDS server. + // + // +optional + ControlPlane *ControlPlaneConfig `json:"controlPlane,omitempty"` + + // Envoy specifies deployment configuration of the managed Envoy data + // plane. + // + // +optional + Envoy *EnvoyConfig `json:"envoy,omitempty"` +} + +// ControlPlaneConfig defines the configuration of the control plane deployment. +type ControlPlaneConfig struct { + // Replicas is the desired number of Contour replicas. If unspecified, + // defaults to 2. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=2 + Replicas *int32 `json:"replicas,omitempty"` + + // NetworkPublishing defines how to publish network endpoint(s) to a network. + // If unspecified, defaults to "KubeLoadBalancerService" which defines a + // Kubernetes service of type LoadBalancer. + // + // +optional. + NetworkPublishing *NetworkPublishing `json:"networkPublishing,omitempty"` +} + +// EnvoyConfig defines the configuration of the Envoy data plane deployment. +type EnvoyConfig struct { + // Replicas is the desired number of Envoy replicas. If unspecified, + // defaults to 2. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=2 + Replicas *int32 `json:"replicas,omitempty"` + + // NetworkPublishing defines how to publish network endpoint(s) to a network. + // If unspecified, defaults to "KubeLoadBalancerService" which defines a + // Kubernetes service of type LoadBalancer. + // + // +optional + // +kubebuilder:default={type: KubeLoadBalancerService} + NetworkPublishing *NetworkPublishing `json:"networkPublishing,omitempty"` +} + +// NetworkPublishing defines how to publish network endpoint(s) to a network. +type NetworkPublishing struct { + // NetworkPublishingType is the type of publishing strategy to use. Valid values are: + // + // * KubeLoadBalancerService + // + // In this configuration, network endpoints for Envoy use container networking. + // A Kubernetes LoadBalancer Service is created to publish Envoy network + // endpoints. + // + // See: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer + // + // * KubeNodePortService + // + // Publishes Envoy network endpoints using a Kubernetes NodePort Service. + // + // In this configuration, Envoy network endpoints use container networking. A Kubernetes + // NodePort Service is created to publish the network endpoints. + // + // See: https://kubernetes.io/docs/concepts/services-networking/service/#nodeport + // + // * KubeClusterIPService + // + // Publishes Envoy network endpoints using a Kubernetes ClusterIP Service. + // + // In this configuration, Envoy network endpoints use container networking. A Kubernetes + // ClusterIP Service is created to publish the network endpoints. + // + // See: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + // + // If unset, defaults to KubeLoadBalancerService. + // + // +optional + // +kubebuilder:default=KubeLoadBalancerService + Type *NetworkPublishingType `json:"type,omitempty"` +} + +// NetworkPublishingType is a way to publish Envoy network endpoints to a network. +type NetworkPublishingType string + +const ( + // KubeLoadBalancerPublishingType publishes a network endpoint using a Kubernetes + // LoadBalancer Service. + KubeLoadBalancerPublishingType NetworkPublishingType = "KubeLoadBalancerService" + + // KubeNodePortPublishingType publishes a network endpoint using a Kubernetes + // NodePort Service. + KubeNodePortPublishingType NetworkPublishingType = "KubeNodePortService" + + // KubeClusterIPPublishingType publishes a network endpoint using a Kubernetes + // ClusterIP Service. + KubeClusterIPPublishingType NetworkPublishingType = "KubeClusterIPService" +) + +// BootstrapConfigStatus defines the observed state of BootstrapConfig. +type BootstrapConfigStatus struct { + // TODO: Define status fields. +} + +type BootstrapConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BootstrapConfigSpec `json:"spec,omitempty"` + Status BootstrapConfigStatus `json:"status,omitempty"` +} +``` + +[issue_51]: https://github.com/envoyproxy/gateway/issues/51 +[design_doc]: https://github.com/envoyproxy/gateway/blob/main/docs/design/SYSTEM_DESIGN.md +[kubebuilder]: https://book-v2.book.kubebuilder.io/ From 5e207fceb40d66cc7994790fa76bf4a30a413a89 Mon Sep 17 00:00:00 2001 From: danehans Date: Tue, 7 Jun 2022 11:49:21 -0700 Subject: [PATCH 2/6] Fixes api group, removes provisioner, and refactors config api Signed-off-by: danehans --- docs/design/CONFIG_API.md | 230 +++++++++++--------------------------- 1 file changed, 65 insertions(+), 165 deletions(-) diff --git a/docs/design/CONFIG_API.md b/docs/design/CONFIG_API.md index 2a928495cc..0370af2d31 100644 --- a/docs/design/CONFIG_API.md +++ b/docs/design/CONFIG_API.md @@ -8,13 +8,14 @@ to as the "Bootstrap Config" in the Envoy Gateway high-level [design doc][design ## Goals -* Define an API for configuring Envoy Gateway. +* Define an API for configuring Envoy Gateway, independent of the runtime platform. * Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. ## Non-Goals * Implementation of the API. * Define the `status` subresource of the configuration API. +* Define provisioning configuration for the control/data planes. ## Proposal @@ -29,16 +30,16 @@ following developer workflow for building APIs: After installing the latest Kubebuilder release, create the `BootstrapConfig` API: ```shell -kubebuilder create api --group envoygateway.io --version v1alpha1 --kind BootstrapConfig +kubebuilder create api --group gateway.envoyproxy.io --version v1alpha1 --kind BootstrapConfig ``` -The `v1alpha1` version and `envoygateway.io` API group get generated: +The `v1alpha1` version and `gateway.envoyproxy.io` API group get generated: ```go // gateway/api/v1alpha1/doc.go -// Package v1alpha1 contains API Schema definitions for the envoygateway.io API group. +// Package v1alpha1 contains API Schema definitions for the gateway.envoyproxy.io API group. // -// +groupName=config.envoygateway.io +// +groupName=gateway.envoyproxy.io package v1alpha1 ``` @@ -48,181 +49,80 @@ The initial `BootstrapConfig` API being proposed: // BootstrapConfigSpec defines the desired state of BootstrapConfig. type BootstrapConfigSpec struct { - // EnabledServices defines the services to enable. If unspecified, - // defaults to enabling all supported services. - // - // +optional - // +kubebuilder:validation:MaxItems=12 - // +kubebuilder:validation:Enum=XdsServer;Provisioner - // +kubebuilder:default=XdsServer;Provisioner - EnabledServices []ServiceType `json:"enabledServices,omitempty"` - - // XDSServer contains parameters of the xDS server. If unspecified, - // the xDS server will use default configuration settings. - // - // +optional - XDSServer *XDSServerConfig `json:"xdsServer,omitempty"` - - // Provisioner contains parameters of the provisioner server. If - // unspecified, the provisioner server will use default configuration - // settings. - // - // +optional - Provisioner *ProvisionerConfig `json:"Provisioner,omitempty"` + // ControlPlane contains parameters of the Envoy Gateway control plane. + // If unspecified, the control plane will use default configuration settings. + // + // +optional + ControlPlane *ControlPlaneConfig `json:"controlPlane,omitempty"` } -// ServiceType defines the type of services supported by Envoy Gateway. -// -// +kubebuilder:validation:Enum=XdsServer;Provisioner -type ServiceType string - -const ( - // XdsServerServiceType is the name of the "XdsServer" service type. - XdsServerServiceType ServiceType = "XdsServer" - - // ProvisionerServiceType is the name of the "Provisioner" service type. - ProvisionerServiceType ServiceType = "Provisioner" -) - -// XDSServerConfig contains the configuration of the xDS server. -type XDSServerConfig struct { - // Defines the IP address of the server to serve xDS over gRPC. - // - // If unspecified, defaults to "0.0.0.0", e.g. all IP addresses. - // - // +optional - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:default=0.0.0.0 - Address *string `json:"address,omitempty"` - - // Defines the network port for the xDS server to serve xDS over gRPC. - // - // If unspecified, defaults is 18000. - // - // +optional - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=65535 - // +kubebuilder:default=18000 - Port *int32 `json:"port,omitempty"` -} - -// ProvisionerConfig defines the provisioner server configuration. -type ProvisionerConfig struct { - // ControlPlane specifies deployment configuration of the control plane, - // e.g. the xDS server. - // - // +optional - ControlPlane *ControlPlaneConfig `json:"controlPlane,omitempty"` - - // Envoy specifies deployment configuration of the managed Envoy data - // plane. - // - // +optional - Envoy *EnvoyConfig `json:"envoy,omitempty"` -} - -// ControlPlaneConfig defines the configuration of the control plane deployment. +// ControlPlaneConfig defines configuration of the Envoy Gateway control plane. type ControlPlaneConfig struct { - // Replicas is the desired number of Contour replicas. If unspecified, - // defaults to 2. - // - // +optional - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:default=2 - Replicas *int32 `json:"replicas,omitempty"` - - // NetworkPublishing defines how to publish network endpoint(s) to a network. - // If unspecified, defaults to "KubeLoadBalancerService" which defines a - // Kubernetes service of type LoadBalancer. - // - // +optional. - NetworkPublishing *NetworkPublishing `json:"networkPublishing,omitempty"` -} - -// EnvoyConfig defines the configuration of the Envoy data plane deployment. -type EnvoyConfig struct { - // Replicas is the desired number of Envoy replicas. If unspecified, - // defaults to 2. - // - // +optional - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:default=2 - Replicas *int32 `json:"replicas,omitempty"` - - // NetworkPublishing defines how to publish network endpoint(s) to a network. - // If unspecified, defaults to "KubeLoadBalancerService" which defines a - // Kubernetes service of type LoadBalancer. - // - // +optional - // +kubebuilder:default={type: KubeLoadBalancerService} - NetworkPublishing *NetworkPublishing `json:"networkPublishing,omitempty"` -} - -// NetworkPublishing defines how to publish network endpoint(s) to a network. -type NetworkPublishing struct { - // NetworkPublishingType is the type of publishing strategy to use. Valid values are: - // - // * KubeLoadBalancerService - // - // In this configuration, network endpoints for Envoy use container networking. - // A Kubernetes LoadBalancer Service is created to publish Envoy network - // endpoints. - // - // See: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer - // - // * KubeNodePortService - // - // Publishes Envoy network endpoints using a Kubernetes NodePort Service. - // - // In this configuration, Envoy network endpoints use container networking. A Kubernetes - // NodePort Service is created to publish the network endpoints. - // - // See: https://kubernetes.io/docs/concepts/services-networking/service/#nodeport - // - // * KubeClusterIPService - // - // Publishes Envoy network endpoints using a Kubernetes ClusterIP Service. - // - // In this configuration, Envoy network endpoints use container networking. A Kubernetes - // ClusterIP Service is created to publish the network endpoints. - // - // See: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - // - // If unset, defaults to KubeLoadBalancerService. - // - // +optional - // +kubebuilder:default=KubeLoadBalancerService - Type *NetworkPublishingType `json:"type,omitempty"` + // Type defines the type of control plane used by Envoy Gateway. Valid values are: + // + // * Xds + // + // In this configuration, xDS is used to manage the data plane. + // + // See: https://github.com/cncf/xds + // + // +unionDiscriminator + // +kubebuilder:default=Xds + Type ControlPlaneType `json:"type,omitempty"` + + // XdsServer defines the xDS Server configuration parameters. Present only if + // type is "Xds". + // + // If unspecified, default configuration parameters are used for the xDS server. + // + // +optional + // +kubebuilder:default={address: 0.0.0.0, port: 18000} + XdsServer XdsServerConfig `json:"xdsServer,omitempty"` } -// NetworkPublishingType is a way to publish Envoy network endpoints to a network. -type NetworkPublishingType string +// ControlPlaneType defines a type of control plane used by Envoy Gateway. +// +kubebuilder:validation:Enum=Xds +type ControlPlaneType string const ( - // KubeLoadBalancerPublishingType publishes a network endpoint using a Kubernetes - // LoadBalancer Service. - KubeLoadBalancerPublishingType NetworkPublishingType = "KubeLoadBalancerService" - - // KubeNodePortPublishingType publishes a network endpoint using a Kubernetes - // NodePort Service. - KubeNodePortPublishingType NetworkPublishingType = "KubeNodePortService" - - // KubeClusterIPPublishingType publishes a network endpoint using a Kubernetes - // ClusterIP Service. - KubeClusterIPPublishingType NetworkPublishingType = "KubeClusterIPService" + // XdsControlPlaneType is a control plane that uses xDS to manage a data plane. + // See https://github.com/cncf/xds for additional details. + XdsControlPlaneType ControlPlaneType = "Xds" ) +// XdsServerConfig defines configuration of the Envoy Gateway control plane. +type XdsServerConfig struct { + // Defines the IP address for server to serve xDS over gRPC. + // + // If unspecified, defaults to "0.0.0.0", e.g. all IP addresses. + // + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:default=0.0.0.0 + Address *string `json:"address,omitempty"` + + // Defines the network port for the xDS server to serve xDS over gRPC. + // + // If unspecified, defaults is 18000. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:default=18000 + Port *int32 `json:"port,omitempty"` +} + // BootstrapConfigStatus defines the observed state of BootstrapConfig. type BootstrapConfigStatus struct { - // TODO: Define status fields. + // TODO: Define status fields. } type BootstrapConfig struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` - Spec BootstrapConfigSpec `json:"spec,omitempty"` - Status BootstrapConfigStatus `json:"status,omitempty"` + Spec BootstrapConfigSpec `json:"spec,omitempty"` + Status BootstrapConfigStatus `json:"status,omitempty"` } ``` From 0dcd2f80ab87c4b6a00fb51e02dbcc4284f072a9 Mon Sep 17 00:00:00 2001 From: danehans Date: Mon, 13 Jun 2022 13:47:17 -0700 Subject: [PATCH 3/6] Refactors the API design for Providers API Signed-off-by: danehans --- docs/design/CONFIG_API.md | 322 +++++++++++++++++++++++++++++--------- 1 file changed, 245 insertions(+), 77 deletions(-) diff --git a/docs/design/CONFIG_API.md b/docs/design/CONFIG_API.md index 0370af2d31..38e756507f 100644 --- a/docs/design/CONFIG_API.md +++ b/docs/design/CONFIG_API.md @@ -1,21 +1,27 @@ -Config API Design +Configuration API Design =================== ## Motivation -[Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway. This configuration is referred -to as the "Bootstrap Config" in the Envoy Gateway high-level [design doc][design_doc]. +[Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway at runtime. This configuration is +referred to as the "Bootstrap Config" in the Envoy Gateway high-level [design doc][design_doc]. This document replaces +the term "bootstrap" with "runtime" as some users may confuse bootstrap with Envoy's [bootstrap][envoy_boot] +configuration. ## Goals -* Define an API for configuring Envoy Gateway, independent of the runtime platform. +* Define an __initial__ API for configuring Envoy Gateway at runtime, independent of the runtime platform, e.g. + Kubernetes. * Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. ## Non-Goals -* Implementation of the API. -* Define the `status` subresource of the configuration API. -* Define provisioning configuration for the control/data planes. +* Implementation of the Envoy Gateway runtime configuration API. +* Define the `status` subresource of the runtime configuration API. +* Define a __complete__ API for configuring Envoy Gateway at runtime. As stated in the [Goals](#goals), this document + defines the initial runtime configuration API for Envoy Gateway. +* Define an API for deploying/provisioning/operating Envoy Gateway. A separate design document should be used to define + the Envoy Gateway operator API. ## Proposal @@ -28,9 +34,9 @@ following developer workflow for building APIs: - Update bootstrapped integration tests to test new fields and business logic. - Build and publish a container from a Dockerfile. -After installing the latest Kubebuilder release, create the `BootstrapConfig` API: +After installing the latest Kubebuilder release, create the `RuntimeConfig` API: ```shell -kubebuilder create api --group gateway.envoyproxy.io --version v1alpha1 --kind BootstrapConfig +kubebuilder create api --group gateway.envoyproxy.io --version v1alpha1 --kind RuntimeConfig ``` The `v1alpha1` version and `gateway.envoyproxy.io` API group get generated: @@ -43,89 +49,251 @@ The `v1alpha1` version and `gateway.envoyproxy.io` API group get generated: package v1alpha1 ``` -The initial `BootstrapConfig` API being proposed: +The initial `RuntimeConfig` API being proposed: ```go -// gateway/api/v1alpha1/bootstrapconfig_types.go - -// BootstrapConfigSpec defines the desired state of BootstrapConfig. -type BootstrapConfigSpec struct { - // ControlPlane contains parameters of the Envoy Gateway control plane. - // If unspecified, the control plane will use default configuration settings. - // - // +optional - ControlPlane *ControlPlaneConfig `json:"controlPlane,omitempty"` +package valpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/gateway/pkg/provider/file" + kube "github.com/gateway/pkg/provider/kubernetes" +) + +// RuntimeConfigSpec defines the desired state of RuntimeConfig. +type RuntimeConfigSpec struct { + // Providers define provider configuration. If unspecified, the Kubernetes + // provider is used with default parameters. + // + // +optional + // Note: the following default is provided for illustration purposes only. + // +kubebuilder:default={kubernetes: {foo: bar}} + Providers *Providers `json:"providers,omitempty"` + + // ControlPlane defines control plane configuration parameters. If unspecified, + // the control plane will use default configuration parameters. + // + // +optional + ControlPlane *ControlPlane `json:"controlPlane,omitempty"` + + // DataPlane defines data plane configuration parameters. If unspecified, + // the data plane will use default configuration parameters. + // + // +optional + DataPlane *DataPlane `json:"dataPlane,omitempty"` } -// ControlPlaneConfig defines configuration of the Envoy Gateway control plane. -type ControlPlaneConfig struct { - // Type defines the type of control plane used by Envoy Gateway. Valid values are: - // - // * Xds - // - // In this configuration, xDS is used to manage the data plane. - // - // See: https://github.com/cncf/xds - // - // +unionDiscriminator - // +kubebuilder:default=Xds - Type ControlPlaneType `json:"type,omitempty"` - - // XdsServer defines the xDS Server configuration parameters. Present only if - // type is "Xds". - // - // If unspecified, default configuration parameters are used for the xDS server. - // - // +optional - // +kubebuilder:default={address: 0.0.0.0, port: 18000} - XdsServer XdsServerConfig `json:"xdsServer,omitempty"` +// Providers defines configuration of Envoy Gateway providers. +type Providers struct { + // Kubernetes defines the configuration of the Kubernetes provider. Kubernetes + // provides runtime configuration via the Kubernetes API. + // + // +optional + Kubernetes *kube.Provider `json:"kubernetes,omitempty"` + + // File defines the configuration of the File provider. File provides runtime + // configuration defined by one or more files. + // + // +optional + File *file.Provider `json:"file,omitempty"` } -// ControlPlaneType defines a type of control plane used by Envoy Gateway. -// +kubebuilder:validation:Enum=Xds -type ControlPlaneType string +// ControlPlane defines configuration of the Envoy Gateway control plane. +type ControlPlane struct { + // XdsServer defines the xDS Server configuration parameters. If unspecified, + // default configuration parameters are used for the xDS server. + // + // +optional + // +kubebuilder:default={address: 0.0.0.0, port: 18000} + XdsServer XdsServer `json:"xdsServer,omitempty"` +} -const ( - // XdsControlPlaneType is a control plane that uses xDS to manage a data plane. - // See https://github.com/cncf/xds for additional details. - XdsControlPlaneType ControlPlaneType = "Xds" -) +// XdsServer defines configuration of the Envoy Gateway xDS server. +type XdsServer struct { + // Defines the IP address for server to serve xDS over gRPC. + // + // If unspecified, defaults to "0.0.0.0", e.g. all IP addresses. + // + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:default=0.0.0.0 + Address *string `json:"address,omitempty"` -// XdsServerConfig defines configuration of the Envoy Gateway control plane. -type XdsServerConfig struct { - // Defines the IP address for server to serve xDS over gRPC. - // - // If unspecified, defaults to "0.0.0.0", e.g. all IP addresses. - // - // +optional - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:default=0.0.0.0 - Address *string `json:"address,omitempty"` - - // Defines the network port for the xDS server to serve xDS over gRPC. - // - // If unspecified, defaults is 18000. - // - // +optional - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=65535 - // +kubebuilder:default=18000 - Port *int32 `json:"port,omitempty"` + // Defines the network port for the xDS server to serve xDS over gRPC. + // + // If unspecified, defaults is 18000. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:default=18000 + Port *int32 `json:"port,omitempty"` } -// BootstrapConfigStatus defines the observed state of BootstrapConfig. -type BootstrapConfigStatus struct { - // TODO: Define status fields. +// DataPlane defines configuration of the Envoy Gateway data plane. +type DataPlane struct { + // TODO: Define data plane configuration fields. } -type BootstrapConfig struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` +// RuntimeConfigStatus defines the observed state of RuntimeConfig. +type RuntimeConfigStatus struct { + // TODO: Define status fields. +} + +type RuntimeConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` - Spec BootstrapConfigSpec `json:"spec,omitempty"` - Status BootstrapConfigStatus `json:"status,omitempty"` + Spec RuntimeConfigSpec `json:"spec,omitempty"` + Status RuntimeConfigStatus `json:"status,omitempty"` } ``` +Provider-specific configuration is defined in the provider package. The following is an example of the Kubernetes +provider: +```go +package kubernetes + +// Provider defines the configuration of the Kubernetes provider. +type Provider struct { + // TODO: Define Kubernetes configuration fields, e.g. restrict namespaces to watch + // Gateway/HTTPRoute resources. +} +``` + +The following is an example of the File provider: +```go +package file + +// Provider defines the configuration of the File provider. +type Provider struct { + // TODO: Define Kubernetes configuration fields, e.g. path to GatewayClass, Gateway, + // and HTTPRoute manifests. +} +``` + +### Spec + +Top-level configuration is exposed through `spec` fields. The only spec fields specified by this design are `providers`, +`controlplane`, and `dataplane`. However, additional fields can be introduced in the future to expose system-wide +configuration. The following sections provide details of these top-level fields. + +### Providers + +Envoy Gateway runtime configuration is achieved through Providers. The providers are infrastructure components that +Envoy Gateway calls to establish its runtime configuration. Refer to the Envoy Gateway [design doc][design_doc] for +additional details. + +### Control Plane + +Envoy Gateway control plane configuration is exposed through the `ControlPlane` API. This API allows users to +specify provider-agnostic control plane configuration parameters. Currently, `ControlPlane` only exposes +[Xds][xds] server configuration parameters but the API supports adding configuration management of other Envoy Gateway +control plane components in the future. + +### Data Plane + +Envoy Gateway data plane configuration is exposed through the `DataPlane` API. This API allows users to specify +provider-agnostic data plane configuration settings. This API is provided in this design doc for completeness, but +should not be a defined type until data plane configuration management use cases are better understood. + +### Configuration Examples + +Run Envoy Gateway using the Kubernetes provider: +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: RuntimeConfig +metadata: + name: example +spec: {} +``` +Since Kubernetes is the default provider, no configuration is required. + +The Kubernetes provider can be configured explicitly using `spec.providers.kubernetes`: +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: RuntimeConfig +metadata: + name: example +spec: + providers: + kubernetes: {} +``` +This configuration will cause Envoy Gateway to use the Kubernetes provider with default configuration settings. + +The Kubernetes provider can be configured using the `Provider` API from the Kubernetes provider package. For example, +the `foo` field can be set to "bar": +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: RuntimeConfig +metadata: + name: example +spec: + providers: + kubernetes: + foo: bar +``` +__Note:__ The Kubernetes `Provider` API is currently undefined and `foo: bar` is provided for illustration purposes +only. + +The same API structure is followed for each supported provider. Multiple providers can be specified, for example: +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: RuntimeConfig +metadata: + name: example +spec: + providers: + - file: + foo: bar + - futureProvider: + foo: bar +``` + +Envoy Gateway will parse the list of providers to ensure only supported provider combinations are specified. If the +provider list is valid, Envoy Gateway will merge the configuration from each provider into its IR. Envoy Gateway's +Translator is then responsible for translating the IR into xDS resources that are pushed to managed data plane +instances. + +__Note:__ To learn more about the Translator, refer to the Envoy Gateway high-level [design doc][design_doc]. + +As with Providers, the control plane and data plane can be configured using `spec.controlplane` and `spec.dataplane` +respectively. For example: +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: RuntimeConfig +metadata: + name: example +spec: + controlplane: {} + dataplane: {} +``` +When unspecified, Envoy Gateway will use default configuration parameters for control and data planes. + +The following example causes the control plane to listen on `127.0.0.1:1234` instead of the default `0.0.0.0:18000` +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: RuntimeConfig +metadata: + name: example +spec: + controlplane: + type: Xds + xdsServer: + address: 127.0.0.1 + port: 1234 +``` + +### Outstanding Questions +- Should an invalid provider list cause Envoy Gateway to not start or should providers be given priority. For example, + should `providers=Kubernetes,Docker` cause Envoy Gateway to not start or start using the Kubernetes provider + (assuming the Kubernetes provider configuration is valid)? +- Should we establish a maximum number of configured providers? + [issue_51]: https://github.com/envoyproxy/gateway/issues/51 [design_doc]: https://github.com/envoyproxy/gateway/blob/main/docs/design/SYSTEM_DESIGN.md +[envoy_boot]: https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/bootstrap [kubebuilder]: https://book-v2.book.kubebuilder.io/ +[docker]: https://docs.docker.com/engine/api/v1.41/ +[xds]: https://github.com/cncf/xds +[gw_api]: https://gateway-api.sigs.k8s.io/ From d154d2adbfa4d2ad2a0c4186ab04d497a8a2a98e Mon Sep 17 00:00:00 2001 From: danehans Date: Tue, 21 Jun 2022 09:39:04 -0700 Subject: [PATCH 4/6] Post Deploy Decision API Refactor Signed-off-by: danehans --- docs/design/CONFIG_API.md | 285 ++++++++++++++------------------------ 1 file changed, 107 insertions(+), 178 deletions(-) diff --git a/docs/design/CONFIG_API.md b/docs/design/CONFIG_API.md index 38e756507f..15e8616daa 100644 --- a/docs/design/CONFIG_API.md +++ b/docs/design/CONFIG_API.md @@ -2,42 +2,26 @@ Configuration API Design =================== ## Motivation - [Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway at runtime. This configuration is -referred to as the "Bootstrap Config" in the Envoy Gateway high-level [design doc][design_doc]. This document replaces -the term "bootstrap" with "runtime" as some users may confuse bootstrap with Envoy's [bootstrap][envoy_boot] -configuration. +referred to as the "static configuration" in the Envoy Gateway [design doc][design_doc]. ## Goals - * Define an __initial__ API for configuring Envoy Gateway at runtime, independent of the runtime platform, e.g. Kubernetes. -* Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. ## Non-Goals - * Implementation of the Envoy Gateway runtime configuration API. * Define the `status` subresource of the runtime configuration API. * Define a __complete__ API for configuring Envoy Gateway at runtime. As stated in the [Goals](#goals), this document defines the initial runtime configuration API for Envoy Gateway. -* Define an API for deploying/provisioning/operating Envoy Gateway. A separate design document should be used to define - the Envoy Gateway operator API. - -## Proposal - -Utilize the [Kubebuilder][kubebuilder] framework to build and manage Envoy Gateway APIs. Kubebuilder facilitates the -following developer workflow for building APIs: - -- Create one or more resource APIs as CRDs and then add fields to the resources. -- Implement reconcile loops in controllers and watch additional resources. -- Test by running against a cluster (self-installs CRDs and starts controllers automatically). -- Update bootstrapped integration tests to test new fields and business logic. -- Build and publish a container from a Dockerfile. +* Define an API for deploying/provisioning/operating Envoy Gateway. +* Define an __initial__ provider-independent API for configuring the managed Envoy proxy infrastructure. +* Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. -After installing the latest Kubebuilder release, create the `RuntimeConfig` API: -```shell -kubebuilder create api --group gateway.envoyproxy.io --version v1alpha1 --kind RuntimeConfig -``` +## API Definition +`ControlPlaneSpec` defines the runtime configuration of Envoy Gateway. The name `ControlPlaneSpec` is being used +instead of `ControlPlane` since the API may be embedded in a higher-level API that is represented as a Kubernetes +resource. The `v1alpha1` version and `gateway.envoyproxy.io` API group get generated: ```go @@ -49,74 +33,74 @@ The `v1alpha1` version and `gateway.envoyproxy.io` API group get generated: package v1alpha1 ``` -The initial `RuntimeConfig` API being proposed: +The initial `ControlPlaneSpec` API being proposed: ```go +// gateway/api/v1alpha1/controlplane.go + package valpha1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/gateway/pkg/provider/file" - kube "github.com/gateway/pkg/provider/kubernetes" + "github.com/gateway/pkg/provider/kubernetes" ) -// RuntimeConfigSpec defines the desired state of RuntimeConfig. -type RuntimeConfigSpec struct { +// ControlPlaneSpec defines the desired state of Envoy Gateway configuration. +type ControlPlaneSpec struct { + // Gateway defines Gateway-API specific configuration. If unset, default + // configuration parameters will apply. + // + // +optional + Gateway *Gateway `json:"gateway,omitempty"` + // Providers define provider configuration. If unspecified, the Kubernetes // provider is used with default parameters. // // +optional - // Note: the following default is provided for illustration purposes only. - // +kubebuilder:default={kubernetes: {foo: bar}} Providers *Providers `json:"providers,omitempty"` - // ControlPlane defines control plane configuration parameters. If unspecified, - // the control plane will use default configuration parameters. + // XdsServer defines the xDS Server configuration parameters. If unspecified, + // default configuration parameters are applied. // // +optional - ControlPlane *ControlPlane `json:"controlPlane,omitempty"` + XdsServer XdsServer `json:"xdsServer,omitempty"` +} - // DataPlane defines data plane configuration parameters. If unspecified, - // the data plane will use default configuration parameters. +// Gateway defines desired Gateway API configuration of Envoy Gateway. +type Gateway struct { + // controllerName defines the name of the Gateway API controller. If unspecified, + // defaults to "gateway.envoyproxy.io/gatewayclass-controller". See the following + // for additional details: + // + // https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayClass // // +optional - DataPlane *DataPlane `json:"dataPlane,omitempty"` + controllerName string `json:"controllerName,omitempty"` } -// Providers defines configuration of Envoy Gateway providers. +// Providers define desired configuration of Envoy Gateway providers. type Providers struct { // Kubernetes defines the configuration of the Kubernetes provider. Kubernetes // provides runtime configuration via the Kubernetes API. // + // +support:alpha // +optional - Kubernetes *kube.Provider `json:"kubernetes,omitempty"` + Kubernetes *kubernetes.Provider `json:"kubernetes,omitempty"` // File defines the configuration of the File provider. File provides runtime // configuration defined by one or more files. // + // +support:alpha // +optional File *file.Provider `json:"file,omitempty"` } -// ControlPlane defines configuration of the Envoy Gateway control plane. -type ControlPlane struct { - // XdsServer defines the xDS Server configuration parameters. If unspecified, - // default configuration parameters are used for the xDS server. - // - // +optional - // +kubebuilder:default={address: 0.0.0.0, port: 18000} - XdsServer XdsServer `json:"xdsServer,omitempty"` -} - -// XdsServer defines configuration of the Envoy Gateway xDS server. +// XdsServer defines the desired configuration of the Envoy Gateway xDS server. type XdsServer struct { // Defines the IP address for server to serve xDS over gRPC. // // If unspecified, defaults to "0.0.0.0", e.g. all IP addresses. // // +optional - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:default=0.0.0.0 Address *string `json:"address,omitempty"` // Defines the network port for the xDS server to serve xDS over gRPC. @@ -124,34 +108,14 @@ type XdsServer struct { // If unspecified, defaults is 18000. // // +optional - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=65535 - // +kubebuilder:default=18000 Port *int32 `json:"port,omitempty"` } - -// DataPlane defines configuration of the Envoy Gateway data plane. -type DataPlane struct { - // TODO: Define data plane configuration fields. -} - -// RuntimeConfigStatus defines the observed state of RuntimeConfig. -type RuntimeConfigStatus struct { - // TODO: Define status fields. -} - -type RuntimeConfig struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec RuntimeConfigSpec `json:"spec,omitempty"` - Status RuntimeConfigStatus `json:"status,omitempty"` -} ``` - -Provider-specific configuration is defined in the provider package. The following is an example of the Kubernetes -provider: +Note that a provider-specific configuration is defined in the provider package. The following is an example of the +Kubernetes provider: ```go +// gateway/internal/kubernetes/kubernetes.go + package kubernetes // Provider defines the configuration of the Kubernetes provider. @@ -161,139 +125,104 @@ type Provider struct { } ``` -The following is an example of the File provider: -```go -package file - -// Provider defines the configuration of the File provider. -type Provider struct { - // TODO: Define Kubernetes configuration fields, e.g. path to GatewayClass, Gateway, - // and HTTPRoute manifests. -} -``` - -### Spec - -Top-level configuration is exposed through `spec` fields. The only spec fields specified by this design are `providers`, -`controlplane`, and `dataplane`. However, additional fields can be introduced in the future to expose system-wide -configuration. The following sections provide details of these top-level fields. +### Gateway +Gateway defines desired configuration of [Gateway API][gw_api] controllers that reconcile and translate Gateway API +resources into the Intermediate Representation (IR). Refer to the Envoy Gateway [design doc][design_doc] for additional +details. ### Providers +Providers define desired configuration of Envoy Gateway providers. Providers are infrastructure components that Envoy +Gateway calls to establish its runtime configuration. Providers are defined by a `+support` marker to indicate the +stability of the provider. Refer to the Envoy Gateway [design doc][design_doc] for additional details. -Envoy Gateway runtime configuration is achieved through Providers. The providers are infrastructure components that -Envoy Gateway calls to establish its runtime configuration. Refer to the Envoy Gateway [design doc][design_doc] for -additional details. - -### Control Plane +### XdsServer +XdsServer defines desired configuration of [Xds][xds] server configuration parameters. Refer to the Envoy Gateway +[design doc][design_doc] for additional details. -Envoy Gateway control plane configuration is exposed through the `ControlPlane` API. This API allows users to -specify provider-agnostic control plane configuration parameters. Currently, `ControlPlane` only exposes -[Xds][xds] server configuration parameters but the API supports adding configuration management of other Envoy Gateway -control plane components in the future. +### Configuration Example +The configuration file is defined by the ControlPlaneSpec API type. At startup, Envoy Gateway searches for the +configuration at "/etc/envoy-gateway/config.yaml". -### Data Plane - -Envoy Gateway data plane configuration is exposed through the `DataPlane` API. This API allows users to specify -provider-agnostic data plane configuration settings. This API is provided in this design doc for completeness, but -should not be a defined type until data plane configuration management use cases are better understood. - -### Configuration Examples - -Run Envoy Gateway using the Kubernetes provider: -```yaml -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: RuntimeConfig -metadata: - name: example -spec: {} +Start Envoy Gateway: +```shell +envoy-gateway start ``` -Since Kubernetes is the default provider, no configuration is required. +Since the configuration file does not exist, Envoy Gateway will start with default configuration parameters. -The Kubernetes provider can be configured explicitly using `spec.providers.kubernetes`: +The Kubernetes provider can be configured explicitly using `providers.kubernetes`: ```yaml -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: RuntimeConfig -metadata: - name: example -spec: - providers: - kubernetes: {} +$ cat << EOF > /etc/envoy-gateway/config.yaml +providers: + kubernetes: {} +EOF ``` This configuration will cause Envoy Gateway to use the Kubernetes provider with default configuration settings. -The Kubernetes provider can be configured using the `Provider` API from the Kubernetes provider package. For example, -the `foo` field can be set to "bar": +The Kubernetes provider can be configured using the `Providers` API. For example, the `foo` field can be set to "bar": ```yaml -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: RuntimeConfig -metadata: - name: example -spec: - providers: - kubernetes: - foo: bar +$ cat << EOF > /etc/envoy-gateway/config.yaml +providers: + kubernetes: + foo: bar +EOF ``` -__Note:__ The Kubernetes `Provider` API is currently undefined and `foo: bar` is provided for illustration purposes -only. +__Note:__ The `Providers` API from the Kubernetes package is currently undefined and `foo: bar` is provided for +illustration purposes only. The same API structure is followed for each supported provider. Multiple providers can be specified, for example: ```yaml -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: RuntimeConfig -metadata: - name: example -spec: - providers: - - file: - foo: bar - - futureProvider: +$ cat << EOF > /etc/envoy-gateway/config.yaml +providers: + file: + foo: bar + futureProvider: foo: bar +EOF ``` -Envoy Gateway will parse the list of providers to ensure only supported provider combinations are specified. If the -provider list is valid, Envoy Gateway will merge the configuration from each provider into its IR. Envoy Gateway's -Translator is then responsible for translating the IR into xDS resources that are pushed to managed data plane -instances. +Envoy Gateway will parse providers to ensure only supported provider combinations are specified. If providers are valid, +Envoy Gateway will merge the configuration from each provider into its IR. Envoy Gateway's Translator is then +responsible for translating the IR into xDS resources that are pushed to managed data plane instances. -__Note:__ To learn more about the Translator, refer to the Envoy Gateway high-level [design doc][design_doc]. +__Note:__ To learn more about the Translator, refer to the Envoy Gateway [design doc][design_doc]. -As with Providers, the control plane and data plane can be configured using `spec.controlplane` and `spec.dataplane` -respectively. For example: +As with Providers, Gateway API and xDS control plane services can be configured using `spec.gateway` and +`spec.xdsServer`respectively. For example: ```yaml -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: RuntimeConfig -metadata: - name: example -spec: - controlplane: {} - dataplane: {} +$ cat << EOF > /etc/envoy-gateway/config.yaml + gateway: {} + xdsServer: {} ``` -When unspecified, Envoy Gateway will use default configuration parameters for control and data planes. +When unspecified, Envoy Gateway will use default configuration parameters for gateway and xdsServer fields. -The following example causes the control plane to listen on `127.0.0.1:1234` instead of the default `0.0.0.0:18000` +The following example causes the GatewayClass controller to manage GatewayClasses with controllerName `foo` instead of +the default `gateway.envoyproxy.io/gatewayclass-controller`: ```yaml -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: RuntimeConfig -metadata: - name: example -spec: - controlplane: - type: Xds - xdsServer: - address: 127.0.0.1 - port: 1234 +$ cat << EOF > /etc/envoy-gateway/config.yaml +gateway: + controllerName: foo +``` + +The following example causes the xDS Server to listen on localhost instead of all IP addresses (default): +```yaml +$ cat << EOF > /etc/envoy-gateway/config.yaml +xdsServer: + address: 127.0.0.1 +``` + +With any of the above configuration examples, you can Start Envoy Gateway without any additional arguments: +```shell +envoy-gateway start ``` ### Outstanding Questions -- Should an invalid provider list cause Envoy Gateway to not start or should providers be given priority. For example, - should `providers=Kubernetes,Docker` cause Envoy Gateway to not start or start using the Kubernetes provider +- Should an invalid provider combination cause Envoy Gateway to not start or should providers be given priority. For + example, should `providers=Kubernetes,File` cause Envoy Gateway to not start or start using the Kubernetes provider (assuming the Kubernetes provider configuration is valid)? - Should we establish a maximum number of configured providers? [issue_51]: https://github.com/envoyproxy/gateway/issues/51 [design_doc]: https://github.com/envoyproxy/gateway/blob/main/docs/design/SYSTEM_DESIGN.md -[envoy_boot]: https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/bootstrap -[kubebuilder]: https://book-v2.book.kubebuilder.io/ -[docker]: https://docs.docker.com/engine/api/v1.41/ [xds]: https://github.com/cncf/xds [gw_api]: https://gateway-api.sigs.k8s.io/ +[config_guide]: https://github.com/envoyproxy/gateway/blob/main/docs/CONFIG.md From ed82b44ff7f00e064842b099a47adb904f68b324 Mon Sep 17 00:00:00 2001 From: danehans Date: Fri, 24 Jun 2022 14:55:35 -0700 Subject: [PATCH 5/6] Updates API spec based on 6_23_22 meeting feedback Signed-off-by: danehans --- docs/design/CONFIG_API.md | 295 +++++++++++++++++++++++++------------- 1 file changed, 193 insertions(+), 102 deletions(-) diff --git a/docs/design/CONFIG_API.md b/docs/design/CONFIG_API.md index 15e8616daa..d8f3d92a89 100644 --- a/docs/design/CONFIG_API.md +++ b/docs/design/CONFIG_API.md @@ -1,41 +1,58 @@ Configuration API Design =================== +# Table of Contents +1. [Motivation](#motiviation) +2. [Goals](#goals) +3. [Non-Goals](#non-goals) +4. [Control Plane API](#control_plane_api) + 1. [Gateway Type](#gateway) + 2. [Provider Type](#provider) + 3. [Configuration Examples](#control_plane_configuration) +6. [Data Plane API](#data_plane_api) + 1. [Configuration Examples](#data_plane_configuration) + ## Motivation -[Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway at runtime. This configuration is -referred to as the "static configuration" in the Envoy Gateway [design doc][design_doc]. +[Issue 51][issue_51] specifies the need to design an API for configuring Envoy Gateway. The control plane is configured +statically at startup and the data plane is configured dynamically through Kubernetes resources, primarily +[Gateway API][gw_api] objects. Refer to the Envoy Gateway [design doc][design_doc] for additional details regarding +Envoy Gateway terminology and configuration. ## Goals -* Define an __initial__ API for configuring Envoy Gateway at runtime, independent of the runtime platform, e.g. - Kubernetes. +* Define an __initial__ API to configure Envoy Gateway at startup. +* Define an __initial__ API for configuring the managed data plane, e.g. Envoy proxies. ## Non-Goals -* Implementation of the Envoy Gateway runtime configuration API. -* Define the `status` subresource of the runtime configuration API. -* Define a __complete__ API for configuring Envoy Gateway at runtime. As stated in the [Goals](#goals), this document - defines the initial runtime configuration API for Envoy Gateway. -* Define an API for deploying/provisioning/operating Envoy Gateway. -* Define an __initial__ provider-independent API for configuring the managed Envoy proxy infrastructure. +* Implementation of the configuration APIs. +* Define the `status` subresource of the configuration APIs. +* Define a __complete__ set of APIs for configuring Envoy Gateway. As stated in the [Goals](#goals), this document + defines the initial configuration APIs. +* Define an API for deploying/provisioning/operating Envoy Gateway. If needed, a future Envoy Gateway operator would be + responsible for designing and implementing this type of API. * Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. -## API Definition -`ControlPlaneSpec` defines the runtime configuration of Envoy Gateway. The name `ControlPlaneSpec` is being used -instead of `ControlPlane` since the API may be embedded in a higher-level API that is represented as a Kubernetes -resource. +## Control Plane API +The `EnvoyGatewaySpec` API defines the control plane configuration, e.g. Envoy Gateway. Key points of this API are: + +* The name `EnvoyGatewaySpec` is being used instead of `EnvoyGateway` since the API may be embedded in a higher-level + API to surface control plane status. +* It will define Envoy Gateway's startup configuration file. If the file does not exist, Envoy Gateway will start up + with default configuration parameters. +* If data plane static configuration is required in the future, Envoy Gateway will use a separate file for this purpose. -The `v1alpha1` version and `gateway.envoyproxy.io` API group get generated: +The `v1alpha1` version and `config.gateway.envoyproxy.io` API group get generated: ```go -// gateway/api/v1alpha1/doc.go +// gateway/api/config/v1alpha1/doc.go -// Package v1alpha1 contains API Schema definitions for the gateway.envoyproxy.io API group. +// Package v1alpha1 contains API Schema definitions for the config.gateway.envoyproxy.io API group. // -// +groupName=gateway.envoyproxy.io +// +groupName=config.gateway.envoyproxy.io package v1alpha1 ``` -The initial `ControlPlaneSpec` API being proposed: +The initial `EnvoyGatewaySpec` API being proposed: ```go -// gateway/api/v1alpha1/controlplane.go +// gateway/api/config/v1alpha1/envoygateway.go package valpha1 @@ -44,77 +61,68 @@ import ( "github.com/gateway/pkg/provider/kubernetes" ) -// ControlPlaneSpec defines the desired state of Envoy Gateway configuration. -type ControlPlaneSpec struct { +// EnvoyGatewaySpec defines the desired state of Envoy Gateway configuration. +type EnvoyGatewaySpec struct { // Gateway defines Gateway-API specific configuration. If unset, default // configuration parameters will apply. // // +optional Gateway *Gateway `json:"gateway,omitempty"` - // Providers define provider configuration. If unspecified, the Kubernetes - // provider is used with default parameters. + // Provider defines the desired provider configuration. If unspecified, + // the Kubernetes provider is used with default parameters. // // +optional - Providers *Providers `json:"providers,omitempty"` - - // XdsServer defines the xDS Server configuration parameters. If unspecified, - // default configuration parameters are applied. - // - // +optional - XdsServer XdsServer `json:"xdsServer,omitempty"` + Provider *Provider `json:"provider,omitempty"` } // Gateway defines desired Gateway API configuration of Envoy Gateway. type Gateway struct { - // controllerName defines the name of the Gateway API controller. If unspecified, + // ControllerName defines the name of the Gateway API controller. If unspecified, // defaults to "gateway.envoyproxy.io/gatewayclass-controller". See the following // for additional details: // // https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayClass // // +optional - controllerName string `json:"controllerName,omitempty"` + ControllerName string `json:"controllerName,omitempty"` } -// Providers define desired configuration of Envoy Gateway providers. -type Providers struct { +// Provider defines the desired configuration of a provider. +// +union +type Provider struct { + // Type is the type of provider to use. If unset, the Kubernetes provider is used. + // + // +unionDiscriminator + Type ProviderType `json:"type,omitempty"` // Kubernetes defines the configuration of the Kubernetes provider. Kubernetes // provides runtime configuration via the Kubernetes API. // - // +support:alpha // +optional Kubernetes *kubernetes.Provider `json:"kubernetes,omitempty"` // File defines the configuration of the File provider. File provides runtime // configuration defined by one or more files. // - // +support:alpha // +optional File *file.Provider `json:"file,omitempty"` } -// XdsServer defines the desired configuration of the Envoy Gateway xDS server. -type XdsServer struct { - // Defines the IP address for server to serve xDS over gRPC. - // - // If unspecified, defaults to "0.0.0.0", e.g. all IP addresses. - // - // +optional - Address *string `json:"address,omitempty"` +// ProviderType defines the types of providers supported by Envoy Gateway. +type ProviderType string - // Defines the network port for the xDS server to serve xDS over gRPC. - // - // If unspecified, defaults is 18000. - // - // +optional - Port *int32 `json:"port,omitempty"` -} +const ( + // KubernetesProviderType defines the "Kubernetes" provider. + KubernetesProviderType ProviderType = "Kubernetes" + + // FileProviderType defines the "File" provider. + FileProviderType ProviderType = "File" +) ``` Note that a provider-specific configuration is defined in the provider package. The following is an example of the Kubernetes provider: ```go -// gateway/internal/kubernetes/kubernetes.go +// gateway/pkg/kubernetes/kubernetes.go package kubernetes @@ -130,17 +138,14 @@ Gateway defines desired configuration of [Gateway API][gw_api] controllers that resources into the Intermediate Representation (IR). Refer to the Envoy Gateway [design doc][design_doc] for additional details. -### Providers -Providers define desired configuration of Envoy Gateway providers. Providers are infrastructure components that Envoy -Gateway calls to establish its runtime configuration. Providers are defined by a `+support` marker to indicate the -stability of the provider. Refer to the Envoy Gateway [design doc][design_doc] for additional details. - -### XdsServer -XdsServer defines desired configuration of [Xds][xds] server configuration parameters. Refer to the Envoy Gateway +### Provider +Provider defines the desired configuration of an Envoy Gateway provider. A provider is an infrastructure component that +Envoy Gateway calls to establish its runtime configuration. Provider is a [union type][union]. Therefore, Envoy Gateway +can be configured with only one provider based on the `type` discriminator field. Refer to the Envoy Gateway [design doc][design_doc] for additional details. -### Configuration Example -The configuration file is defined by the ControlPlaneSpec API type. At startup, Envoy Gateway searches for the +### Configuration Examples +The configuration file is defined by the EnvoyGatewaySpec API type. At startup, Envoy Gateway searches for the configuration at "/etc/envoy-gateway/config.yaml". Start Envoy Gateway: @@ -149,80 +154,166 @@ envoy-gateway start ``` Since the configuration file does not exist, Envoy Gateway will start with default configuration parameters. -The Kubernetes provider can be configured explicitly using `providers.kubernetes`: +The Kubernetes provider can be configured explicitly using `provider.kubernetes`: ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml -providers: +provider: + type: Kubernetes kubernetes: {} EOF ``` -This configuration will cause Envoy Gateway to use the Kubernetes provider with default configuration settings. +This configuration will cause Envoy Gateway to use the Kubernetes provider with default configuration parameters. -The Kubernetes provider can be configured using the `Providers` API. For example, the `foo` field can be set to "bar": +The Kubernetes provider can be configured using the `provider` field. For example, the `foo` field can be set to "bar": ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml -providers: +provider: + type: Kubernetes kubernetes: foo: bar EOF ``` -__Note:__ The `Providers` API from the Kubernetes package is currently undefined and `foo: bar` is provided for +__Note:__ The Provider API from the Kubernetes package is currently undefined and `foo: bar` is provided for illustration purposes only. -The same API structure is followed for each supported provider. Multiple providers can be specified, for example: +The same API structure is followed for each supported provider. The following example causes Envoy Gateway to use the +File provider: ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml -providers: +provider: + type: File file: foo: bar - futureProvider: - foo: bar EOF ``` +__Note:__ The Provider API from the File package is currently undefined and `foo: bar` is provided for illustration +purposes only. -Envoy Gateway will parse providers to ensure only supported provider combinations are specified. If providers are valid, -Envoy Gateway will merge the configuration from each provider into its IR. Envoy Gateway's Translator is then -responsible for translating the IR into xDS resources that are pushed to managed data plane instances. - -__Note:__ To learn more about the Translator, refer to the Envoy Gateway [design doc][design_doc]. - -As with Providers, Gateway API and xDS control plane services can be configured using `spec.gateway` and -`spec.xdsServer`respectively. For example: -```yaml -$ cat << EOF > /etc/envoy-gateway/config.yaml - gateway: {} - xdsServer: {} -``` -When unspecified, Envoy Gateway will use default configuration parameters for gateway and xdsServer fields. - -The following example causes the GatewayClass controller to manage GatewayClasses with controllerName `foo` instead of -the default `gateway.envoyproxy.io/gatewayclass-controller`: +Gateway API-related configuration is expressed through the `gateway` field. If unspecified, Envoy Gateway will use +default configuration parameters for `gateway`. The following example causes the [GatewayClass][gc] controller to +manage GatewayClasses with controllerName `foo` instead of the default `gateway.envoyproxy.io/gatewayclass-controller`: ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml gateway: controllerName: foo ``` -The following example causes the xDS Server to listen on localhost instead of all IP addresses (default): -```yaml -$ cat << EOF > /etc/envoy-gateway/config.yaml -xdsServer: - address: 127.0.0.1 -``` - -With any of the above configuration examples, you can Start Envoy Gateway without any additional arguments: +With any of the above configuration examples, Envoy Gateway can be started without any additional arguments: ```shell envoy-gateway start ``` -### Outstanding Questions -- Should an invalid provider combination cause Envoy Gateway to not start or should providers be given priority. For - example, should `providers=Kubernetes,File` cause Envoy Gateway to not start or start using the Kubernetes provider - (assuming the Kubernetes provider configuration is valid)? -- Should we establish a maximum number of configured providers? +## Data Plane API +The data plane is configured dynamically through Kubernetes resources, primarily [Gateway API][gw_api] objects. +Optionally, the data plane infrastructure can be configured by referencing a [custom resource (CR)][cr] through +`spec.parametersRef` of the managed GatewayClass. The `envoyproxies` API defines the data plane infrastructure +configuration and is represented as the CR referenced by the managed GatewayClass. Key points of this API are: + +* If unreferenced by `spec.parametersRef`, default parameters will be used to configure the data plane infrastructure, + e.g. expose Envoy network endpoints using a LoadBalancer service. +* Envoy Gateway will follow Gateway API [recommendations][gc] regarding updates to the EnvoyProxy CR: + > It is recommended that this resource be used as a template for Gateways. This means that a Gateway is based on the + > state of the GatewayClass at the time it was created and changes to the GatewayClass or associated parameters are + > not propagated down to existing Gateways. + +The initial `envoyproxies` API being proposed: +```go +// gateway/api/config/v1alpha1/envoyproxy.go + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EnvoyProxy is the Schema for the envoyproxies API. +type EnvoyProxy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EnvoyProxySpec `json:"spec,omitempty"` + Status EnvoyProxyStatus `json:"status,omitempty"` +} + +// EnvoyProxySpec defines the desired state of Envoy Proxy infrastructure +// configuration. +type EnvoyProxySpec struct { + // Undefined by this design spec. +} + +// EnvoyProxyStatus defines the observed state of EnvoyProxy. +type EnvoyProxyStatus struct { + // Undefined by this design spec. +} +``` +The EnvoyProxySpec and EnvoyProxyStatus fields will be defined in the future as proxy infrastructure configuration use +cases are better understood. + +### Configuration Examples +GatewayClass and Gateway resources define the data plane infrastructure. Note that all examples assume Envoy Gateway is +running with the Kubernetes provider. +```yaml +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: GatewayClass +metadata: + name: example-class +spec: + controllerName: gateway.envoyproxy.io/gateway-controller +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: example-gateway +spec: + gatewayClassName: example-class + listeners: + - name: http + protocol: HTTP + port: 80 +``` +Since the GatewayClass does not define `spec.parametersRef`, the data plane is provisioned using default configuration +parameters. All Envoy proxies will be configured with a http listener and a Kubernetes LoadBalancer service listening +on port 80. + +The following example will configure the data plane to use a ClusterIP service instead of the default LoadBalancer +service: +```yaml +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: GatewayClass +metadata: + name: example-class +spec: + controllerName: gateway.envoyproxy.io/gateway-controller + parametersRef: + name: example-config + group: config.gateway.envoyproxy.io + kind: EnvoyProxy +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: example-gateway +spec: + gatewayClassName: example-class + listeners: + - name: http + protocol: HTTP + port: 80 +--- +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: example-config +spec: + networkPublishing: + type: NodePortService +``` +__Note:__ The NetworkPublishing API is currently undefined and is provided here for illustration purposes only. [issue_51]: https://github.com/envoyproxy/gateway/issues/51 [design_doc]: https://github.com/envoyproxy/gateway/blob/main/docs/design/SYSTEM_DESIGN.md [xds]: https://github.com/cncf/xds [gw_api]: https://gateway-api.sigs.k8s.io/ [config_guide]: https://github.com/envoyproxy/gateway/blob/main/docs/CONFIG.md +[gc]: https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayClass +[cr]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ From 41894ec4183f8101874159b5654752c8939926f3 Mon Sep 17 00:00:00 2001 From: danehans Date: Mon, 27 Jun 2022 16:45:59 -0700 Subject: [PATCH 6/6] Updates EnvoyGateway API to be a GKV schema Signed-off-by: danehans --- docs/design/CONFIG_API.md | 44 ++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/docs/design/CONFIG_API.md b/docs/design/CONFIG_API.md index d8f3d92a89..324c5e29a6 100644 --- a/docs/design/CONFIG_API.md +++ b/docs/design/CONFIG_API.md @@ -32,12 +32,15 @@ Envoy Gateway terminology and configuration. * Specify tooling for managing the API, e.g. generate protos, CRDs, controller RBAC, etc. ## Control Plane API -The `EnvoyGatewaySpec` API defines the control plane configuration, e.g. Envoy Gateway. Key points of this API are: +The `EnvoyGateway` API defines the control plane configuration, e.g. Envoy Gateway. Key points of this API are: -* The name `EnvoyGatewaySpec` is being used instead of `EnvoyGateway` since the API may be embedded in a higher-level - API to surface control plane status. * It will define Envoy Gateway's startup configuration file. If the file does not exist, Envoy Gateway will start up with default configuration parameters. +* EnvoyGateway inlines the `TypeMeta` API. This allows EnvoyGateway to be versioned and managed as a GroupVersionKind + scheme. +* EnvoyGateway does not contain a metadata field since it's currently represented as a static configuration file instead of + a Kubernetes resource. +* Since EnvoyGateway does not surface status, EnvoyGatewaySpec is inlined. * If data plane static configuration is required in the future, Envoy Gateway will use a separate file for this purpose. The `v1alpha1` version and `config.gateway.envoyproxy.io` API group get generated: @@ -50,17 +53,27 @@ The `v1alpha1` version and `config.gateway.envoyproxy.io` API group get generate package v1alpha1 ``` -The initial `EnvoyGatewaySpec` API being proposed: +The initial `EnvoyGateway` API being proposed: ```go // gateway/api/config/v1alpha1/envoygateway.go package valpha1 import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/gateway/pkg/provider/file" "github.com/gateway/pkg/provider/kubernetes" ) +// EnvoyGateway is the Schema for the envoygateways API +type EnvoyGateway struct { + metav1.TypeMeta `json:",inline"` + + // EnvoyGatewaySpec defines the desired state of Envoy Gateway. + EnvoyGatewaySpec `json:",inline"` +} + // EnvoyGatewaySpec defines the desired state of Envoy Gateway configuration. type EnvoyGatewaySpec struct { // Gateway defines Gateway-API specific configuration. If unset, default @@ -94,7 +107,7 @@ type Provider struct { // Type is the type of provider to use. If unset, the Kubernetes provider is used. // // +unionDiscriminator - Type ProviderType `json:"type,omitempty"` + Type ProviderType `json:"type,omitempty"` // Kubernetes defines the configuration of the Kubernetes provider. Kubernetes // provides runtime configuration via the Kubernetes API. // @@ -144,13 +157,13 @@ Envoy Gateway calls to establish its runtime configuration. Provider is a [union can be configured with only one provider based on the `type` discriminator field. Refer to the Envoy Gateway [design doc][design_doc] for additional details. -### Configuration Examples -The configuration file is defined by the EnvoyGatewaySpec API type. At startup, Envoy Gateway searches for the -configuration at "/etc/envoy-gateway/config.yaml". +### Control Plane Configuration +The configuration file is defined by the EnvoyGateway API type. At startup, Envoy Gateway searches for the configuration +at "/etc/envoy-gateway/config.yaml". Start Envoy Gateway: ```shell -envoy-gateway start +$ ./envoy-gateway ``` Since the configuration file does not exist, Envoy Gateway will start with default configuration parameters. @@ -167,6 +180,8 @@ This configuration will cause Envoy Gateway to use the Kubernetes provider with The Kubernetes provider can be configured using the `provider` field. For example, the `foo` field can be set to "bar": ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway provider: type: Kubernetes kubernetes: @@ -180,6 +195,8 @@ The same API structure is followed for each supported provider. The following ex File provider: ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway provider: type: File file: @@ -194,13 +211,15 @@ default configuration parameters for `gateway`. The following example causes the manage GatewayClasses with controllerName `foo` instead of the default `gateway.envoyproxy.io/gatewayclass-controller`: ```yaml $ cat << EOF > /etc/envoy-gateway/config.yaml +apiVersion: config.gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway gateway: controllerName: foo ``` With any of the above configuration examples, Envoy Gateway can be started without any additional arguments: ```shell -envoy-gateway start +$ ./envoy-gateway ``` ## Data Plane API @@ -249,7 +268,7 @@ type EnvoyProxyStatus struct { The EnvoyProxySpec and EnvoyProxyStatus fields will be defined in the future as proxy infrastructure configuration use cases are better understood. -### Configuration Examples +### Data Plane Configuration GatewayClass and Gateway resources define the data plane infrastructure. Note that all examples assume Envoy Gateway is running with the Kubernetes provider. ```yaml @@ -306,7 +325,7 @@ metadata: name: example-config spec: networkPublishing: - type: NodePortService + type: ClusterIPService ``` __Note:__ The NetworkPublishing API is currently undefined and is provided here for illustration purposes only. @@ -317,3 +336,4 @@ __Note:__ The NetworkPublishing API is currently undefined and is provided here [config_guide]: https://github.com/envoyproxy/gateway/blob/main/docs/CONFIG.md [gc]: https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io/v1alpha2.GatewayClass [cr]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ +[union]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#unions