From 566e1220a7d437b4d632216634cdd57773820ecb Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Fri, 13 Oct 2017 16:14:33 -0400 Subject: [PATCH 01/40] Adding missing filters Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 19 ++++++++ api/filter/http_fault.proto | 93 ++++++++++++++++++++++++++++++++++++ api/filter/mongo_proxy.proto | 8 ++++ api/filter/redis_proxy.proto | 34 +++++++++++++ api/filter/tcp_fault.proto | 68 ++++++++++++++++++++++++++ 5 files changed, 222 insertions(+) create mode 100644 api/filter/http_fault.proto create mode 100644 api/filter/redis_proxy.proto create mode 100644 api/filter/tcp_fault.proto diff --git a/api/filter/BUILD b/api/filter/BUILD index 812aae71e..e1288e1ae 100644 --- a/api/filter/BUILD +++ b/api/filter/BUILD @@ -15,6 +15,9 @@ api_proto_library( api_proto_library( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], + deps = [ + ":http_fault", + ], ) api_proto_library( @@ -26,3 +29,19 @@ api_proto_library( name = "tcp_proxy", srcs = ["tcp_proxy.proto"], ) + +api_proto_library( + name = "http_fault", + srcs = ["http_fault.proto"], + deps = [ + "//api:rds", + ], +) + +api_proto_library( + name = "redis_proxy", + srcs = ["redis_proxy.proto"], + deps = [ + ":http_fault", + ], +) diff --git a/api/filter/http_fault.proto b/api/filter/http_fault.proto new file mode 100644 index 000000000..c4bb57bbb --- /dev/null +++ b/api/filter/http_fault.proto @@ -0,0 +1,93 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +import "api/rds.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +// Delay specification is used to inject latency into the rpc operations. +message Delay { + // Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is supported. + string type = 1; + + // An integer between 0-100 indicating the percentage of operations + // on which the delay will be injected. + google.protobuf.UInt32Value percent = 2; + + oneof http_delay_type { + // Add a fixed delay before forwarding the operation upstream. Format: 1h/1m/1s/1ms. MUST be >=1ms. + google.protobuf.Duration fixed_delay = 2; + // (-- Add a delay (based on an exponential function) before forwarding + // the operation. Mean delay needed to derive the exponential delay + // values. NOT IMPLEMENTED --) + google.protobuf.Duration exponential_delay = 3 ; + } + // (-- Specify delay duration as part of Http request. NOT IMPLEMENTED --) + string override_header_name = 4 ; +} + +// HTTPFault can be used to specify one or more faults to inject +// while forwarding http requests to the upstream cluster. +// Faults include aborting the Http request from downstream service, +// and/or delaying proxying of HTTP requests. Fault filter is executed +// before proxying a request. Hence timeouts, retries, circuit breakers will +// not be activated on an upstream cluster due to errors injected by the +// fault filter. +// +// *Note:* Delay and abort faults are independent of one another, even if +// both are specified simultaneously. +message HTTPFault { + // Delay requests before forwarding, emulating various failures such as + // network issues, overloaded upstream service, etc. + Delay delay = 1; + + // Abort Http request attempts and return error codes back to downstream + // service, giving the impression that the upstream service is faulty. + Abort abort = 2; + + // Specifies the name of the upstream cluster that the + // filter should match on. Fault injection will be restricted to requests + // bound to the specific upstream cluster. + string upstream_cluster = 3; + + // Specifies a set of headers that the filter should match on. The fault + // injection filter can be applied selectively to requests that match a + // set of headers specified in the fault filter config. The chances of + // actual fault injection further depend on the values of abort_percent + // and fixed_delay_percent parameters.The filter will check the request’s + // headers against all the specified headers in the filter config. A + // match will happen if all the headers in the config are present in the + // request with the same values (or based on presence if the value field + // is not in the config). TODO: allow runtime configuration on per entry + // basis for headers match. + repeated HeaderMatcher headers = 4; + + // Faults are injected for the specified list of downstream hosts. If + // this setting is not set, faults are injected for all downstream + // nodes. Downstream node name is taken from the HTTP + // x-envoy-downstream-service-node header and compared against + // downstream_nodes list. + repeated string downstream_nodes =5; + + // Abort specification is used to prematurely abort a request with a + // pre-specified error code. + message Abort { + // An integer between 0-100 indicating the percentage of requests + // that will be aborted with the error code provided. + google.protobuf.UInt32Value percent = 1; + + oneof error_type { + // gRPC status code to use to abort a gRPC request. NOT IMPLEMENTED + string grpc_status = 2 ; + // HTTP2 error code used to abort a Http2 request. NOT IMPLEMENTED + string http2_error = 3 ; + // HTTP status code to use to abort the Http request. + int32 http_status = 4; + } + // (-- Specify abort code as part of Http request.--) + string override_header_name = 5 ; + } +} diff --git a/api/filter/mongo_proxy.proto b/api/filter/mongo_proxy.proto index 5a0fb7851..7d9515a60 100644 --- a/api/filter/mongo_proxy.proto +++ b/api/filter/mongo_proxy.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.api.v2.filter; +import "http_fault.proto"; + message MongoProxy { // The human readable prefix to use when emitting statistics for the // MongoDB proxy filter. See the statistics documentation for more information. @@ -11,4 +13,10 @@ message MongoProxy { // path is specified no access logs will be written. Note that access log is // also gated by runtime. string access_log = 2; + + // Inject a fixed delay before proxying a Mongo operation. Delays are + // applied to the following MongoDB operations: Query, Insert, GetMore, + // and KillCursors. Once an active delay is in progress, all incoming + // data up until the timer event fires will be a part of the delay. + Delay delay = 3; } diff --git a/api/filter/redis_proxy.proto b/api/filter/redis_proxy.proto new file mode 100644 index 000000000..62ed357b1 --- /dev/null +++ b/api/filter/redis_proxy.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +import "http_fault.proto"; +import "google/protobuf/duration.proto"; + +message RedisProxy { + // The human readable prefix to use when emitting statistics for the + // Redis proxy filter. See the statistics documentation for more information. + string stat_prefix = 1; + + // Indicates the upstream cluster to which the operation should be routed to. + string cluster = 2; + + // Network settings for the connection pool to the upstream cluster. + ConnPoolSettings settings = 3; + + // Inject a fixed delay before proxying a Redis operation. NOT IMPLEMENTED + Delay delay = 4; + + // Redis connection pool settings + message ConnPoolSettings { + // Per-operation timeout. Format: 1h/1m/1s/1ms. MUST be >=1ms. The + // timer starts when the first command of a pipeline is written to the + // backend connection. Each response received from Redis resets the + // timer since it signifies that the next command is being processed by + // the backend. The only exception to this behavior is when a + // connection to a backend is not yet established. In that case, the + // connect timeout on the cluster will govern the timeout until the + // connection is ready. + google.protobuf.Duration op_timeout = 3; + } +} diff --git a/api/filter/tcp_fault.proto b/api/filter/tcp_fault.proto new file mode 100644 index 000000000..aae388f60 --- /dev/null +++ b/api/filter/tcp_fault.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +// (-- Faults can be injected into the connections from downstream by the +// Envoy, for testing the failure recovery capabilities of downstream +// services. Faults include aborting the connection from downstream +// service, delaying proxying of connection to the destination +// service, and throttling the bandwidth of the connection (either +// end). Bandwidth throttling for failure testing should not be confused +// with the rate limiting policy enforcement provided by the Mixer +// component. L4 fault injection is not supported at the moment. NOT IMPLEMENTED --) +message TCPFault { + // Unlike Http services, we have very little context for raw TCP|UDP + // connections. We could throttle bandwidth of the connections (slow down + // the connection) and/or abruptly reset (terminate) the Tcp connection + // after it has been established. + // We first throttle (if set) and then terminate the connection. + Throttle throttle = 1; + Terminate terminate = 2; + + // Specifies the name of the upstream cluster that the + // filter should match on. Fault injection will be restricted to connections + // bound to the specific upstream cluster. + string upstream_cluster = 3; + + // Bandwidth throttling for Tcp and Udp connections + message Throttle { + // percentage of connections to throttle. + float percent = 1; + // bandwidth limit in "bits" per second between downstream and Envoy + int64 downstream_limit_bps = 2; + // bandwidth limits in "bits" per second between Envoy and upstream + int64 upstream_limit_bps = 3; + + oneof throttle_after { + // Wait a while after the connection is established, before + // starting bandwidth throttling. This would allow us to inject fault + // after the application protocol (e.g., MySQL) has had time to + // establish sessions/whatever handshake necessary. + google.protobuf.Duration throttle_after_period = 4; + + // Alternatively, we could wait for a certain number of bytes to be + // transferred to upstream before throttling the bandwidth. + double throttle_after_bytes = 5; + } + + // Stop throttling after the given duration. If not set, the connection + // will be throttled for its lifetime. + google.protobuf.Duration throttle_for_period = 6; + } + + // Abruptly reset (terminate) the Tcp connection after it has been + // established, emulating remote server crash or link failure. + message Terminate { + // percentage of established Tcp connections to be terminated/reset + float percent = 1; + + // Wait a while after the connection is established, before + // terminating the connection. Set to 0 to terminate immediately on + // connection establishment. + google.protobuf.Duration terminate_after_period = 2; + } +} From 6aed0e5c357e1a662ff3c81376fdb7050864ad14 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Fri, 13 Oct 2017 16:19:16 -0400 Subject: [PATCH 02/40] adding tcp fault to bazel Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/filter/BUILD b/api/filter/BUILD index e1288e1ae..7164ade1e 100644 --- a/api/filter/BUILD +++ b/api/filter/BUILD @@ -38,6 +38,11 @@ api_proto_library( ], ) +api_proto_library( + name = "tcp_fault", + srcs = ["tcp_fault.proto"], +) + api_proto_library( name = "redis_proxy", srcs = ["redis_proxy.proto"], From ff6eeea4b9e701e5d9c1b3f0de6d9d21fde3247d Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Fri, 13 Oct 2017 17:47:25 -0400 Subject: [PATCH 03/40] review feedback Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 16 +--- api/filter/{http_fault.proto => fault.proto} | 91 ++++++++++++-------- api/filter/mongo_proxy.proto | 4 +- api/filter/redis_proxy.proto | 5 +- api/filter/tcp_fault.proto | 68 --------------- 5 files changed, 62 insertions(+), 122 deletions(-) rename api/filter/{http_fault.proto => fault.proto} (52%) delete mode 100644 api/filter/tcp_fault.proto diff --git a/api/filter/BUILD b/api/filter/BUILD index 7164ade1e..9780179ff 100644 --- a/api/filter/BUILD +++ b/api/filter/BUILD @@ -15,9 +15,7 @@ api_proto_library( api_proto_library( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], - deps = [ - ":http_fault", - ], + deps = [":fault"], ) api_proto_library( @@ -31,22 +29,14 @@ api_proto_library( ) api_proto_library( - name = "http_fault", - srcs = ["http_fault.proto"], + name = "fault", + srcs = ["fault.proto"], deps = [ "//api:rds", ], ) -api_proto_library( - name = "tcp_fault", - srcs = ["tcp_fault.proto"], -) - api_proto_library( name = "redis_proxy", srcs = ["redis_proxy.proto"], - deps = [ - ":http_fault", - ], ) diff --git a/api/filter/http_fault.proto b/api/filter/fault.proto similarity index 52% rename from api/filter/http_fault.proto rename to api/filter/fault.proto index c4bb57bbb..9da7d9d70 100644 --- a/api/filter/http_fault.proto +++ b/api/filter/fault.proto @@ -8,30 +8,49 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; -// Delay specification is used to inject latency into the rpc operations. -message Delay { +// Delay specification is used to inject latency into the rpc/TCP proxy operations. +message FaultDelay { + enum FaultDelayType { + // Fixed delay (step function). + FIXED = 0; + // Exponential delay. + EXPONENTIAL = 1; + } + // Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is supported. - string type = 1; + FaultDelayType type = 1; - // An integer between 0-100 indicating the percentage of operations + // An integer between 0-100 indicating the percentage of operations/connection requests // on which the delay will be injected. google.protobuf.UInt32Value percent = 2; - oneof http_delay_type { + oneof fault_delay_type { // Add a fixed delay before forwarding the operation upstream. Format: 1h/1m/1s/1ms. MUST be >=1ms. - google.protobuf.Duration fixed_delay = 2; - // (-- Add a delay (based on an exponential function) before forwarding - // the operation. Mean delay needed to derive the exponential delay - // values. NOT IMPLEMENTED --) - google.protobuf.Duration exponential_delay = 3 ; + // For HTTP/Mongo/Redis, the specified delay will be injected before a new request/operation. For TCP + // connections, the proxying of the connection upstream will be delayed for the specified period. + google.protobuf.Duration fixed_delay = 3; + } +} + +// Abort specification is used to prematurely abort a request/TCP connection +// with a pre-specified error code. +message FaultAbort { + // An integer between 0-100 indicating the percentage of requests/operations/connections + // that will be aborted with the error code provided. + google.protobuf.UInt32Value percent = 1; + + // Applicable only for HTTP connections. + oneof error_type { + // HTTP status code to use to abort the HTTP request. + int32 http_status = 2; + // gRPC status code to use to abort the gRPC request. + int32 grpc_status = 3; } - // (-- Specify delay duration as part of Http request. NOT IMPLEMENTED --) - string override_header_name = 4 ; } // HTTPFault can be used to specify one or more faults to inject -// while forwarding http requests to the upstream cluster. -// Faults include aborting the Http request from downstream service, +// while forwarding HTTP requests to the upstream cluster. +// Faults include aborting the HTTP request from downstream service, // and/or delaying proxying of HTTP requests. Fault filter is executed // before proxying a request. Hence timeouts, retries, circuit breakers will // not be activated on an upstream cluster due to errors injected by the @@ -39,12 +58,15 @@ message Delay { // // *Note:* Delay and abort faults are independent of one another, even if // both are specified simultaneously. + +// * Note:* The fault injection filter must be inserted before any +// other filter, including the router filter. message HTTPFault { // Delay requests before forwarding, emulating various failures such as // network issues, overloaded upstream service, etc. Delay delay = 1; - // Abort Http request attempts and return error codes back to downstream + // Abort HTTP request attempts and return error codes back to downstream // service, giving the impression that the upstream service is faulty. Abort abort = 2; @@ -70,24 +92,23 @@ message HTTPFault { // nodes. Downstream node name is taken from the HTTP // x-envoy-downstream-service-node header and compared against // downstream_nodes list. - repeated string downstream_nodes =5; - - // Abort specification is used to prematurely abort a request with a - // pre-specified error code. - message Abort { - // An integer between 0-100 indicating the percentage of requests - // that will be aborted with the error code provided. - google.protobuf.UInt32Value percent = 1; - - oneof error_type { - // gRPC status code to use to abort a gRPC request. NOT IMPLEMENTED - string grpc_status = 2 ; - // HTTP2 error code used to abort a Http2 request. NOT IMPLEMENTED - string http2_error = 3 ; - // HTTP status code to use to abort the Http request. - int32 http_status = 4; - } - // (-- Specify abort code as part of Http request.--) - string override_header_name = 5 ; - } + repeated string downstream_nodes = 5; +} + +// Faults can be injected into the connections from downstream by the +// Envoy, for testing the failure recovery capabilities of downstream +// services. Faults include aborting the connection from downstream +// service, and delaying proxying of connection to the destination. +message TCPFault { + // Delay proxying of the TCP connection for a specified period. + Delay delay = 1; + + // Abort a specified percentage of downstream TCP connections without + // establishing an upstream connection. The connection will be abruptly reset. + Abort abort = 2; + + // Specifies the name of the upstream cluster that the + // filter should match on. Fault injection will be restricted to connections + // bound to the specific upstream cluster. + string upstream_cluster = 3; } diff --git a/api/filter/mongo_proxy.proto b/api/filter/mongo_proxy.proto index 7d9515a60..441ec16a2 100644 --- a/api/filter/mongo_proxy.proto +++ b/api/filter/mongo_proxy.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter; -import "http_fault.proto"; +import "fault.proto"; message MongoProxy { // The human readable prefix to use when emitting statistics for the @@ -18,5 +18,5 @@ message MongoProxy { // applied to the following MongoDB operations: Query, Insert, GetMore, // and KillCursors. Once an active delay is in progress, all incoming // data up until the timer event fires will be a part of the delay. - Delay delay = 3; + FaultDelay delay = 3; } diff --git a/api/filter/redis_proxy.proto b/api/filter/redis_proxy.proto index 62ed357b1..fa07fdae3 100644 --- a/api/filter/redis_proxy.proto +++ b/api/filter/redis_proxy.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter; -import "http_fault.proto"; +import "fault.proto"; import "google/protobuf/duration.proto"; message RedisProxy { @@ -16,9 +16,6 @@ message RedisProxy { // Network settings for the connection pool to the upstream cluster. ConnPoolSettings settings = 3; - // Inject a fixed delay before proxying a Redis operation. NOT IMPLEMENTED - Delay delay = 4; - // Redis connection pool settings message ConnPoolSettings { // Per-operation timeout. Format: 1h/1m/1s/1ms. MUST be >=1ms. The diff --git a/api/filter/tcp_fault.proto b/api/filter/tcp_fault.proto deleted file mode 100644 index aae388f60..000000000 --- a/api/filter/tcp_fault.proto +++ /dev/null @@ -1,68 +0,0 @@ -syntax = "proto3"; - -package envoy.api.v2.filter; - -import "google/protobuf/duration.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; - -// (-- Faults can be injected into the connections from downstream by the -// Envoy, for testing the failure recovery capabilities of downstream -// services. Faults include aborting the connection from downstream -// service, delaying proxying of connection to the destination -// service, and throttling the bandwidth of the connection (either -// end). Bandwidth throttling for failure testing should not be confused -// with the rate limiting policy enforcement provided by the Mixer -// component. L4 fault injection is not supported at the moment. NOT IMPLEMENTED --) -message TCPFault { - // Unlike Http services, we have very little context for raw TCP|UDP - // connections. We could throttle bandwidth of the connections (slow down - // the connection) and/or abruptly reset (terminate) the Tcp connection - // after it has been established. - // We first throttle (if set) and then terminate the connection. - Throttle throttle = 1; - Terminate terminate = 2; - - // Specifies the name of the upstream cluster that the - // filter should match on. Fault injection will be restricted to connections - // bound to the specific upstream cluster. - string upstream_cluster = 3; - - // Bandwidth throttling for Tcp and Udp connections - message Throttle { - // percentage of connections to throttle. - float percent = 1; - // bandwidth limit in "bits" per second between downstream and Envoy - int64 downstream_limit_bps = 2; - // bandwidth limits in "bits" per second between Envoy and upstream - int64 upstream_limit_bps = 3; - - oneof throttle_after { - // Wait a while after the connection is established, before - // starting bandwidth throttling. This would allow us to inject fault - // after the application protocol (e.g., MySQL) has had time to - // establish sessions/whatever handshake necessary. - google.protobuf.Duration throttle_after_period = 4; - - // Alternatively, we could wait for a certain number of bytes to be - // transferred to upstream before throttling the bandwidth. - double throttle_after_bytes = 5; - } - - // Stop throttling after the given duration. If not set, the connection - // will be throttled for its lifetime. - google.protobuf.Duration throttle_for_period = 6; - } - - // Abruptly reset (terminate) the Tcp connection after it has been - // established, emulating remote server crash or link failure. - message Terminate { - // percentage of established Tcp connections to be terminated/reset - float percent = 1; - - // Wait a while after the connection is established, before - // terminating the connection. Set to 0 to terminate immediately on - // connection establishment. - google.protobuf.Duration terminate_after_period = 2; - } -} From 523a11af2291487ffb72cbd9d3febff27507fe33 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Fri, 13 Oct 2017 17:48:30 -0400 Subject: [PATCH 04/40] nit Signed-off-by: Shriram Rajagopalan --- api/filter/redis_proxy.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/api/filter/redis_proxy.proto b/api/filter/redis_proxy.proto index fa07fdae3..0ba6e3373 100644 --- a/api/filter/redis_proxy.proto +++ b/api/filter/redis_proxy.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package envoy.api.v2.filter; -import "fault.proto"; import "google/protobuf/duration.proto"; message RedisProxy { From a00975f767daa091bc3a9076b05be82fd529b1d2 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Sun, 15 Oct 2017 22:43:07 -0400 Subject: [PATCH 05/40] syncing with docs Signed-off-by: Shriram Rajagopalan --- api/filter/fault.proto | 51 ++++++++++++++++++++++-------------- api/filter/redis_proxy.proto | 17 ++++++------ 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/api/filter/fault.proto b/api/filter/fault.proto index 9da7d9d70..d234da34b 100644 --- a/api/filter/fault.proto +++ b/api/filter/fault.proto @@ -25,9 +25,12 @@ message FaultDelay { google.protobuf.UInt32Value percent = 2; oneof fault_delay_type { - // Add a fixed delay before forwarding the operation upstream. Format: 1h/1m/1s/1ms. MUST be >=1ms. - // For HTTP/Mongo/Redis, the specified delay will be injected before a new request/operation. For TCP - // connections, the proxying of the connection upstream will be delayed for the specified period. + // Add a fixed delay before forwarding the operation upstream. See + // https://developers.google.com/protocol-buffers/docs/proto3#json for + // the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified + // delay will be injected before a new request/operation. For TCP + // connections, the proxying of the connection upstream will be delayed + // for the specified period. google.protobuf.Duration fixed_delay = 3; } } @@ -48,29 +51,39 @@ message FaultAbort { } } -// HTTPFault can be used to specify one or more faults to inject -// while forwarding HTTP requests to the upstream cluster. -// Faults include aborting the HTTP request from downstream service, -// and/or delaying proxying of HTTP requests. Fault filter is executed -// before proxying a request. Hence timeouts, retries, circuit breakers will -// not be activated on an upstream cluster due to errors injected by the -// fault filter. +// The fault injection filter can be used to test the resiliency of +// microservices to different forms of failures. The filter can be used to +// inject delays and abort requests with user-specified error codes, +// thereby providing the ability to stage different failure scenarios such +// as service failures, service overloads, high network latency, network +// partitions, etc. Faults injection can be limited to a specific set of +// requests based on the (destination) upstream cluster of a request and/or +// a set of pre-defined request headers. +// +// The scope of failures is restricted to those that are observable by an +// application communicating over the network. CPU and disk failures on the +// local host cannot be emulated. +// +// Currently, the fault injection filter has the following limitations: +// +// * Abort codes are restricted to HTTP status codes only +// * Delays are restricted to fixed duration. +// +// Future versions will include support for restricting faults to specific +// routes, and delay durations based on distributions. // -// *Note:* Delay and abort faults are independent of one another, even if -// both are specified simultaneously. - // * Note:* The fault injection filter must be inserted before any // other filter, including the router filter. message HTTPFault { - // Delay requests before forwarding, emulating various failures such as - // network issues, overloaded upstream service, etc. + // If specified, the filter will inject delays based on the values in the + // object. At least abort or delay must be specified. Delay delay = 1; - // Abort HTTP request attempts and return error codes back to downstream - // service, giving the impression that the upstream service is faulty. + // If specified, the filter will abort requests based on the values in + // the object. At least abort or delay must be specified. Abort abort = 2; - // Specifies the name of the upstream cluster that the + // Specifies the name of the (destination) upstream cluster that the // filter should match on. Fault injection will be restricted to requests // bound to the specific upstream cluster. string upstream_cluster = 3; @@ -82,7 +95,7 @@ message HTTPFault { // and fixed_delay_percent parameters.The filter will check the request’s // headers against all the specified headers in the filter config. A // match will happen if all the headers in the config are present in the - // request with the same values (or based on presence if the value field + // request with the same values (or based on presence if the `value` field // is not in the config). TODO: allow runtime configuration on per entry // basis for headers match. repeated HeaderMatcher headers = 4; diff --git a/api/filter/redis_proxy.proto b/api/filter/redis_proxy.proto index 0ba6e3373..2f7e275f3 100644 --- a/api/filter/redis_proxy.proto +++ b/api/filter/redis_proxy.proto @@ -17,14 +17,15 @@ message RedisProxy { // Redis connection pool settings message ConnPoolSettings { - // Per-operation timeout. Format: 1h/1m/1s/1ms. MUST be >=1ms. The - // timer starts when the first command of a pipeline is written to the - // backend connection. Each response received from Redis resets the - // timer since it signifies that the next command is being processed by - // the backend. The only exception to this behavior is when a - // connection to a backend is not yet established. In that case, the - // connect timeout on the cluster will govern the timeout until the - // connection is ready. + // Per-operation timeout. See + // https://developers.google.com/protocol-buffers/docs/proto3#json for + // the JSON/YAML Duration mapping. The timer starts when the first + // command of a pipeline is written to the backend connection. Each + // response received from Redis resets the timer since it signifies + // that the next command is being processed by the backend. The only + // exception to this behavior is when a connection to a backend is not + // yet established. In that case, the connect timeout on the cluster + // will govern the timeout until the connection is ready. google.protobuf.Duration op_timeout = 3; } } From 867aac0cb4e0d4c83c8df14a2fd73b4624259e21 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Mon, 16 Oct 2017 11:39:05 -0400 Subject: [PATCH 06/40] nit Signed-off-by: Shriram Rajagopalan --- api/filter/mongo_proxy.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/filter/mongo_proxy.proto b/api/filter/mongo_proxy.proto index 441ec16a2..042c235ff 100644 --- a/api/filter/mongo_proxy.proto +++ b/api/filter/mongo_proxy.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter; -import "fault.proto"; +import "api/filter/fault.proto"; message MongoProxy { // The human readable prefix to use when emitting statistics for the From a729bc154226cfceec8e71351d983331827791a2 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 17 Oct 2017 00:39:39 -0400 Subject: [PATCH 07/40] cleanups Signed-off-by: Shriram Rajagopalan --- api/filter/fault.proto | 18 ++++++++---------- api/filter/mongo_proxy.proto | 2 +- api/filter/redis_proxy.proto | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/api/filter/fault.proto b/api/filter/fault.proto index d234da34b..722e4ff33 100644 --- a/api/filter/fault.proto +++ b/api/filter/fault.proto @@ -35,7 +35,7 @@ message FaultDelay { } } -// Abort specification is used to prematurely abort a request/TCP connection +// Abort specification is used to prematurely abort a rpc operation/TCP connection // with a pre-specified error code. message FaultAbort { // An integer between 0-100 indicating the percentage of requests/operations/connections @@ -45,9 +45,7 @@ message FaultAbort { // Applicable only for HTTP connections. oneof error_type { // HTTP status code to use to abort the HTTP request. - int32 http_status = 2; - // gRPC status code to use to abort the gRPC request. - int32 grpc_status = 3; + uint32 http_status = 2; } } @@ -77,11 +75,11 @@ message FaultAbort { message HTTPFault { // If specified, the filter will inject delays based on the values in the // object. At least abort or delay must be specified. - Delay delay = 1; + FaultDelay delay = 1; // If specified, the filter will abort requests based on the values in // the object. At least abort or delay must be specified. - Abort abort = 2; + FaultAbort abort = 2; // Specifies the name of the (destination) upstream cluster that the // filter should match on. Fault injection will be restricted to requests @@ -91,8 +89,8 @@ message HTTPFault { // Specifies a set of headers that the filter should match on. The fault // injection filter can be applied selectively to requests that match a // set of headers specified in the fault filter config. The chances of - // actual fault injection further depend on the values of abort_percent - // and fixed_delay_percent parameters.The filter will check the request’s + // actual fault injection further depend on the values of FaultAbort.percent + // and FaultDelay.percent parameters. The filter will check the request’s // headers against all the specified headers in the filter config. A // match will happen if all the headers in the config are present in the // request with the same values (or based on presence if the `value` field @@ -114,11 +112,11 @@ message HTTPFault { // service, and delaying proxying of connection to the destination. message TCPFault { // Delay proxying of the TCP connection for a specified period. - Delay delay = 1; + FaultDelay delay = 1; // Abort a specified percentage of downstream TCP connections without // establishing an upstream connection. The connection will be abruptly reset. - Abort abort = 2; + FaultAbort abort = 2; // Specifies the name of the upstream cluster that the // filter should match on. Fault injection will be restricted to connections diff --git a/api/filter/mongo_proxy.proto b/api/filter/mongo_proxy.proto index 042c235ff..cd4f8e39d 100644 --- a/api/filter/mongo_proxy.proto +++ b/api/filter/mongo_proxy.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter; -import "api/filter/fault.proto"; +import "filter/fault.proto"; message MongoProxy { // The human readable prefix to use when emitting statistics for the diff --git a/api/filter/redis_proxy.proto b/api/filter/redis_proxy.proto index 2f7e275f3..b5bc5beb3 100644 --- a/api/filter/redis_proxy.proto +++ b/api/filter/redis_proxy.proto @@ -26,6 +26,6 @@ message RedisProxy { // exception to this behavior is when a connection to a backend is not // yet established. In that case, the connect timeout on the cluster // will govern the timeout until the connection is ready. - google.protobuf.Duration op_timeout = 3; + google.protobuf.Duration op_timeout = 1; } } From 9456f270cc24836c9bb1108f9ff9baf28090cde4 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 17 Oct 2017 13:11:40 -0400 Subject: [PATCH 08/40] trying out a longer import Signed-off-by: Shriram Rajagopalan --- api/filter/mongo_proxy.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/filter/mongo_proxy.proto b/api/filter/mongo_proxy.proto index cd4f8e39d..042c235ff 100644 --- a/api/filter/mongo_proxy.proto +++ b/api/filter/mongo_proxy.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter; -import "filter/fault.proto"; +import "api/filter/fault.proto"; message MongoProxy { // The human readable prefix to use when emitting statistics for the From 5e0d5c21e00eb226ec9f575653d6d40f29e15e1f Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Mon, 23 Oct 2017 12:23:21 -0400 Subject: [PATCH 09/40] new filters Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 25 +++++++++++++++ api/filter/buffer.proto | 18 +++++++++++ api/filter/client_ssl_auth.proto | 29 ++++++++++++++++++ api/filter/fault.proto | 5 +-- api/filter/ip_tagging.proto | 26 ++++++++++++++++ api/filter/router.proto | 14 +++++++++ api/filter/transcoder.proto | 52 ++++++++++++++++++++++++++++++++ 7 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 api/filter/buffer.proto create mode 100644 api/filter/client_ssl_auth.proto create mode 100644 api/filter/ip_tagging.proto create mode 100644 api/filter/router.proto create mode 100644 api/filter/transcoder.proto diff --git a/api/filter/BUILD b/api/filter/BUILD index 9780179ff..495195c7b 100644 --- a/api/filter/BUILD +++ b/api/filter/BUILD @@ -40,3 +40,28 @@ api_proto_library( name = "redis_proxy", srcs = ["redis_proxy.proto"], ) + +api_proto_library( + name = "router", + srcs = ["router.proto"], +) + +api_proto_library( + name = "buffer", + srcs = ["buffer.proto"], +) + +api_proto_library( + name = "transcoder", + srcs = ["transcoder.proto"], +) + +api_proto_library( + name = "ip_tagging", + srcs = ["ip_tagging.proto"], +) + +api_proto_library( + name = "client_ssl_auth", + srcs = ["client_ssl_auth.proto"], +) diff --git a/api/filter/buffer.proto b/api/filter/buffer.proto new file mode 100644 index 000000000..d50f2fe9d --- /dev/null +++ b/api/filter/buffer.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +import "google/protobuf/duration.proto"; + +// The buffer filter is used to stop filter iteration and wait for a fully +// buffered complete request. This is useful in different situations +// including protecting some applications from having to deal with partial +// requests and high network latency. +message Buffer { + // The maximum request size that the filter will before the connection + // manager will stop buffering and return a 413 response. + uint32 max_request_bytes = 1; + // The maximum amount of time that the filter will wait for a complete + // request before returning a 408 response. + google.protobuf.Duration max_request_time = 2; +} diff --git a/api/filter/client_ssl_auth.proto b/api/filter/client_ssl_auth.proto new file mode 100644 index 000000000..01d897052 --- /dev/null +++ b/api/filter/client_ssl_auth.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +import "google/protobuf/duration.proto"; + +// Envoy provides a network filter that performs TLS client authentication +// via principals fetched from a REST VPN service. This filter matches the +// presented client certificate hash against the principal list to +// determine whether the connection should be allowed or not. Optional IP +// white listing can also be configured. This functionality can be used to +// build edge proxy VPN support for web infrastructure. +message ClientSSLAuth { + // The cluster manager cluster that runs the authentication service. The + // filter will connect to the service every 60s to fetch the list of + // principals. The service must support the expected REST API. + string auth_api_cluster = 1; + // The prefix to use when emitting statistics. + string stat_prefix = 2; + // Time in milliseconds between principal refreshes from the + // authentication service. Default is 60000 (60s). The actual fetch time + // will be this value plus a random jittered value between + // 0-refresh_delay_ms milliseconds. + google.protobuf.Duration refresh_delay = 3; + // An optional list of IP address and subnet masks that should be white + // listed for access by the filter. If no list is provided, there is no + // IP white list. + repeated string ip_white_list = 4; +} diff --git a/api/filter/fault.proto b/api/filter/fault.proto index 722e4ff33..70362c9aa 100644 --- a/api/filter/fault.proto +++ b/api/filter/fault.proto @@ -8,7 +8,8 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; -// Delay specification is used to inject latency into the rpc/TCP proxy operations. +// Delay specification is used to inject latency into the +// HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections. message FaultDelay { enum FaultDelayType { // Fixed delay (step function). @@ -35,7 +36,7 @@ message FaultDelay { } } -// Abort specification is used to prematurely abort a rpc operation/TCP connection +// Abort specification is used to prematurely abort a HTTP/gRPC/Mongo/Redis operation/TCP connection // with a pre-specified error code. message FaultAbort { // An integer between 0-100 indicating the percentage of requests/operations/connections diff --git a/api/filter/ip_tagging.proto b/api/filter/ip_tagging.proto new file mode 100644 index 000000000..0070490d0 --- /dev/null +++ b/api/filter/ip_tagging.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +// This is an HTTP filter which enables Envoy to tag requests with extra +// information such as location, cloud source, and any extra data. This is +// useful to prevent against DDoS. +message IPTagging { + // The type of requests the filter should apply to. The supported types + // are internal, external or both. A request is considered internal if + // x-envoy-internal is set to true. If x-envoy-internal is not set or + // false, a request is considered external. The filter defaults to both, + // and it will apply to all request types. + string request_type = 1; + + repeated IPTag ip_tags = 2; + + message IPTag { + // Specifies the ip tag name to apply. + string ip_tag_name = 1; + + // A list of IP address and subnet masks that will be tagged with the + // ip_tag_name. Both IPv4 and IPv6 CIDR addresses are allowed here. + repeated string ip_list = 2; + } +} diff --git a/api/filter/router.proto b/api/filter/router.proto new file mode 100644 index 000000000..4f91a3cd9 --- /dev/null +++ b/api/filter/router.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +//The router filter implements HTTP forwarding. It will be used in almost +//all HTTP proxy scenarios that Envoy is deployed for. The filter’s main +//job is to follow the instructions specified in the configured route +//table. In addition to forwarding and redirection, the filter also handles +//retry, statistics, etc. +message Router { + // Whether the router generates dynamic cluster statistics. Defaults to + // true. Can be disabled in high performance scenarios. + bool dynamic_stats = 1; +} diff --git a/api/filter/transcoder.proto b/api/filter/transcoder.proto new file mode 100644 index 000000000..5c241140a --- /dev/null +++ b/api/filter/transcoder.proto @@ -0,0 +1,52 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +// This is a filter which allows a RESTful JSON API client to send requests +// to Envoy over HTTP and get proxied to a gRPC service. The HTTP mapping +// for the gRPC service has to be defined by custom options, defined in +// https://cloud.google.com/service-management/reference/rpc/google.api#http +message gRPCJSONTranscoder { + // The filter config for the filter requires the descriptor file as well + // as a list of the gRPC services to be transcoded. + + // Supplies the binary protobuf descriptor set for the gRPC services. The + // descriptor set has to include all of the types that are used in the + // services. Make sure to use the --include_import option for protoc. + string proto_descriptor = 1; + // A list of strings that supplies the service names that the transcoder + // will translate. If the service name doesn’t exist in proto_descriptor, + // Envoy will fail at startup. The proto_descriptor may contain more + // services than the service names specified here, but they won’t be + // translated. + repeated string services = 2; + + // Control options for response json. These options are passed directly + // to JsonPrintOptions. + PrintOptions print_options = 3; + + // Control options for response json. These options are passed directly + // to JsonPrintOptions. + message PrintOptions { + // Whether to add spaces, line breaks and indentation to make the JSON + // output easy to read. Default to false. + bool add_whitespace = 1; + + // Whether to always print primitive fields. By default primitive + // fields with default values will be omitted in JSON output. For + // example, an int32 field set to 0 will be omitted. Set this flag to + // true will override the default behavior and print primitive fields + // regardless of their values. Default to false. + bool always_print_primitive_fields = 2; + + // Whether to always print enums as ints. By default they are rendered + // as strings. Default to false. + bool always_print_enums_as_ints = 3; + + // Whether to preserve proto field names. By default protobuf will + // generate JSON field names use json_name option, or lower camel case, + // in that order. Set this flag will preserve original field + // names. Default to false. + bool preserve_proto_field_names = 4; + }; +} From 3819e579763ca02f2e0cb277bcf76af936d05958 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Mon, 23 Oct 2017 13:04:03 -0400 Subject: [PATCH 10/40] rate limit and health check filters Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 5 +++ api/filter/health_check.proto | 20 +++++++++++ api/filter/rate_limit.proto | 65 +++++++++++++++++++++++++++++------ 3 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 api/filter/health_check.proto diff --git a/api/filter/BUILD b/api/filter/BUILD index 495195c7b..4124254a1 100644 --- a/api/filter/BUILD +++ b/api/filter/BUILD @@ -65,3 +65,8 @@ api_proto_library( name = "client_ssl_auth", srcs = ["client_ssl_auth.proto"], ) + +api_proto_library( + name = "health_check", + srcs = ["health_check.proto"], +) diff --git a/api/filter/health_check.proto b/api/filter/health_check.proto new file mode 100644 index 000000000..a09cbf0ee --- /dev/null +++ b/api/filter/health_check.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +import "google/protobuf/duration.proto"; + +// Note that the filter will automatically fail health checks and set the +// x-envoy-immediate-health-check-fail header if the /healthcheck/fail +// admin endpoint has been called. (The /healthcheck/ok admin endpoint +// reverses this behavior). +message HealthCheck { + // Specifies whether the filter operates in pass through mode or not. + bool pass_through_mode = 1; + // Specifies the incoming HTTP endpoint that should be considered the + // health check endpoint. For example /healthcheck. + string endpoint = 2; + // If operating in pass through mode, the amount of time in milliseconds + // that the filter should cache the upstream response. + google.protobuf.Duration cache_time = 3; +} diff --git a/api/filter/rate_limit.proto b/api/filter/rate_limit.proto index 582315f98..4e4bd5e72 100644 --- a/api/filter/rate_limit.proto +++ b/api/filter/rate_limit.proto @@ -2,6 +2,19 @@ syntax = "proto3"; package envoy.api.v2.filter; +import "google/protobuf/duration.proto"; + +// TODO(htuch): This should be the shared canonical RateLimitDescriptor +// when we import the rate limit protos +// (https://github.com/lyft/envoy-api/issues/26). +message RateLimitDescriptor { + message Entry { + string key = 1; + string value = 2; + } + repeated Entry entries = 1; +} + message RateLimit { // The human readable prefix to use when emitting statistics for the // rate limit filter. See the statistics documentation for more information. @@ -10,16 +23,46 @@ message RateLimit { // The rate limit domain to use in the rate limit service request. string domain = 2; - // The rate limit descriptor list to use in the rate limit service request. - // TODO(htuch): This should be the shared canonical RateLimitDescriptor when - // we import the rate limit protos - // (https://github.com/lyft/envoy-api/issues/26). - message RateLimitDescriptor { - message Entry { - string key = 1; - string value = 2; - } - repeated Entry entries = 1; - } + // The rate limit descriptor list to use in the rate limit service + // request. repeated RateLimitDescriptor rate_limit_descriptors = 3; } + +// HTTP Rate limit filter configuration. The HTTP rate limit filter will +// call the rate limit service when the request’s route or virtual host has +// one or more rate limit configurations that match the filter stage +// setting. The route can optionally include the virtual host rate limit +// configurations. More than one configuration can apply to a request. Each +// configuration results in a descriptor being sent to the rate limit +// service. If the rate limit service is called, and the response for any +// of the descriptors is over limit, a 429 response is returned. +message HTTPRateLimit { + // The rate limit domain to use when calling the rate limit service. + string domain = 1; + // Specifies the rate limit configurations to be applied with the same + // stage number. If not set, the default stage number is 0. NOTE: The + // filter supports a range of 0 - 10 inclusively for stage numbers. + uint32 stage = 2; + // The type of requests the filter should apply to. The supported types + // are internal, external or both. A request is considered internal if + // x-envoy-internal is set to true. If x-envoy-internal is not set or + // false, a request is considered external. The filter defaults to both, + // and it will apply to all request types. + string request_type = 3; + // The timeout in milliseconds for the rate limit service RPC. If not + // set, this defaults to 20ms. + google.protobuf.Duration timeout = 4; +} + +// TCP Rate limit filter configuration +message TCPRateLimit { + // The prefix to use when emitting statistics. + string stat_prefix = 1; + // The rate limit domain to use in the rate limit service request. + string domain = 2; + // The rate limit descriptor list to use in the rate limit service request. + repeated RateLimitDescriptor descriptors = 3; + // The timeout in milliseconds for the rate limit service RPC. If not + // set, this defaults to 20ms. + google.protobuf.Duration timeout = 4; +} From 1c0144ae8562d55f0ea4916b52d18d486a6a8ab1 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Mon, 23 Oct 2017 22:13:49 -0400 Subject: [PATCH 11/40] shuffling protos Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 56 ------------------- api/filter/README.md | 5 +- api/filter/fault.proto | 2 - api/filter/http/BUILD | 38 +++++++++++++ api/filter/{ => http}/buffer.proto | 2 +- api/filter/{ => http}/health_check.proto | 2 +- .../{ => http}/http_connection_manager.proto | 2 +- api/filter/{ => http}/ip_tagging.proto | 10 +++- api/filter/{ => http}/router.proto | 2 +- api/filter/{ => http}/transcoder.proto | 4 +- api/filter/network/BUILD | 36 ++++++++++++ .../{ => network}/client_ssl_auth.proto | 2 +- api/filter/{ => network}/mongo_proxy.proto | 2 +- api/filter/{ => network}/redis_proxy.proto | 2 +- api/filter/{ => network}/tcp_proxy.proto | 2 +- 15 files changed, 94 insertions(+), 73 deletions(-) create mode 100644 api/filter/http/BUILD rename api/filter/{ => http}/buffer.proto (95%) rename api/filter/{ => http}/health_check.proto (95%) rename api/filter/{ => http}/http_connection_manager.proto (99%) rename api/filter/{ => http}/ip_tagging.proto (85%) rename api/filter/{ => http}/router.proto (93%) rename api/filter/{ => http}/transcoder.proto (97%) create mode 100644 api/filter/network/BUILD rename api/filter/{ => network}/client_ssl_auth.proto (97%) rename api/filter/{ => network}/mongo_proxy.proto (95%) rename api/filter/{ => network}/redis_proxy.proto (96%) rename api/filter/{ => network}/tcp_proxy.proto (96%) diff --git a/api/filter/BUILD b/api/filter/BUILD index 4124254a1..68ed860ae 100644 --- a/api/filter/BUILD +++ b/api/filter/BUILD @@ -2,32 +2,11 @@ load("//bazel:api_build_system.bzl", "api_proto_library") licenses(["notice"]) # Apache 2 -api_proto_library( - name = "http_connection_manager", - srcs = ["http_connection_manager.proto"], - deps = [ - "//api:base", - "//api:protocol", - "//api:rds", - ], -) - -api_proto_library( - name = "mongo_proxy", - srcs = ["mongo_proxy.proto"], - deps = [":fault"], -) - api_proto_library( name = "rate_limit", srcs = ["rate_limit.proto"], ) -api_proto_library( - name = "tcp_proxy", - srcs = ["tcp_proxy.proto"], -) - api_proto_library( name = "fault", srcs = ["fault.proto"], @@ -35,38 +14,3 @@ api_proto_library( "//api:rds", ], ) - -api_proto_library( - name = "redis_proxy", - srcs = ["redis_proxy.proto"], -) - -api_proto_library( - name = "router", - srcs = ["router.proto"], -) - -api_proto_library( - name = "buffer", - srcs = ["buffer.proto"], -) - -api_proto_library( - name = "transcoder", - srcs = ["transcoder.proto"], -) - -api_proto_library( - name = "ip_tagging", - srcs = ["ip_tagging.proto"], -) - -api_proto_library( - name = "client_ssl_auth", - srcs = ["client_ssl_auth.proto"], -) - -api_proto_library( - name = "health_check", - srcs = ["health_check.proto"], -) diff --git a/api/filter/README.md b/api/filter/README.md index 35191da12..dd923ed61 100644 --- a/api/filter/README.md +++ b/api/filter/README.md @@ -1,8 +1,7 @@ ## NOTE -The list of filters here is incomplete. There are no proto specifications for Fault filter, Redis filter, CORS filter, etc. -These specifications will be added in the near future. In the interim, you can still supply plain JSON configuration objects -for these missing filters by setting the `"deprecated_v1"` field to true in the filter's configuration. For example, +If a filter configuration is not captured in the proto specification, you can still supply plain JSON configuration objects +for such filters by setting the `"deprecated_v1"` field to true in the filter's configuration. For example, ```json { diff --git a/api/filter/fault.proto b/api/filter/fault.proto index 70362c9aa..00463b1f7 100644 --- a/api/filter/fault.proto +++ b/api/filter/fault.proto @@ -14,8 +14,6 @@ message FaultDelay { enum FaultDelayType { // Fixed delay (step function). FIXED = 0; - // Exponential delay. - EXPONENTIAL = 1; } // Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is supported. diff --git a/api/filter/http/BUILD b/api/filter/http/BUILD new file mode 100644 index 000000000..e5b186429 --- /dev/null +++ b/api/filter/http/BUILD @@ -0,0 +1,38 @@ +load("//bazel:api_build_system.bzl", "api_proto_library") + +licenses(["notice"]) # Apache 2 + +api_proto_library( + name = "http_connection_manager", + srcs = ["http_connection_manager.proto"], + deps = [ + "//api:base", + "//api:protocol", + "//api:rds", + ], +) + +api_proto_library( + name = "router", + srcs = ["router.proto"], +) + +api_proto_library( + name = "buffer", + srcs = ["buffer.proto"], +) + +api_proto_library( + name = "transcoder", + srcs = ["transcoder.proto"], +) + +api_proto_library( + name = "ip_tagging", + srcs = ["ip_tagging.proto"], +) + +api_proto_library( + name = "health_check", + srcs = ["health_check.proto"], +) diff --git a/api/filter/buffer.proto b/api/filter/http/buffer.proto similarity index 95% rename from api/filter/buffer.proto rename to api/filter/http/buffer.proto index d50f2fe9d..3d3e986fe 100644 --- a/api/filter/buffer.proto +++ b/api/filter/http/buffer.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; import "google/protobuf/duration.proto"; diff --git a/api/filter/health_check.proto b/api/filter/http/health_check.proto similarity index 95% rename from api/filter/health_check.proto rename to api/filter/http/health_check.proto index a09cbf0ee..d38f037bf 100644 --- a/api/filter/health_check.proto +++ b/api/filter/http/health_check.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; import "google/protobuf/duration.proto"; diff --git a/api/filter/http_connection_manager.proto b/api/filter/http/http_connection_manager.proto similarity index 99% rename from api/filter/http_connection_manager.proto rename to api/filter/http/http_connection_manager.proto index 3cafac6a8..cc8295727 100644 --- a/api/filter/http_connection_manager.proto +++ b/api/filter/http/http_connection_manager.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; import "api/base.proto"; import "api/protocol.proto"; diff --git a/api/filter/ip_tagging.proto b/api/filter/http/ip_tagging.proto similarity index 85% rename from api/filter/ip_tagging.proto rename to api/filter/http/ip_tagging.proto index 0070490d0..83c05ad04 100644 --- a/api/filter/ip_tagging.proto +++ b/api/filter/http/ip_tagging.proto @@ -1,17 +1,23 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; // This is an HTTP filter which enables Envoy to tag requests with extra // information such as location, cloud source, and any extra data. This is // useful to prevent against DDoS. message IPTagging { + enum RequestType { + INTERNAL = 0; + EXTERNAL = 1; + BOTH = 2; + } + // The type of requests the filter should apply to. The supported types // are internal, external or both. A request is considered internal if // x-envoy-internal is set to true. If x-envoy-internal is not set or // false, a request is considered external. The filter defaults to both, // and it will apply to all request types. - string request_type = 1; + RequestType request_type = 1; repeated IPTag ip_tags = 2; diff --git a/api/filter/router.proto b/api/filter/http/router.proto similarity index 93% rename from api/filter/router.proto rename to api/filter/http/router.proto index 4f91a3cd9..a8e11f249 100644 --- a/api/filter/router.proto +++ b/api/filter/http/router.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; //The router filter implements HTTP forwarding. It will be used in almost //all HTTP proxy scenarios that Envoy is deployed for. The filter’s main diff --git a/api/filter/transcoder.proto b/api/filter/http/transcoder.proto similarity index 97% rename from api/filter/transcoder.proto rename to api/filter/http/transcoder.proto index 5c241140a..14c92e7c6 100644 --- a/api/filter/transcoder.proto +++ b/api/filter/http/transcoder.proto @@ -1,12 +1,12 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; // This is a filter which allows a RESTful JSON API client to send requests // to Envoy over HTTP and get proxied to a gRPC service. The HTTP mapping // for the gRPC service has to be defined by custom options, defined in // https://cloud.google.com/service-management/reference/rpc/google.api#http -message gRPCJSONTranscoder { +message gRPCJsonTranscoder { // The filter config for the filter requires the descriptor file as well // as a list of the gRPC services to be transcoded. diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD new file mode 100644 index 000000000..4be622a1c --- /dev/null +++ b/api/filter/network/BUILD @@ -0,0 +1,36 @@ +load("//bazel:api_build_system.bzl", "api_proto_library") + +licenses(["notice"]) # Apache 2 + +api_proto_library( + name = "http_connection_manager", + srcs = ["http_connection_manager.proto"], + deps = [ + "//api:base", + "//api:protocol", + "//api:rds", + ], +) + +api_proto_library( + name = "mongo_proxy", + srcs = ["mongo_proxy.proto"], + deps = [ + "//api/filter:fault", + ], +) + +api_proto_library( + name = "tcp_proxy", + srcs = ["tcp_proxy.proto"], +) + +api_proto_library( + name = "redis_proxy", + srcs = ["redis_proxy.proto"], +) + +api_proto_library( + name = "client_ssl_auth", + srcs = ["client_ssl_auth.proto"], +) diff --git a/api/filter/client_ssl_auth.proto b/api/filter/network/client_ssl_auth.proto similarity index 97% rename from api/filter/client_ssl_auth.proto rename to api/filter/network/client_ssl_auth.proto index 01d897052..e1af3a5bb 100644 --- a/api/filter/client_ssl_auth.proto +++ b/api/filter/network/client_ssl_auth.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.network; import "google/protobuf/duration.proto"; diff --git a/api/filter/mongo_proxy.proto b/api/filter/network/mongo_proxy.proto similarity index 95% rename from api/filter/mongo_proxy.proto rename to api/filter/network/mongo_proxy.proto index 042c235ff..a1833503f 100644 --- a/api/filter/mongo_proxy.proto +++ b/api/filter/network/mongo_proxy.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.network; import "api/filter/fault.proto"; diff --git a/api/filter/redis_proxy.proto b/api/filter/network/redis_proxy.proto similarity index 96% rename from api/filter/redis_proxy.proto rename to api/filter/network/redis_proxy.proto index b5bc5beb3..15a32d959 100644 --- a/api/filter/redis_proxy.proto +++ b/api/filter/network/redis_proxy.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.network; import "google/protobuf/duration.proto"; diff --git a/api/filter/tcp_proxy.proto b/api/filter/network/tcp_proxy.proto similarity index 96% rename from api/filter/tcp_proxy.proto rename to api/filter/network/tcp_proxy.proto index 5c6fdc6f5..663294645 100644 --- a/api/filter/tcp_proxy.proto +++ b/api/filter/network/tcp_proxy.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.network; import "google/protobuf/duration.proto"; From e0dd9315d99fc4fbe43e3c3b0d3a76b131b59240 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Mon, 23 Oct 2017 23:17:31 -0400 Subject: [PATCH 12/40] fix BUILD Signed-off-by: Shriram Rajagopalan --- api/filter/network/BUILD | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD index 4be622a1c..12f38b40e 100644 --- a/api/filter/network/BUILD +++ b/api/filter/network/BUILD @@ -2,16 +2,6 @@ load("//bazel:api_build_system.bzl", "api_proto_library") licenses(["notice"]) # Apache 2 -api_proto_library( - name = "http_connection_manager", - srcs = ["http_connection_manager.proto"], - deps = [ - "//api:base", - "//api:protocol", - "//api:rds", - ], -) - api_proto_library( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], From b867cc5c0594360018654aeac1ea382c0351e23e Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Mon, 23 Oct 2017 23:49:25 -0400 Subject: [PATCH 13/40] fix BUILD 2 Signed-off-by: Shriram Rajagopalan --- tools/BUILD | 2 +- tools/generate_listeners.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/BUILD b/tools/BUILD index 015badd70..59ba741f0 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -5,7 +5,7 @@ py_binary( visibility = ["//visibility:public"], deps = [ "//api:lds_py", - "//api/filter:http_connection_manager_py", + "//api/filter/http:http_connection_manager_py", ], ) diff --git a/tools/generate_listeners.py b/tools/generate_listeners.py index eba986000..fee83096d 100644 --- a/tools/generate_listeners.py +++ b/tools/generate_listeners.py @@ -17,7 +17,7 @@ from google.protobuf import text_format from api import lds_pb2 -from api.filter import http_connection_manager_pb2 +from api.filter.http import http_connection_manager_pb2 # Convert an arbitrary proto object to its Struct proto representation. From e6f8369c3b78223c25d8738e52a46beac723dfe2 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 11:05:41 -0400 Subject: [PATCH 14/40] feedback Signed-off-by: Shriram Rajagopalan --- api/filter/http/ip_tagging.proto | 6 +++--- api/filter/http/router.proto | 4 +++- api/filter/http/transcoder.proto | 2 +- api/filter/rate_limit.proto | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/api/filter/http/ip_tagging.proto b/api/filter/http/ip_tagging.proto index 83c05ad04..863001466 100644 --- a/api/filter/http/ip_tagging.proto +++ b/api/filter/http/ip_tagging.proto @@ -7,9 +7,9 @@ package envoy.api.v2.filter.http; // useful to prevent against DDoS. message IPTagging { enum RequestType { - INTERNAL = 0; - EXTERNAL = 1; - BOTH = 2; + BOTH = 0; + INTERNAL = 1; + EXTERNAL = 2; } // The type of requests the filter should apply to. The supported types diff --git a/api/filter/http/router.proto b/api/filter/http/router.proto index a8e11f249..39f8906b9 100644 --- a/api/filter/http/router.proto +++ b/api/filter/http/router.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.api.v2.filter.http; +import "google/protobuf/wrappers.proto"; + //The router filter implements HTTP forwarding. It will be used in almost //all HTTP proxy scenarios that Envoy is deployed for. The filter’s main //job is to follow the instructions specified in the configured route @@ -10,5 +12,5 @@ package envoy.api.v2.filter.http; message Router { // Whether the router generates dynamic cluster statistics. Defaults to // true. Can be disabled in high performance scenarios. - bool dynamic_stats = 1; + google.protobuf.BoolValue dynamic_stats = 1; } diff --git a/api/filter/http/transcoder.proto b/api/filter/http/transcoder.proto index 14c92e7c6..10f0b47b4 100644 --- a/api/filter/http/transcoder.proto +++ b/api/filter/http/transcoder.proto @@ -6,7 +6,7 @@ package envoy.api.v2.filter.http; // to Envoy over HTTP and get proxied to a gRPC service. The HTTP mapping // for the gRPC service has to be defined by custom options, defined in // https://cloud.google.com/service-management/reference/rpc/google.api#http -message gRPCJsonTranscoder { +message GrpcJsonTranscoder { // The filter config for the filter requires the descriptor file as well // as a list of the gRPC services to be transcoded. diff --git a/api/filter/rate_limit.proto b/api/filter/rate_limit.proto index 4e4bd5e72..edb286a34 100644 --- a/api/filter/rate_limit.proto +++ b/api/filter/rate_limit.proto @@ -28,7 +28,7 @@ message RateLimit { repeated RateLimitDescriptor rate_limit_descriptors = 3; } -// HTTP Rate limit filter configuration. The HTTP rate limit filter will +// HTTP rate limit filter configuration. The HTTP rate limit filter will // call the rate limit service when the request’s route or virtual host has // one or more rate limit configurations that match the filter stage // setting. The route can optionally include the virtual host rate limit @@ -54,7 +54,7 @@ message HTTPRateLimit { google.protobuf.Duration timeout = 4; } -// TCP Rate limit filter configuration +// TCP rate limit filter configuration message TCPRateLimit { // The prefix to use when emitting statistics. string stat_prefix = 1; From f331fa8a8a106869929429755006ef9dccfde88b Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 13:10:57 -0400 Subject: [PATCH 15/40] remove optional percent Signed-off-by: Shriram Rajagopalan --- api/filter/fault.proto | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/filter/fault.proto b/api/filter/fault.proto index 00463b1f7..763a7738d 100644 --- a/api/filter/fault.proto +++ b/api/filter/fault.proto @@ -5,8 +5,6 @@ package envoy.api.v2.filter; import "api/rds.proto"; import "google/protobuf/duration.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; // Delay specification is used to inject latency into the // HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections. @@ -21,7 +19,7 @@ message FaultDelay { // An integer between 0-100 indicating the percentage of operations/connection requests // on which the delay will be injected. - google.protobuf.UInt32Value percent = 2; + uint32 percent = 2; oneof fault_delay_type { // Add a fixed delay before forwarding the operation upstream. See @@ -39,7 +37,7 @@ message FaultDelay { message FaultAbort { // An integer between 0-100 indicating the percentage of requests/operations/connections // that will be aborted with the error code provided. - google.protobuf.UInt32Value percent = 1; + uint32 percent = 1; // Applicable only for HTTP connections. oneof error_type { From 889628ae46059bcda49978f5b172eee57c7acb5e Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 14:25:07 -0400 Subject: [PATCH 16/40] use wrappers Signed-off-by: Shriram Rajagopalan --- api/address.proto | 7 +++++++ api/filter/http/buffer.proto | 3 ++- api/filter/http/health_check.proto | 3 ++- api/filter/http/ip_tagging.proto | 4 +++- api/filter/network/client_ssl_auth.proto | 3 ++- api/lds.proto | 4 ---- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/api/address.proto b/api/address.proto index 4f5cec9e5..4739fbb60 100644 --- a/api/address.proto +++ b/api/address.proto @@ -46,3 +46,10 @@ message Address { Pipe pipe = 2; } } + +// CidrRange specifies an IP Address and a prefix length to construct +// the subnet mask. +message CidrRange { + string address_prefix = 1; + google.protobuf.UInt32Value prefix_len = 2; +} diff --git a/api/filter/http/buffer.proto b/api/filter/http/buffer.proto index 3d3e986fe..82ddfeb14 100644 --- a/api/filter/http/buffer.proto +++ b/api/filter/http/buffer.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.api.v2.filter.http; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; // The buffer filter is used to stop filter iteration and wait for a fully // buffered complete request. This is useful in different situations @@ -11,7 +12,7 @@ import "google/protobuf/duration.proto"; message Buffer { // The maximum request size that the filter will before the connection // manager will stop buffering and return a 413 response. - uint32 max_request_bytes = 1; + google.protobuf.UInt32Value max_request_bytes = 1; // The maximum amount of time that the filter will wait for a complete // request before returning a 408 response. google.protobuf.Duration max_request_time = 2; diff --git a/api/filter/http/health_check.proto b/api/filter/http/health_check.proto index d38f037bf..b2070fd86 100644 --- a/api/filter/http/health_check.proto +++ b/api/filter/http/health_check.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.api.v2.filter.http; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; // Note that the filter will automatically fail health checks and set the // x-envoy-immediate-health-check-fail header if the /healthcheck/fail @@ -10,7 +11,7 @@ import "google/protobuf/duration.proto"; // reverses this behavior). message HealthCheck { // Specifies whether the filter operates in pass through mode or not. - bool pass_through_mode = 1; + google.protobuf.BoolValue pass_through_mode = 1; // Specifies the incoming HTTP endpoint that should be considered the // health check endpoint. For example /healthcheck. string endpoint = 2; diff --git a/api/filter/http/ip_tagging.proto b/api/filter/http/ip_tagging.proto index 863001466..85f99df97 100644 --- a/api/filter/http/ip_tagging.proto +++ b/api/filter/http/ip_tagging.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.api.v2.filter.http; +import "api/address.proto"; + // This is an HTTP filter which enables Envoy to tag requests with extra // information such as location, cloud source, and any extra data. This is // useful to prevent against DDoS. @@ -27,6 +29,6 @@ message IPTagging { // A list of IP address and subnet masks that will be tagged with the // ip_tag_name. Both IPv4 and IPv6 CIDR addresses are allowed here. - repeated string ip_list = 2; + repeated CidrRange ip_list = 2; } } diff --git a/api/filter/network/client_ssl_auth.proto b/api/filter/network/client_ssl_auth.proto index e1af3a5bb..50aba55a8 100644 --- a/api/filter/network/client_ssl_auth.proto +++ b/api/filter/network/client_ssl_auth.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter.network; +import "api/address.proto"; import "google/protobuf/duration.proto"; // Envoy provides a network filter that performs TLS client authentication @@ -25,5 +26,5 @@ message ClientSSLAuth { // An optional list of IP address and subnet masks that should be white // listed for access by the filter. If no list is provided, there is no // IP white list. - repeated string ip_white_list = 4; + repeated CidrRange ip_white_list = 4; } diff --git a/api/lds.proto b/api/lds.proto index eeff7dff2..7257e057e 100644 --- a/api/lds.proto +++ b/api/lds.proto @@ -56,10 +56,6 @@ message FilterChainMatch { // If non-empty, an IP address and prefix length to match addresses when the // listener is bound to 0.0.0.0/:: or when use_original_dst is specified. - message CidrRange { - string address_prefix = 1; - google.protobuf.UInt32Value prefix_len = 2; - } repeated CidrRange prefix_ranges = 3; // If non-empty, an IP address and suffix length to match addresses when the From 8638cbbeb358082e2107e5e465cc622481818cce Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 15:08:10 -0400 Subject: [PATCH 17/40] BUILD Signed-off-by: Shriram Rajagopalan --- api/filter/http/BUILD | 3 +++ api/filter/network/BUILD | 3 +++ 2 files changed, 6 insertions(+) diff --git a/api/filter/http/BUILD b/api/filter/http/BUILD index e5b186429..c3edcd7b6 100644 --- a/api/filter/http/BUILD +++ b/api/filter/http/BUILD @@ -30,6 +30,9 @@ api_proto_library( api_proto_library( name = "ip_tagging", srcs = ["ip_tagging.proto"], + deps = [ + "//api:address", + ], ) api_proto_library( diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD index 12f38b40e..985f4fd87 100644 --- a/api/filter/network/BUILD +++ b/api/filter/network/BUILD @@ -23,4 +23,7 @@ api_proto_library( api_proto_library( name = "client_ssl_auth", srcs = ["client_ssl_auth.proto"], + deps = [ + "//api:address", + ], ) From 8cb7a69c32f7a7e50823876d99145c2914636d65 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 16:28:01 -0400 Subject: [PATCH 18/40] fix Signed-off-by: Shriram Rajagopalan --- api/address.proto | 2 ++ ci/ci_steps.sh | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api/address.proto b/api/address.proto index 4739fbb60..04ea6db5c 100644 --- a/api/address.proto +++ b/api/address.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.api.v2; +import "google/protobuf/wrappers.proto"; + // [V2-API-DIFF] Addresses now have .proto structure. message Pipe { diff --git a/ci/ci_steps.sh b/ci/ci_steps.sh index 7542a0884..3d5a10409 100755 --- a/ci/ci_steps.sh +++ b/ci/ci_steps.sh @@ -7,11 +7,12 @@ set -e ENVOY_BUILD_SHA=44d539cb572d04c81b62425373440c54934cf267 # Lint travis file. -travis lint .travis.yml --skip-completion-check +#travis lint .travis.yml --skip-completion-check # Where the Envoy build takes place. export ENVOY_API_BUILD_DIR=/tmp/envoy-api-docker-build +TRAVIS_BUILD_DIR=`pwd` # Do a build matrix with different types of builds docs, coverage, bazel.release, etc. docker run -t -i -v "$ENVOY_API_BUILD_DIR":/build -v $TRAVIS_BUILD_DIR:/source \ - lyft/envoy-build:$ENVOY_BUILD_SHA /bin/bash -c "cd /source && ci/do_ci.sh $TEST_TYPE" + lyft/envoy-build:$ENVOY_BUILD_SHA /bin/bash -c "cd /source && ci/do_ci.sh bazel.test" From 7502b0c10efd885704e05a4c089bc8e4ed5d4b82 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 16:57:20 -0400 Subject: [PATCH 19/40] feedback Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 5 ---- api/filter/http/BUILD | 9 ++++-- api/filter/{ => http}/rate_limit.proto | 41 ++------------------------ api/filter/http/router.proto | 10 +++---- api/filter/network/BUILD | 10 +++++-- api/filter/network/rate_limit.proto | 19 ++++++++++++ 6 files changed, 39 insertions(+), 55 deletions(-) rename api/filter/{ => http}/rate_limit.proto (54%) create mode 100644 api/filter/network/rate_limit.proto diff --git a/api/filter/BUILD b/api/filter/BUILD index 68ed860ae..71601ab31 100644 --- a/api/filter/BUILD +++ b/api/filter/BUILD @@ -2,11 +2,6 @@ load("//bazel:api_build_system.bzl", "api_proto_library") licenses(["notice"]) # Apache 2 -api_proto_library( - name = "rate_limit", - srcs = ["rate_limit.proto"], -) - api_proto_library( name = "fault", srcs = ["fault.proto"], diff --git a/api/filter/http/BUILD b/api/filter/http/BUILD index c3edcd7b6..a2f0299c4 100644 --- a/api/filter/http/BUILD +++ b/api/filter/http/BUILD @@ -27,12 +27,15 @@ api_proto_library( srcs = ["transcoder.proto"], ) +api_proto_library( + name = "rate_limit", + srcs = ["rate_limit.proto"], +) + api_proto_library( name = "ip_tagging", srcs = ["ip_tagging.proto"], - deps = [ - "//api:address", - ], + deps = ["//api:address"], ) api_proto_library( diff --git a/api/filter/rate_limit.proto b/api/filter/http/rate_limit.proto similarity index 54% rename from api/filter/rate_limit.proto rename to api/filter/http/rate_limit.proto index edb286a34..af9d20540 100644 --- a/api/filter/rate_limit.proto +++ b/api/filter/http/rate_limit.proto @@ -1,33 +1,9 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; import "google/protobuf/duration.proto"; -// TODO(htuch): This should be the shared canonical RateLimitDescriptor -// when we import the rate limit protos -// (https://github.com/lyft/envoy-api/issues/26). -message RateLimitDescriptor { - message Entry { - string key = 1; - string value = 2; - } - repeated Entry entries = 1; -} - -message RateLimit { - // The human readable prefix to use when emitting statistics for the - // rate limit filter. See the statistics documentation for more information. - string stat_prefix = 1; - - // The rate limit domain to use in the rate limit service request. - string domain = 2; - - // The rate limit descriptor list to use in the rate limit service - // request. - repeated RateLimitDescriptor rate_limit_descriptors = 3; -} - // HTTP rate limit filter configuration. The HTTP rate limit filter will // call the rate limit service when the request’s route or virtual host has // one or more rate limit configurations that match the filter stage @@ -36,7 +12,7 @@ message RateLimit { // configuration results in a descriptor being sent to the rate limit // service. If the rate limit service is called, and the response for any // of the descriptors is over limit, a 429 response is returned. -message HTTPRateLimit { +message RateLimit { // The rate limit domain to use when calling the rate limit service. string domain = 1; // Specifies the rate limit configurations to be applied with the same @@ -53,16 +29,3 @@ message HTTPRateLimit { // set, this defaults to 20ms. google.protobuf.Duration timeout = 4; } - -// TCP rate limit filter configuration -message TCPRateLimit { - // The prefix to use when emitting statistics. - string stat_prefix = 1; - // The rate limit domain to use in the rate limit service request. - string domain = 2; - // The rate limit descriptor list to use in the rate limit service request. - repeated RateLimitDescriptor descriptors = 3; - // The timeout in milliseconds for the rate limit service RPC. If not - // set, this defaults to 20ms. - google.protobuf.Duration timeout = 4; -} diff --git a/api/filter/http/router.proto b/api/filter/http/router.proto index 39f8906b9..1788ae987 100644 --- a/api/filter/http/router.proto +++ b/api/filter/http/router.proto @@ -4,11 +4,11 @@ package envoy.api.v2.filter.http; import "google/protobuf/wrappers.proto"; -//The router filter implements HTTP forwarding. It will be used in almost -//all HTTP proxy scenarios that Envoy is deployed for. The filter’s main -//job is to follow the instructions specified in the configured route -//table. In addition to forwarding and redirection, the filter also handles -//retry, statistics, etc. +// The router filter implements HTTP forwarding. It will be used in almost +// all HTTP proxy scenarios that Envoy is deployed for. The filter’s main +// job is to follow the instructions specified in the configured route +// table. In addition to forwarding and redirection, the filter also handles +// retry, statistics, etc. message Router { // Whether the router generates dynamic cluster statistics. Defaults to // true. Can be disabled in high performance scenarios. diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD index 985f4fd87..c62e95b72 100644 --- a/api/filter/network/BUILD +++ b/api/filter/network/BUILD @@ -23,7 +23,11 @@ api_proto_library( api_proto_library( name = "client_ssl_auth", srcs = ["client_ssl_auth.proto"], - deps = [ - "//api:address", - ], + deps = ["//api:address"], +) + +api_proto_library( + name = "rate_limit", + srcs = ["rate_limit.proto"], + deps = ["//api:rls"], ) diff --git a/api/filter/network/rate_limit.proto b/api/filter/network/rate_limit.proto new file mode 100644 index 000000000..79db32886 --- /dev/null +++ b/api/filter/network/rate_limit.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package envoy.api.v2.filter.network; + +import "api/rls.proto"; +import "google/protobuf/duration.proto"; + +// TCP rate limit filter configuration +message RateLimit { + // The prefix to use when emitting statistics. + string stat_prefix = 1; + // The rate limit domain to use in the rate limit service request. + string domain = 2; + // The rate limit descriptor list to use in the rate limit service request. + repeated RateLimitDescriptor descriptors = 3; + // The timeout in milliseconds for the rate limit service RPC. If not + // set, this defaults to 20ms. + google.protobuf.Duration timeout = 4; +} From 81186d76aa23226a2f403fce84efd2c5fb585855 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:03:12 -0400 Subject: [PATCH 20/40] moving fault to http Signed-off-by: Shriram Rajagopalan --- api/filter/http/BUILD | 8 ++++++++ api/filter/{ => http}/fault.proto | 22 ++-------------------- api/filter/network/BUILD | 2 +- api/filter/network/mongo_proxy.proto | 2 +- 4 files changed, 12 insertions(+), 22 deletions(-) rename api/filter/{ => http}/fault.proto (83%) diff --git a/api/filter/http/BUILD b/api/filter/http/BUILD index a2f0299c4..8a2f87a1d 100644 --- a/api/filter/http/BUILD +++ b/api/filter/http/BUILD @@ -42,3 +42,11 @@ api_proto_library( name = "health_check", srcs = ["health_check.proto"], ) + +api_proto_library( + name = "fault", + srcs = ["fault.proto"], + deps = [ + "//api:rds", + ], +) diff --git a/api/filter/fault.proto b/api/filter/http/fault.proto similarity index 83% rename from api/filter/fault.proto rename to api/filter/http/fault.proto index 763a7738d..4d929173a 100644 --- a/api/filter/fault.proto +++ b/api/filter/http/fault.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.api.v2.filter; +package envoy.api.v2.filter.http; import "api/rds.proto"; @@ -27,7 +27,7 @@ message FaultDelay { // the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified // delay will be injected before a new request/operation. For TCP // connections, the proxying of the connection upstream will be delayed - // for the specified period. + // for the specified period. REQUIRED. google.protobuf.Duration fixed_delay = 3; } } @@ -102,21 +102,3 @@ message HTTPFault { // downstream_nodes list. repeated string downstream_nodes = 5; } - -// Faults can be injected into the connections from downstream by the -// Envoy, for testing the failure recovery capabilities of downstream -// services. Faults include aborting the connection from downstream -// service, and delaying proxying of connection to the destination. -message TCPFault { - // Delay proxying of the TCP connection for a specified period. - FaultDelay delay = 1; - - // Abort a specified percentage of downstream TCP connections without - // establishing an upstream connection. The connection will be abruptly reset. - FaultAbort abort = 2; - - // Specifies the name of the upstream cluster that the - // filter should match on. Fault injection will be restricted to connections - // bound to the specific upstream cluster. - string upstream_cluster = 3; -} diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD index c62e95b72..304bb7a35 100644 --- a/api/filter/network/BUILD +++ b/api/filter/network/BUILD @@ -6,7 +6,7 @@ api_proto_library( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], deps = [ - "//api/filter:fault", + "//api/filter/http:fault", ], ) diff --git a/api/filter/network/mongo_proxy.proto b/api/filter/network/mongo_proxy.proto index a1833503f..eb85bc093 100644 --- a/api/filter/network/mongo_proxy.proto +++ b/api/filter/network/mongo_proxy.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter.network; -import "api/filter/fault.proto"; +import "api/filter/http/fault.proto"; message MongoProxy { // The human readable prefix to use when emitting statistics for the From 95286bb28c2e974fbf58c27fe0681786ffed2325 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:03:30 -0400 Subject: [PATCH 21/40] stray build Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 api/filter/BUILD diff --git a/api/filter/BUILD b/api/filter/BUILD deleted file mode 100644 index 71601ab31..000000000 --- a/api/filter/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -load("//bazel:api_build_system.bzl", "api_proto_library") - -licenses(["notice"]) # Apache 2 - -api_proto_library( - name = "fault", - srcs = ["fault.proto"], - deps = [ - "//api:rds", - ], -) From 7ba311ccf39310a1ea9f25c2236f9e1766d0935b Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:07:19 -0400 Subject: [PATCH 22/40] beautify bazel build Signed-off-by: Shriram Rajagopalan --- api/filter/http/BUILD | 4 +--- api/filter/network/BUILD | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/api/filter/http/BUILD b/api/filter/http/BUILD index 8a2f87a1d..ce9587a1b 100644 --- a/api/filter/http/BUILD +++ b/api/filter/http/BUILD @@ -46,7 +46,5 @@ api_proto_library( api_proto_library( name = "fault", srcs = ["fault.proto"], - deps = [ - "//api:rds", - ], + deps = ["//api:rds"], ) diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD index 304bb7a35..52e5fcd98 100644 --- a/api/filter/network/BUILD +++ b/api/filter/network/BUILD @@ -5,9 +5,7 @@ licenses(["notice"]) # Apache 2 api_proto_library( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], - deps = [ - "//api/filter/http:fault", - ], + deps = ["//api/filter/http:fault"], ) api_proto_library( From 34128800285b889969e8d9333e8167d450ce4970 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:10:45 -0400 Subject: [PATCH 23/40] REQUIRED fields Signed-off-by: Shriram Rajagopalan --- api/filter/http/buffer.proto | 4 ++-- api/filter/http/health_check.proto | 4 ++-- api/filter/network/redis_proxy.proto | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/filter/http/buffer.proto b/api/filter/http/buffer.proto index 82ddfeb14..6a6367ad7 100644 --- a/api/filter/http/buffer.proto +++ b/api/filter/http/buffer.proto @@ -11,9 +11,9 @@ import "google/protobuf/wrappers.proto"; // requests and high network latency. message Buffer { // The maximum request size that the filter will before the connection - // manager will stop buffering and return a 413 response. + // manager will stop buffering and return a 413 response. REQUIRED. google.protobuf.UInt32Value max_request_bytes = 1; // The maximum amount of time that the filter will wait for a complete - // request before returning a 408 response. + // request before returning a 408 response. REQUIRED. google.protobuf.Duration max_request_time = 2; } diff --git a/api/filter/http/health_check.proto b/api/filter/http/health_check.proto index b2070fd86..7c265998e 100644 --- a/api/filter/http/health_check.proto +++ b/api/filter/http/health_check.proto @@ -10,12 +10,12 @@ import "google/protobuf/wrappers.proto"; // admin endpoint has been called. (The /healthcheck/ok admin endpoint // reverses this behavior). message HealthCheck { - // Specifies whether the filter operates in pass through mode or not. + // Specifies whether the filter operates in pass through mode or not. REQUIRED. google.protobuf.BoolValue pass_through_mode = 1; // Specifies the incoming HTTP endpoint that should be considered the // health check endpoint. For example /healthcheck. string endpoint = 2; // If operating in pass through mode, the amount of time in milliseconds - // that the filter should cache the upstream response. + // that the filter should cache the upstream response. REQUIRED. google.protobuf.Duration cache_time = 3; } diff --git a/api/filter/network/redis_proxy.proto b/api/filter/network/redis_proxy.proto index 15a32d959..131cb2f07 100644 --- a/api/filter/network/redis_proxy.proto +++ b/api/filter/network/redis_proxy.proto @@ -25,7 +25,7 @@ message RedisProxy { // that the next command is being processed by the backend. The only // exception to this behavior is when a connection to a backend is not // yet established. In that case, the connect timeout on the cluster - // will govern the timeout until the connection is ready. + // will govern the timeout until the connection is ready. REQUIRED. google.protobuf.Duration op_timeout = 1; } } From f01ef4f79b0b7e60690d0acfa4a0f6574c2b3357 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:29:57 -0400 Subject: [PATCH 24/40] bad commit Signed-off-by: Shriram Rajagopalan --- ci/ci_steps.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/ci_steps.sh b/ci/ci_steps.sh index 9230a07ff..6170f415e 100755 --- a/ci/ci_steps.sh +++ b/ci/ci_steps.sh @@ -12,7 +12,6 @@ ENVOY_BUILD_SHA=114e24c6fd05fc026492e9d2ca5608694e5ea59d # Where the Envoy build takes place. export ENVOY_API_BUILD_DIR=/tmp/envoy-api-docker-build -TRAVIS_BUILD_DIR=`pwd` # Do a build matrix with different types of builds docs, coverage, bazel.release, etc. docker run -t -i -v "$ENVOY_API_BUILD_DIR":/build -v $TRAVIS_BUILD_DIR:/source \ lyft/envoy-build:$ENVOY_BUILD_SHA /bin/bash -c "cd /source && ci/do_ci.sh bazel.test" From 24da3eb0013c4e90e07f61790568f8f3cc2683d1 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:31:31 -0400 Subject: [PATCH 25/40] fix Signed-off-by: Shriram Rajagopalan --- api/filter/network/redis_proxy.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/filter/network/redis_proxy.proto b/api/filter/network/redis_proxy.proto index 131cb2f07..3fd672626 100644 --- a/api/filter/network/redis_proxy.proto +++ b/api/filter/network/redis_proxy.proto @@ -12,10 +12,7 @@ message RedisProxy { // Indicates the upstream cluster to which the operation should be routed to. string cluster = 2; - // Network settings for the connection pool to the upstream cluster. - ConnPoolSettings settings = 3; - - // Redis connection pool settings + // Redis connection pool settings. message ConnPoolSettings { // Per-operation timeout. See // https://developers.google.com/protocol-buffers/docs/proto3#json for @@ -28,4 +25,7 @@ message RedisProxy { // will govern the timeout until the connection is ready. REQUIRED. google.protobuf.Duration op_timeout = 1; } + + // Network settings for the connection pool to the upstream cluster. + ConnPoolSettings settings = 3; } From 3abb34534cb0535ac91ae6820f351872e92fe2f5 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:56:32 -0400 Subject: [PATCH 26/40] fix Signed-off-by: Shriram Rajagopalan --- api/filter/http/transcoder.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/filter/http/transcoder.proto b/api/filter/http/transcoder.proto index 10f0b47b4..9e0f298e2 100644 --- a/api/filter/http/transcoder.proto +++ b/api/filter/http/transcoder.proto @@ -21,10 +21,6 @@ message GrpcJsonTranscoder { // translated. repeated string services = 2; - // Control options for response json. These options are passed directly - // to JsonPrintOptions. - PrintOptions print_options = 3; - // Control options for response json. These options are passed directly // to JsonPrintOptions. message PrintOptions { @@ -49,4 +45,8 @@ message GrpcJsonTranscoder { // names. Default to false. bool preserve_proto_field_names = 4; }; + + // Control options for response json. These options are passed directly + // to JsonPrintOptions. + PrintOptions print_options = 3; } From c7784fb513c1772e06a35c240dd0f6ce202292b6 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 17:57:22 -0400 Subject: [PATCH 27/40] bad commit Signed-off-by: Shriram Rajagopalan --- ci/ci_steps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/ci_steps.sh b/ci/ci_steps.sh index 6170f415e..52ae200e9 100755 --- a/ci/ci_steps.sh +++ b/ci/ci_steps.sh @@ -7,7 +7,7 @@ set -e ENVOY_BUILD_SHA=114e24c6fd05fc026492e9d2ca5608694e5ea59d # Lint travis file. -#travis lint .travis.yml --skip-completion-check +travis lint .travis.yml --skip-completion-check # Where the Envoy build takes place. export ENVOY_API_BUILD_DIR=/tmp/envoy-api-docker-build From 77ce4e86f0030315986a53cb9cc2e09b1e45b603 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 18:04:41 -0400 Subject: [PATCH 28/40] fix Signed-off-by: Shriram Rajagopalan --- ci/ci_steps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/ci_steps.sh b/ci/ci_steps.sh index 52ae200e9..cc18881e8 100755 --- a/ci/ci_steps.sh +++ b/ci/ci_steps.sh @@ -14,4 +14,4 @@ export ENVOY_API_BUILD_DIR=/tmp/envoy-api-docker-build # Do a build matrix with different types of builds docs, coverage, bazel.release, etc. docker run -t -i -v "$ENVOY_API_BUILD_DIR":/build -v $TRAVIS_BUILD_DIR:/source \ - lyft/envoy-build:$ENVOY_BUILD_SHA /bin/bash -c "cd /source && ci/do_ci.sh bazel.test" + lyft/envoy-build:$ENVOY_BUILD_SHA /bin/bash -c "cd /source && ci/do_ci.sh $TEST_TYPE" From c3665d46860f348c349abc3caaa6f873deb3ee3e Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 18:28:50 -0400 Subject: [PATCH 29/40] new option to route Signed-off-by: Shriram Rajagopalan --- api/filter/http/router.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/filter/http/router.proto b/api/filter/http/router.proto index 1788ae987..2644ebca7 100644 --- a/api/filter/http/router.proto +++ b/api/filter/http/router.proto @@ -13,4 +13,9 @@ message Router { // Whether the router generates dynamic cluster statistics. Defaults to // true. Can be disabled in high performance scenarios. google.protobuf.BoolValue dynamic_stats = 1; + // Whether to start a child span for egress routed calls. This can be + // useful in scenarios where other filters (auth, ratelimit, etc.) make + // outbound calls and have child spans rooted at the same ingress + // parent. Defaults to false. + bool start_child_span = 2; } From 74d96b63d831d8a21427c870f9012baf666eec7b Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 18:47:24 -0400 Subject: [PATCH 30/40] fix cross package ref Signed-off-by: Shriram Rajagopalan --- api/filter/network/mongo_proxy.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/filter/network/mongo_proxy.proto b/api/filter/network/mongo_proxy.proto index eb85bc093..a51af7b39 100644 --- a/api/filter/network/mongo_proxy.proto +++ b/api/filter/network/mongo_proxy.proto @@ -18,5 +18,5 @@ message MongoProxy { // applied to the following MongoDB operations: Query, Insert, GetMore, // and KillCursors. Once an active delay is in progress, all incoming // data up until the timer event fires will be a part of the delay. - FaultDelay delay = 3; + envoy.api.v2.filter.http.FaultDelay delay = 3; } From c50c5e9478d278ea5255b6ecc517fce237b44c81 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 20:05:22 -0400 Subject: [PATCH 31/40] nits Signed-off-by: Shriram Rajagopalan --- api/filter/BUILD | 8 +++++++ api/filter/README.md | 6 +++-- api/filter/fault.proto | 31 ++++++++++++++++++++++++++ api/filter/http/BUILD | 5 ++++- api/filter/http/fault.proto | 33 +++------------------------- api/filter/network/BUILD | 2 +- api/filter/network/mongo_proxy.proto | 4 ++-- 7 files changed, 53 insertions(+), 36 deletions(-) create mode 100644 api/filter/BUILD create mode 100644 api/filter/fault.proto diff --git a/api/filter/BUILD b/api/filter/BUILD new file mode 100644 index 000000000..de3473b43 --- /dev/null +++ b/api/filter/BUILD @@ -0,0 +1,8 @@ +load("//bazel:api_build_system.bzl", "api_proto_library") + +licenses(["notice"]) # Apache 2 + +api_proto_library( + name = "fault", + srcs = ["fault.proto"], +) diff --git a/api/filter/README.md b/api/filter/README.md index dd923ed61..04b72afff 100644 --- a/api/filter/README.md +++ b/api/filter/README.md @@ -1,7 +1,9 @@ ## NOTE -If a filter configuration is not captured in the proto specification, you can still supply plain JSON configuration objects -for such filters by setting the `"deprecated_v1"` field to true in the filter's configuration. For example, +If a filter configuration is not captured in the proto specification, you +can still supply plain JSON configuration objects for such filters by +setting the `"deprecated_v1"` field to true in the filter's +configuration. For example, ```json { diff --git a/api/filter/fault.proto b/api/filter/fault.proto new file mode 100644 index 000000000..a796d8a35 --- /dev/null +++ b/api/filter/fault.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package envoy.api.v2.filter; + +import "google/protobuf/duration.proto"; + +// Delay specification is used to inject latency into the +// HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections. +message FaultDelay { + enum FaultDelayType { + // Fixed delay (step function). + FIXED = 0; + } + + // Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is supported. + FaultDelayType type = 1; + + // An integer between 0-100 indicating the percentage of operations/connection requests + // on which the delay will be injected. + uint32 percent = 2; + + oneof fault_delay_type { + // Add a fixed delay before forwarding the operation upstream. See + // https://developers.google.com/protocol-buffers/docs/proto3#json for + // the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified + // delay will be injected before a new request/operation. For TCP + // connections, the proxying of the connection upstream will be delayed + // for the specified period. REQUIRED. + google.protobuf.Duration fixed_delay = 3; + } +} diff --git a/api/filter/http/BUILD b/api/filter/http/BUILD index ce9587a1b..6e9d74451 100644 --- a/api/filter/http/BUILD +++ b/api/filter/http/BUILD @@ -46,5 +46,8 @@ api_proto_library( api_proto_library( name = "fault", srcs = ["fault.proto"], - deps = ["//api:rds"], + deps = [ + "//api:rds", + "//api/filter:fault", + ], ) diff --git a/api/filter/http/fault.proto b/api/filter/http/fault.proto index 4d929173a..360a270d2 100644 --- a/api/filter/http/fault.proto +++ b/api/filter/http/fault.proto @@ -3,37 +3,10 @@ syntax = "proto3"; package envoy.api.v2.filter.http; import "api/rds.proto"; +import "api/filter/fault.proto"; -import "google/protobuf/duration.proto"; - -// Delay specification is used to inject latency into the -// HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections. -message FaultDelay { - enum FaultDelayType { - // Fixed delay (step function). - FIXED = 0; - } - - // Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is supported. - FaultDelayType type = 1; - - // An integer between 0-100 indicating the percentage of operations/connection requests - // on which the delay will be injected. - uint32 percent = 2; - - oneof fault_delay_type { - // Add a fixed delay before forwarding the operation upstream. See - // https://developers.google.com/protocol-buffers/docs/proto3#json for - // the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified - // delay will be injected before a new request/operation. For TCP - // connections, the proxying of the connection upstream will be delayed - // for the specified period. REQUIRED. - google.protobuf.Duration fixed_delay = 3; - } -} - -// Abort specification is used to prematurely abort a HTTP/gRPC/Mongo/Redis operation/TCP connection -// with a pre-specified error code. +// Abort specification is used to prematurely abort a HTTP/gRPC/Mongo/Redis +// operation/TCP connection with a pre-specified error code. message FaultAbort { // An integer between 0-100 indicating the percentage of requests/operations/connections // that will be aborted with the error code provided. diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD index 52e5fcd98..8b285f29c 100644 --- a/api/filter/network/BUILD +++ b/api/filter/network/BUILD @@ -5,7 +5,7 @@ licenses(["notice"]) # Apache 2 api_proto_library( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], - deps = ["//api/filter/http:fault"], + deps = ["//api/filter:fault"], ) api_proto_library( diff --git a/api/filter/network/mongo_proxy.proto b/api/filter/network/mongo_proxy.proto index a51af7b39..a1833503f 100644 --- a/api/filter/network/mongo_proxy.proto +++ b/api/filter/network/mongo_proxy.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter.network; -import "api/filter/http/fault.proto"; +import "api/filter/fault.proto"; message MongoProxy { // The human readable prefix to use when emitting statistics for the @@ -18,5 +18,5 @@ message MongoProxy { // applied to the following MongoDB operations: Query, Insert, GetMore, // and KillCursors. Once an active delay is in progress, all incoming // data up until the timer event fires will be a part of the delay. - envoy.api.v2.filter.http.FaultDelay delay = 3; + FaultDelay delay = 3; } From 69c9237a45198b0a6bdce6bc06a47b1c4654041c Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 24 Oct 2017 22:16:18 -0400 Subject: [PATCH 32/40] circle ci Signed-off-by: Shriram Rajagopalan --- .circleci/config.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..39b4189e9 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,25 @@ +version: 2 +jobs: + test: + docker: + - image: lyft/envoy-build:114e24c6fd05fc026492e9d2ca5608694e5ea59d + resource_class: xlarge + working_directory: /source + steps: + - checkout + - run: ci/do_ci.sh bazel.test + docs: + docker: + - image: lyft/envoy-build:114e24c6fd05fc026492e9d2ca5608694e5ea59d + resource_class: xlarge + working_directory: /source + steps: + - checkout + - run: ci/do_ci.sh bazel.docs + +workflows: + version: 2 + all: + jobs: + - test + - docs From b5e4eeae7482f89aaaaec29a1698215b0187dba6 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Wed, 25 Oct 2017 10:41:13 -0400 Subject: [PATCH 33/40] updating docs Signed-off-by: Shriram Rajagopalan --- XDS_PROTOCOL.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/XDS_PROTOCOL.md b/XDS_PROTOCOL.md index f433072ad..81f1fc528 100644 --- a/XDS_PROTOCOL.md +++ b/XDS_PROTOCOL.md @@ -191,11 +191,15 @@ drop can't be tolerated, traffic drop could have been avoided by providing a CDS/EDS update with both __X__ and __Y__, then the RDS update repointing from __X__ to __Y__ and then a CDS/EDS update dropping __X__. -In general, to avoid traffic drop: -* Sequencing should be make before break. -* LDS and CDS updates should arrive before the respective RDS and EDS updates. -* CDS/EDS resources corresponding to routes in LDS/RDS should be available at - update. +In general, to avoid traffic drop, sequencing of updates should follow a +`make before break` model, wherein +* CDS updates (if any) must always be pushed first. +* EDS updates (if any) must arrive after CDS updates for the respective clusters. +* LDS updates must arrive after corresponding CDS/EDS updates. +* RDS updates related to the newly added listeners must arrive in the end. + +xDS updates can be pushed independently if no new clusters/routes/listeners +are added. ### Aggregated Discovery Services (ADS) From 91bc65f2e0c3adc0788bf4a0d737e5ebd806f512 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Wed, 25 Oct 2017 13:25:08 -0400 Subject: [PATCH 34/40] fix 1 Signed-off-by: Shriram Rajagopalan --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39b4189e9..3897e6fba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: working_directory: /source steps: - checkout - - run: ci/do_ci.sh bazel.test + - run: ci/do_circle_ci.sh bazel.test docs: docker: - image: lyft/envoy-build:114e24c6fd05fc026492e9d2ca5608694e5ea59d @@ -15,7 +15,7 @@ jobs: working_directory: /source steps: - checkout - - run: ci/do_ci.sh bazel.docs + - run: ci/do_circle_ci.sh bazel.docs workflows: version: 2 From c22f48230d99394e927157d8ca2e6e0e019d44fa Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Wed, 25 Oct 2017 13:30:38 -0400 Subject: [PATCH 35/40] revert Signed-off-by: Shriram Rajagopalan --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3897e6fba..39b4189e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: working_directory: /source steps: - checkout - - run: ci/do_circle_ci.sh bazel.test + - run: ci/do_ci.sh bazel.test docs: docker: - image: lyft/envoy-build:114e24c6fd05fc026492e9d2ca5608694e5ea59d @@ -15,7 +15,7 @@ jobs: working_directory: /source steps: - checkout - - run: ci/do_circle_ci.sh bazel.docs + - run: ci/do_ci.sh bazel.docs workflows: version: 2 From 1a391c426653cc7828902b8e8e440ddfa019e635 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Wed, 25 Oct 2017 13:39:09 -0400 Subject: [PATCH 36/40] remove travis Signed-off-by: Shriram Rajagopalan --- .travis.yml | 16 ---------------- ci/build_setup.sh | 11 ++++++----- ci/ci_steps.sh | 17 ----------------- ci/run_envoy_docker.sh | 11 ----------- 4 files changed, 6 insertions(+), 49 deletions(-) delete mode 100644 .travis.yml delete mode 100755 ci/ci_steps.sh delete mode 100755 ci/run_envoy_docker.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1db9207f1..000000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -language: minimal -sudo: required -services: - - docker -install: - - gem install travis --no-rdoc --no-ri -matrix: - fast_finish: true -env: - - TEST_TYPE=bazel.test - - TEST_TYPE=bazel.docs -script: ./ci/ci_steps.sh - -branches: - only: - - master diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 4643f757d..e879afd81 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -9,11 +9,12 @@ NUM_CPUS=`grep -c ^processor /proc/cpuinfo` export ENVOY_SRCDIR=/source export BUILD_DIR=/build -if [[ ! -d "${BUILD_DIR}" ]] -then - echo "${BUILD_DIR} mount missing - did you forget -v :${BUILD_DIR}?" - exit 1 -fi +mkdir -p ${BUILD_DIR} +# if [[ ! -d "${BUILD_DIR}" ]] +# then +# echo "${BUILD_DIR} mount missing - did you forget -v :${BUILD_DIR}?" +# exit 1 +# fi # Create a fake home. Python site libs tries to do getpwuid(3) if we don't and # the CI Docker image gets confused as it has no passwd entry when running diff --git a/ci/ci_steps.sh b/ci/ci_steps.sh deleted file mode 100755 index cc18881e8..000000000 --- a/ci/ci_steps.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Script that lists all the steps take by the CI system when doing Envoy builds. -set -e - -# We reuse the https://github.com/lyft/envoy/ CI image here to get Bazel. -ENVOY_BUILD_SHA=114e24c6fd05fc026492e9d2ca5608694e5ea59d - -# Lint travis file. -travis lint .travis.yml --skip-completion-check - -# Where the Envoy build takes place. -export ENVOY_API_BUILD_DIR=/tmp/envoy-api-docker-build - -# Do a build matrix with different types of builds docs, coverage, bazel.release, etc. -docker run -t -i -v "$ENVOY_API_BUILD_DIR":/build -v $TRAVIS_BUILD_DIR:/source \ - lyft/envoy-build:$ENVOY_BUILD_SHA /bin/bash -c "cd /source && ci/do_ci.sh $TEST_TYPE" diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh deleted file mode 100755 index c14118709..000000000 --- a/ci/run_envoy_docker.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -e - -[[ -z "${IMAGE_ID}" ]] && IMAGE_ID="latest" -[[ -z "${ENVOY_API_DOCKER_BUILD_DIR}" ]] && ENVOY_API_DOCKER_BUILD_DIR=/tmp/envoy-api-docker-build - -mkdir -p "${ENVOY_API_DOCKER_BUILD_DIR}" -docker pull lyft/envoy-build:"${IMAGE_ID}" -docker run -t -i -u $(id -u):$(id -g) -v "${ENVOY_API_DOCKER_BUILD_DIR}":/build \ - -v "$PWD":/source lyft/envoy-build:"${IMAGE_ID}" /bin/bash -c "cd source && $*" From bc04f8023c23c8d2efd2a4da297009e4df12dbf8 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Wed, 25 Oct 2017 13:45:59 -0400 Subject: [PATCH 37/40] nits Signed-off-by: Shriram Rajagopalan --- XDS_PROTOCOL.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/XDS_PROTOCOL.md b/XDS_PROTOCOL.md index 81f1fc528..146bbfd63 100644 --- a/XDS_PROTOCOL.md +++ b/XDS_PROTOCOL.md @@ -197,9 +197,10 @@ In general, to avoid traffic drop, sequencing of updates should follow a * EDS updates (if any) must arrive after CDS updates for the respective clusters. * LDS updates must arrive after corresponding CDS/EDS updates. * RDS updates related to the newly added listeners must arrive in the end. +* Stale CDS clusters (ones no longer being referenced) can then be removed. xDS updates can be pushed independently if no new clusters/routes/listeners -are added. +are added or or if it's acceptable to temporarily drop traffic during updates. ### Aggregated Discovery Services (ADS) From 5710cec45c62cd25154a6fbf050364896b49f3e8 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Wed, 25 Oct 2017 14:31:28 -0400 Subject: [PATCH 38/40] nits Signed-off-by: Shriram Rajagopalan --- XDS_PROTOCOL.md | 3 ++- ci/build_setup.sh | 5 ----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/XDS_PROTOCOL.md b/XDS_PROTOCOL.md index 146bbfd63..4c947c54d 100644 --- a/XDS_PROTOCOL.md +++ b/XDS_PROTOCOL.md @@ -197,7 +197,8 @@ In general, to avoid traffic drop, sequencing of updates should follow a * EDS updates (if any) must arrive after CDS updates for the respective clusters. * LDS updates must arrive after corresponding CDS/EDS updates. * RDS updates related to the newly added listeners must arrive in the end. -* Stale CDS clusters (ones no longer being referenced) can then be removed. +* Stale CDS clusters and related EDS endpoints (ones no longer being + referenced) can then be removed. xDS updates can be pushed independently if no new clusters/routes/listeners are added or or if it's acceptable to temporarily drop traffic during updates. diff --git a/ci/build_setup.sh b/ci/build_setup.sh index e879afd81..2d0f5d9b4 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -10,11 +10,6 @@ export ENVOY_SRCDIR=/source export BUILD_DIR=/build mkdir -p ${BUILD_DIR} -# if [[ ! -d "${BUILD_DIR}" ]] -# then -# echo "${BUILD_DIR} mount missing - did you forget -v :${BUILD_DIR}?" -# exit 1 -# fi # Create a fake home. Python site libs tries to do getpwuid(3) if we don't and # the CI Docker image gets confused as it has no passwd entry when running From 6f44ef73cfa8a7a2d56bc3516e574410bf3a96f4 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Wed, 25 Oct 2017 16:03:42 -0400 Subject: [PATCH 39/40] typo Signed-off-by: Shriram Rajagopalan --- XDS_PROTOCOL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/XDS_PROTOCOL.md b/XDS_PROTOCOL.md index 4c947c54d..282b5b418 100644 --- a/XDS_PROTOCOL.md +++ b/XDS_PROTOCOL.md @@ -201,7 +201,7 @@ In general, to avoid traffic drop, sequencing of updates should follow a referenced) can then be removed. xDS updates can be pushed independently if no new clusters/routes/listeners -are added or or if it's acceptable to temporarily drop traffic during updates. +are added or if it's acceptable to temporarily drop traffic during updates. ### Aggregated Discovery Services (ADS) From 2c138d9a540e9bd7e6d5c9dc9cd98b9cef691fc0 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Thu, 26 Oct 2017 12:12:51 -0400 Subject: [PATCH 40/40] clarifications Signed-off-by: Shriram Rajagopalan --- XDS_PROTOCOL.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/XDS_PROTOCOL.md b/XDS_PROTOCOL.md index 282b5b418..c6ef39164 100644 --- a/XDS_PROTOCOL.md +++ b/XDS_PROTOCOL.md @@ -201,7 +201,13 @@ In general, to avoid traffic drop, sequencing of updates should follow a referenced) can then be removed. xDS updates can be pushed independently if no new clusters/routes/listeners -are added or if it's acceptable to temporarily drop traffic during updates. +are added or if it's acceptable to temporarily drop traffic during +updates. Note that in case of LDS updates, the listeners will be warmed +before they receive traffic, i.e. the dependent routes are fetched through +RDS if configured. On the other hand, clusters are not warmed when +adding/removing/updating clusters. Similarly, routes are not warmed -- +i.e., the management plane must ensure that clusters referenced by a route +are in place, before pushing the updates for a rotue. ### Aggregated Discovery Services (ADS)