From 4160e077e9047d27a41b02a5a013c653ce44e12d Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Sat, 24 Jul 2021 02:14:27 +0900 Subject: [PATCH 01/12] access logging: gRPC logger retry stream Signed-off-by: Shikugawa --- .../access_loggers/grpc/v3/als.proto | 9 +++- .../access_loggers/grpc/v4alpha/als.proto | 9 +++- .../access_loggers/grpc/v3/als.proto | 9 +++- .../access_loggers/grpc/v4alpha/als.proto | 9 +++- source/common/grpc/async_client_impl.cc | 2 +- .../common/grpc_access_logger.h | 37 +++++++++++--- .../grpc/grpc_access_log_impl.cc | 14 ++++-- .../grpc/grpc_access_log_impl.h | 11 +++-- .../open_telemetry/grpc_access_log_impl.cc | 14 ++++-- .../open_telemetry/grpc_access_log_impl.h | 11 +++-- .../common/grpc_access_logger_test.cc | 49 ++++++++++++++----- 11 files changed, 130 insertions(+), 44 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index fa0a9f0f820d5..c62eb81513ee4 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v3; +import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 8] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.accesslog.v2.CommonGrpcAccessLogConfig"; @@ -86,4 +87,10 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Sets the retry policy when the establishment of a gRPC stream fails. + // If the stream succeeds once in establishing If the stream succeeds + // at least once in establishing itself, no retry will be performed + // no matter what gRPC status is received. + config.core.v3.RetryPolicy grpc_stream_retry_policy = 7; } diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 9e6fb1e48386e..762995032c625 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v4alpha; +import "envoy/config/core/v4alpha/base.proto"; import "envoy/config/core/v4alpha/config_source.proto"; import "envoy/config/core/v4alpha/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 8] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; @@ -86,4 +87,10 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Sets the retry policy when the establishment of a gRPC stream fails. + // If the stream succeeds once in establishing If the stream succeeds + // at least once in establishing itself, no retry will be performed + // no matter what gRPC status is received. + config.core.v4alpha.RetryPolicy grpc_stream_retry_policy = 7; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index fa0a9f0f820d5..c62eb81513ee4 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v3; +import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 8] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.accesslog.v2.CommonGrpcAccessLogConfig"; @@ -86,4 +87,10 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Sets the retry policy when the establishment of a gRPC stream fails. + // If the stream succeeds once in establishing If the stream succeeds + // at least once in establishing itself, no retry will be performed + // no matter what gRPC status is received. + config.core.v3.RetryPolicy grpc_stream_retry_policy = 7; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 9e6fb1e48386e..762995032c625 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v4alpha; +import "envoy/config/core/v4alpha/base.proto"; import "envoy/config/core/v4alpha/config_source.proto"; import "envoy/config/core/v4alpha/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 8] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; @@ -86,4 +87,10 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Sets the retry policy when the establishment of a gRPC stream fails. + // If the stream succeeds once in establishing If the stream succeeds + // at least once in establishing itself, no retry will be performed + // no matter what gRPC status is received. + config.core.v4alpha.RetryPolicy grpc_stream_retry_policy = 7; } diff --git a/source/common/grpc/async_client_impl.cc b/source/common/grpc/async_client_impl.cc index 53aeacf6236e6..5f4bdb4f80e33 100644 --- a/source/common/grpc/async_client_impl.cc +++ b/source/common/grpc/async_client_impl.cc @@ -51,7 +51,7 @@ RawAsyncStream* AsyncClientImpl::startRaw(absl::string_view service_full_name, auto grpc_stream = std::make_unique(*this, service_full_name, method_name, callbacks, options); - grpc_stream->initialize(false); + grpc_stream->initialize(options.buffer_body_for_retry); if (grpc_stream->hasResetStream()) { return nullptr; } diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 2e008a9c1522f..598fa2e0f546c 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -77,13 +77,15 @@ template class GrpcAccessLogge template class GrpcAccessLogClient { public: GrpcAccessLogClient(const Grpc::RawAsyncClientSharedPtr& client, - const Protobuf::MethodDescriptor& service_method) - : GrpcAccessLogClient(client, service_method, absl::nullopt) {} + const Protobuf::MethodDescriptor& service_method, + const absl::optional retry_policy) + : GrpcAccessLogClient(client, service_method, retry_policy, absl::nullopt) {} GrpcAccessLogClient(const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& service_method, + const absl::optional retry_policy, envoy::config::core::v3::ApiVersion transport_api_version) : client_(client), service_method_(service_method), - transport_api_version_(transport_api_version) {} + transport_api_version_(transport_api_version), grpc_stream_retry_policy_(retry_policy) {} public: struct LocalStream : public Grpc::AsyncStreamCallbacks { @@ -115,8 +117,7 @@ template class GrpcAccessLogClient { } if (stream_->stream_ == nullptr) { - stream_->stream_ = - client_->start(service_method_, *stream_, Http::AsyncClient::StreamOptions()); + stream_->stream_ = client_->start(service_method_, *stream_, createStreamOptionsForRetry()); } if (stream_->stream_ != nullptr) { @@ -135,10 +136,28 @@ template class GrpcAccessLogClient { return true; } + Http::AsyncClient::StreamOptions createStreamOptionsForRetry() { + auto opt = Http::AsyncClient::StreamOptions(); + + if (!grpc_stream_retry_policy_) { + return opt; + } + + envoy::config::route::v3::RetryPolicy retry_policy; + retry_policy.mutable_num_retries()->set_value( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(*grpc_stream_retry_policy_, num_retries, 1)); + retry_policy.set_retry_on("connect-failure"); + + opt.setBufferBodyForRetry(true); + opt.setRetryPolicy(retry_policy); + return opt; + } + Grpc::AsyncClient client_; std::unique_ptr stream_; const Protobuf::MethodDescriptor& service_method_; const absl::optional transport_api_version_; + const absl::optional grpc_stream_retry_policy_; }; } // namespace Detail @@ -172,16 +191,18 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger retry_policy) : GrpcAccessLogger(client, buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, - scope, access_log_prefix, service_method, absl::nullopt) {} + scope, access_log_prefix, service_method, retry_policy, absl::nullopt) {} GrpcAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope, std::string access_log_prefix, const Protobuf::MethodDescriptor& service_method, + const absl::optional retry_policy, envoy::config::core::v3::ApiVersion transport_api_version) - : client_(client, service_method, transport_api_version), + : client_(client, service_method, retry_policy, transport_api_version), buffer_flush_interval_msec_(buffer_flush_interval_msec), flush_timer_(dispatcher.createTimer([this]() { flush(); diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 1e5c68f0f3732..17b3f28cc06d9 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -16,7 +16,8 @@ namespace AccessLoggers { namespace GrpcCommon { GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( - const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, + const Grpc::RawAsyncClientSharedPtr& client, + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, envoy::config::core::v3::ApiVersion transport_api_version) @@ -26,8 +27,11 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( Grpc::VersionedMethods("envoy.service.accesslog.v3.AccessLogService.StreamAccessLogs", "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs") .getMethodDescriptorForVersion(transport_api_version), + config.has_grpc_stream_retry_policy() + ? absl::make_optional(config.grpc_stream_retry_policy()) + : absl::nullopt, transport_api_version), - log_name_(log_name), local_info_(local_info) {} + log_name_(config.log_name()), local_info_(local_info) {} void GrpcAccessLoggerImpl::addEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { message_.mutable_http_logs()->mutable_log_entry()->Add(std::move(entry)); @@ -59,9 +63,9 @@ GrpcAccessLoggerImpl::SharedPtr GrpcAccessLoggerCacheImpl::createLogger( const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope) { - return std::make_shared(client, config.log_name(), - buffer_flush_interval_msec, max_buffer_size_bytes, - dispatcher, local_info_, scope, transport_version); + return std::make_shared(client, config, buffer_flush_interval_msec, + max_buffer_size_bytes, dispatcher, local_info_, + scope, transport_version); } } // namespace GrpcCommon diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 07bf8400fdc1c..eb27842281fca 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -23,11 +23,12 @@ class GrpcAccessLoggerImpl envoy::service::accesslog::v3::StreamAccessLogsMessage, envoy::service::accesslog::v3::StreamAccessLogsResponse> { public: - GrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, - envoy::config::core::v3::ApiVersion transport_api_version); + GrpcAccessLoggerImpl( + const Grpc::RawAsyncClientSharedPtr& client, + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, + std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, + envoy::config::core::v3::ApiVersion transport_api_version); private: // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc index cc84242c93d67..1c06854629ae4 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc @@ -21,7 +21,8 @@ namespace AccessLoggers { namespace OpenTelemetry { GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( - const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, + const Grpc::RawAsyncClientSharedPtr& client, + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, envoy::config::core::v3::ApiVersion transport_api_version) @@ -31,8 +32,11 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( Grpc::VersionedMethods("opentelemetry.proto.collector.logs.v1.LogsService.Export", "opentelemetry.proto.collector.logs.v1.LogsService.Export") .getMethodDescriptorForVersion(transport_api_version), + config.has_grpc_stream_retry_policy() + ? absl::make_optional(config.grpc_stream_retry_policy()) + : absl::nullopt, transport_api_version) { - initMessageRoot(log_name, local_info); + initMessageRoot(config.log_name(), local_info); } namespace { @@ -82,9 +86,9 @@ GrpcAccessLoggerImpl::SharedPtr GrpcAccessLoggerCacheImpl::createLogger( const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope) { - return std::make_shared(client, config.log_name(), - buffer_flush_interval_msec, max_buffer_size_bytes, - dispatcher, local_info_, scope, transport_version); + return std::make_shared(client, config, buffer_flush_interval_msec, + max_buffer_size_bytes, dispatcher, local_info_, + scope, transport_version); } } // namespace OpenTelemetry diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h index 52e61e134ee3b..524d5f182a644 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h @@ -35,11 +35,12 @@ class GrpcAccessLoggerImpl ProtobufWkt::Empty, opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest, opentelemetry::proto::collector::logs::v1::ExportLogsServiceResponse> { public: - GrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, - envoy::config::core::v3::ApiVersion transport_api_version); + GrpcAccessLoggerImpl( + const Grpc::RawAsyncClientSharedPtr& client, + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, + std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, + envoy::config::core::v3::ApiVersion transport_api_version); private: void initMessageRoot(const std::string& log_name, const LocalInfo::LocalInfo& local_info); diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index 57f5f2278624a..574cdac9d39ff 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -50,14 +50,18 @@ class MockGrpcAccessLoggerImpl : public Common::GrpcAccessLogger { public: - MockGrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, - Stats::Scope& scope, std::string access_log_prefix, - const Protobuf::MethodDescriptor& service_method, - envoy::config::core::v3::ApiVersion transport_api_version) + MockGrpcAccessLoggerImpl( + const Grpc::RawAsyncClientSharedPtr& client, + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, + std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, + Event::Dispatcher& dispatcher, Stats::Scope& scope, std::string access_log_prefix, + const Protobuf::MethodDescriptor& service_method, + envoy::config::core::v3::ApiVersion transport_api_version) : GrpcAccessLogger(std::move(client), buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, scope, access_log_prefix, service_method, + config.has_grpc_stream_retry_policy() + ? absl::make_optional(config.grpc_stream_retry_policy()) + : absl::nullopt, transport_api_version) {} int numInits() const { return num_inits_; } @@ -120,9 +124,9 @@ class GrpcAccessLogTest : public testing::Test { timer_ = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*timer_, enableTimer(buffer_flush_interval_msec, _)); logger_ = std::make_unique( - Grpc::RawAsyncClientPtr{async_client_}, buffer_flush_interval_msec, buffer_size_bytes, - dispatcher_, stats_store_, "mock_access_log_prefix.", mockMethodDescriptor(), - TRANSPORT_API_VERSION); + Grpc::RawAsyncClientPtr{async_client_}, config_, buffer_flush_interval_msec, + buffer_size_bytes, dispatcher_, stats_store_, "mock_access_log_prefix.", + mockMethodDescriptor(), TRANSPORT_API_VERSION); } void expectStreamStart(MockAccessLogStream& stream, AccessLogCallbacks** callbacks_to_set) { @@ -153,6 +157,7 @@ class GrpcAccessLogTest : public testing::Test { Event::MockDispatcher dispatcher_; Grpc::MockAsyncClient* async_client_{new Grpc::MockAsyncClient}; std::unique_ptr logger_; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config_; }; // Test basic stream logging flow. @@ -261,6 +266,28 @@ TEST_F(GrpcAccessLogTest, StreamFailure) { EXPECT_EQ(1, logger_->numInits()); } +TEST_F(GrpcAccessLogTest, StreamFailureAndRetry) { + config_.mutable_grpc_stream_retry_policy()->mutable_num_retries()->set_value(2); + initLogger(FlushInterval, 1); + + MockAccessLogStream stream; + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)) + .WillOnce(Invoke( + [&stream](absl::string_view, absl::string_view, Grpc::RawAsyncStreamCallbacks& callbacks, + const Http::AsyncClient::StreamOptions& options) -> Grpc::RawAsyncStream* { + EXPECT_TRUE(options.retry_policy.has_value()); + EXPECT_TRUE(options.retry_policy.value().has_num_retries()); + EXPECT_EQ(PROTOBUF_GET_WRAPPED_REQUIRED(options.retry_policy.value(), num_retries), 2); + callbacks.onRemoteClose(Grpc::Status::Unavailable, "unavailable"); + return &stream; + })); + + EXPECT_CALL(stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + EXPECT_CALL(stream, sendMessageRaw_(_, _)); + logger_->log(mockHttpEntry()); +} + // Test that log entries are batched. TEST_F(GrpcAccessLogTest, Batching) { // The approximate log size for buffering is calculated based on each entry's byte size. @@ -329,8 +356,8 @@ class MockGrpcAccessLoggerCache std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope) override { return std::make_shared( - std::move(client), buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, scope, - "mock_access_log_prefix.", mockMethodDescriptor(), config.transport_api_version()); + std::move(client), config, buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, + scope, "mock_access_log_prefix.", mockMethodDescriptor(), config.transport_api_version()); } }; From 56b6e9a48d2957e0656f54356c56c7248284ec48 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Sat, 24 Jul 2021 12:27:36 +0900 Subject: [PATCH 02/12] fix Signed-off-by: Shikugawa --- .../access_loggers/common/grpc_access_logger_test.cc | 12 +++--------- .../access_loggers/grpc/grpc_access_log_impl_test.cc | 4 +++- .../open_telemetry/grpc_access_log_impl_test.cc | 4 +++- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index 574cdac9d39ff..33bf0d2e549e1 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -270,21 +270,15 @@ TEST_F(GrpcAccessLogTest, StreamFailureAndRetry) { config_.mutable_grpc_stream_retry_policy()->mutable_num_retries()->set_value(2); initLogger(FlushInterval, 1); - MockAccessLogStream stream; - EXPECT_CALL(*async_client_, startRaw(_, _, _, _)) - .WillOnce(Invoke( - [&stream](absl::string_view, absl::string_view, Grpc::RawAsyncStreamCallbacks& callbacks, + .WillOnce( + Invoke([](absl::string_view, absl::string_view, Grpc::RawAsyncStreamCallbacks&, const Http::AsyncClient::StreamOptions& options) -> Grpc::RawAsyncStream* { EXPECT_TRUE(options.retry_policy.has_value()); EXPECT_TRUE(options.retry_policy.value().has_num_retries()); EXPECT_EQ(PROTOBUF_GET_WRAPPED_REQUIRED(options.retry_policy.value(), num_retries), 2); - callbacks.onRemoteClose(Grpc::Status::Unavailable, "unavailable"); - return &stream; + return nullptr; })); - - EXPECT_CALL(stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); - EXPECT_CALL(stream, sendMessageRaw_(_, _)); logger_->log(mockHttpEntry()); } diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index eec260b9c347b..a9b545ee25e65 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -70,8 +70,9 @@ class GrpcAccessLoggerImplTest : public testing::Test { : async_client_(new Grpc::MockAsyncClient), timer_(new Event::MockTimer(&dispatcher_)), grpc_access_logger_impl_test_helper_(local_info_, async_client_) { EXPECT_CALL(*timer_, enableTimer(_, _)); + *config_.mutable_log_name() = "test_log_name"; logger_ = std::make_unique( - Grpc::RawAsyncClientPtr{async_client_}, "test_log_name", FlushInterval, BUFFER_SIZE_BYTES, + Grpc::RawAsyncClientPtr{async_client_}, config_, FlushInterval, BUFFER_SIZE_BYTES, dispatcher_, local_info_, stats_store_, envoy::config::core::v3::ApiVersion::AUTO); } @@ -82,6 +83,7 @@ class GrpcAccessLoggerImplTest : public testing::Test { Event::MockTimer* timer_; std::unique_ptr logger_; GrpcAccessLoggerImplTestHelper grpc_access_logger_impl_test_helper_; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config_; }; TEST_F(GrpcAccessLoggerImplTest, LogHttp) { diff --git a/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc index ecd8d6b96a83e..d4bb17de8c077 100644 --- a/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc @@ -79,8 +79,9 @@ class GrpcAccessLoggerImplTest : public testing::Test { : async_client_(new Grpc::MockAsyncClient), timer_(new Event::MockTimer(&dispatcher_)), grpc_access_logger_impl_test_helper_(local_info_, async_client_) { EXPECT_CALL(*timer_, enableTimer(_, _)); + *config_.mutable_log_name() = "test_log_name"; logger_ = std::make_unique( - Grpc::RawAsyncClientPtr{async_client_}, "test_log_name", FlushInterval, BUFFER_SIZE_BYTES, + Grpc::RawAsyncClientPtr{async_client_}, config_, FlushInterval, BUFFER_SIZE_BYTES, dispatcher_, local_info_, stats_store_, envoy::config::core::v3::ApiVersion::V3); } @@ -91,6 +92,7 @@ class GrpcAccessLoggerImplTest : public testing::Test { Event::MockTimer* timer_; std::unique_ptr logger_; GrpcAccessLoggerImplTestHelper grpc_access_logger_impl_test_helper_; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config_; }; TEST_F(GrpcAccessLoggerImplTest, LogHttp) { From 2ad2f790da709c233a833d3dc93d2881bb1594b7 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 28 Jul 2021 07:01:29 +0000 Subject: [PATCH 03/12] fix Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 4 +++- api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto | 4 +++- .../envoy/extensions/access_loggers/grpc/v3/als.proto | 4 +++- .../envoy/extensions/access_loggers/grpc/v4alpha/als.proto | 4 +++- source/extensions/access_loggers/common/grpc_access_logger.h | 4 ++-- .../extensions/access_loggers/grpc/grpc_access_log_impl.cc | 5 +---- .../access_loggers/open_telemetry/grpc_access_log_impl.cc | 5 +---- .../access_loggers/common/grpc_access_logger_test.cc | 5 +---- 8 files changed, 17 insertions(+), 18 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index c62eb81513ee4..e9a4dbae979ef 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -91,6 +91,8 @@ message CommonGrpcAccessLogConfig { // Sets the retry policy when the establishment of a gRPC stream fails. // If the stream succeeds once in establishing If the stream succeeds // at least once in establishing itself, no retry will be performed - // no matter what gRPC status is received. + // no matter what gRPC status is received. Note that only + // :ref:`num_retries retry_policy) + const envoy::config::core::v3::RetryPolicy& retry_policy) : GrpcAccessLogger(client, buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, scope, access_log_prefix, service_method, retry_policy, absl::nullopt) {} GrpcAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, @@ -200,7 +200,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger retry_policy, + const envoy::config::core::v3::RetryPolicy& retry_policy, envoy::config::core::v3::ApiVersion transport_api_version) : client_(client, service_method, retry_policy, transport_api_version), buffer_flush_interval_msec_(buffer_flush_interval_msec), diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 17b3f28cc06d9..6e79ed0f956b7 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -27,10 +27,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( Grpc::VersionedMethods("envoy.service.accesslog.v3.AccessLogService.StreamAccessLogs", "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs") .getMethodDescriptorForVersion(transport_api_version), - config.has_grpc_stream_retry_policy() - ? absl::make_optional(config.grpc_stream_retry_policy()) - : absl::nullopt, - transport_api_version), + config.grpc_stream_retry_policy(), transport_api_version), log_name_(config.log_name()), local_info_(local_info) {} void GrpcAccessLoggerImpl::addEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc index 1c06854629ae4..b47838b24c4cb 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc @@ -32,10 +32,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( Grpc::VersionedMethods("opentelemetry.proto.collector.logs.v1.LogsService.Export", "opentelemetry.proto.collector.logs.v1.LogsService.Export") .getMethodDescriptorForVersion(transport_api_version), - config.has_grpc_stream_retry_policy() - ? absl::make_optional(config.grpc_stream_retry_policy()) - : absl::nullopt, - transport_api_version) { + config.grpc_stream_retry_policy(), transport_api_version) { initMessageRoot(config.log_name(), local_info); } diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index 33bf0d2e549e1..e5727d3206aca 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -59,10 +59,7 @@ class MockGrpcAccessLoggerImpl envoy::config::core::v3::ApiVersion transport_api_version) : GrpcAccessLogger(std::move(client), buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, scope, access_log_prefix, service_method, - config.has_grpc_stream_retry_policy() - ? absl::make_optional(config.grpc_stream_retry_policy()) - : absl::nullopt, - transport_api_version) {} + config.grpc_stream_retry_policy(), transport_api_version) {} int numInits() const { return num_inits_; } From c1288fd6048253a251ba30f14f3fdcce28de5f08 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 28 Jul 2021 07:10:41 +0000 Subject: [PATCH 04/12] add docs Signed-off-by: Shikugawa --- docs/root/version_history/current.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index ef1983f2996d7..cf0858432ed89 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -39,6 +39,7 @@ Removed Config or Runtime New Features ------------ +* access_logs: Added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. * http: added :ref:`string_match ` in the header matcher. * http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. From 16826becef6771f869eb00631bec845b84747413 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 28 Jul 2021 13:20:29 +0000 Subject: [PATCH 05/12] docs Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 2 +- api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto | 2 +- docs/root/version_history/current.rst | 2 +- .../envoy/extensions/access_loggers/grpc/v3/als.proto | 2 +- .../envoy/extensions/access_loggers/grpc/v4alpha/als.proto | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index e9a4dbae979ef..d8f2175dc8898 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -92,7 +92,7 @@ message CommonGrpcAccessLogConfig { // If the stream succeeds once in establishing If the stream succeeds // at least once in establishing itself, no retry will be performed // no matter what gRPC status is received. Note that only - // :ref:`num_retries ` // will be used in this configuration. config.core.v3.RetryPolicy grpc_stream_retry_policy = 7; } diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 0f0021286fdd3..3499689d4450d 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -92,7 +92,7 @@ message CommonGrpcAccessLogConfig { // If the stream succeeds once in establishing If the stream succeeds // at least once in establishing itself, no retry will be performed // no matter what gRPC status is received. Note that only - // :ref:`num_retries ` // will be used in this configuration. config.core.v4alpha.RetryPolicy grpc_stream_retry_policy = 7; } diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index df4166df5c758..6a21ff9004617 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -52,7 +52,7 @@ Removed Config or Runtime New Features ------------ -* access_logs: Added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. +* access log: Added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. * bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. * http: added :ref:`string_match ` in the header matcher. * http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index e9a4dbae979ef..d8f2175dc8898 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -92,7 +92,7 @@ message CommonGrpcAccessLogConfig { // If the stream succeeds once in establishing If the stream succeeds // at least once in establishing itself, no retry will be performed // no matter what gRPC status is received. Note that only - // :ref:`num_retries ` // will be used in this configuration. config.core.v3.RetryPolicy grpc_stream_retry_policy = 7; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 0f0021286fdd3..3499689d4450d 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -92,7 +92,7 @@ message CommonGrpcAccessLogConfig { // If the stream succeeds once in establishing If the stream succeeds // at least once in establishing itself, no retry will be performed // no matter what gRPC status is received. Note that only - // :ref:`num_retries ` // will be used in this configuration. config.core.v4alpha.RetryPolicy grpc_stream_retry_policy = 7; } From e78a14149a12c535c0e5dfd72353fce689eac0bf Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 28 Jul 2021 14:17:10 +0000 Subject: [PATCH 06/12] fix Signed-off-by: Shikugawa --- docs/root/version_history/current.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 6a21ff9004617..b4415aaa27935 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -52,7 +52,7 @@ Removed Config or Runtime New Features ------------ -* access log: Added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. +* access log: added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. * bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. * http: added :ref:`string_match ` in the header matcher. * http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. From dd752edf2fa06358c2ec922a8e7dee73fe59a380 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 4 Aug 2021 05:55:53 +0000 Subject: [PATCH 07/12] fix Signed-off-by: Shikugawa --- .../access_loggers/common/grpc_access_logger.h | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 3dc01d2c01fa7..7b557b5a4b168 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -78,11 +78,11 @@ template class GrpcAccessLogClient { public: GrpcAccessLogClient(const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& service_method, - const absl::optional retry_policy) + const envoy::config::core::v3::RetryPolicy& retry_policy) : GrpcAccessLogClient(client, service_method, retry_policy, absl::nullopt) {} GrpcAccessLogClient(const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& service_method, - const absl::optional retry_policy, + const envoy::config::core::v3::RetryPolicy& retry_policy, envoy::config::core::v3::ApiVersion transport_api_version) : client_(client), service_method_(service_method), transport_api_version_(transport_api_version), grpc_stream_retry_policy_(retry_policy) {} @@ -146,6 +146,18 @@ template class GrpcAccessLogClient { envoy::config::route::v3::RetryPolicy retry_policy; retry_policy.mutable_num_retries()->set_value( PROTOBUF_GET_WRAPPED_OR_DEFAULT(*grpc_stream_retry_policy_, num_retries, 1)); + + if (grpc_stream_retry_policy_->has_retry_back_off()) { + const auto& base_retry_backoff = grpc_stream_retry_policy_->retry_back_off(); + const auto base_interval_ms = PROTOBUF_GET_MS_REQUIRED(base_retry_backoff, base_interval); + + auto* mutable_retry_back_off = retry_policy.mutable_retry_back_off(); + mutable_retry_back_off->mutable_base_interval()->CopyFrom( + Protobuf::util::TimeUtil::MillisecondsToDuration(base_interval_ms)); + mutable_retry_back_off->mutable_max_interval()->CopyFrom( + Protobuf::util::TimeUtil::MillisecondsToDuration( + PROTOBUF_GET_MS_OR_DEFAULT(base_retry_backoff, max_interval, 10 * base_interval_ms))); + } retry_policy.set_retry_on("connect-failure"); opt.setBufferBodyForRetry(true); From 995dd706deee9bab5371c8c2f01413bb80659383 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 4 Aug 2021 13:04:03 +0000 Subject: [PATCH 08/12] coverage Signed-off-by: Shikugawa --- .../access_loggers/common/grpc_access_logger_test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index e5727d3206aca..5e453cb79f697 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -265,6 +265,10 @@ TEST_F(GrpcAccessLogTest, StreamFailure) { TEST_F(GrpcAccessLogTest, StreamFailureAndRetry) { config_.mutable_grpc_stream_retry_policy()->mutable_num_retries()->set_value(2); + config_.mutable_grpc_stream_retry_policy() + ->mutable_retry_back_off() + ->mutable_base_interval() + ->set_seconds(1); initLogger(FlushInterval, 1); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)) From e380b775b098f01ab138d80089351c931a6e6861 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 16 Aug 2021 04:23:07 +0000 Subject: [PATCH 09/12] add TODO Signed-off-by: Shikugawa --- source/extensions/access_loggers/common/grpc_access_logger.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 7b557b5a4b168..7b124ab859a7f 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -143,6 +143,8 @@ template class GrpcAccessLogClient { return opt; } + // TODO(shikugawa): refactor conversion from route retry policy to core retry policy. + // This conversion is duplicated with jwks_fetcher.cc. envoy::config::route::v3::RetryPolicy retry_policy; retry_policy.mutable_num_retries()->set_value( PROTOBUF_GET_WRAPPED_OR_DEFAULT(*grpc_stream_retry_policy_, num_retries, 1)); From 7c6059aec8925512f2430dd8390b04dcbad4fedc Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 30 Sep 2021 04:53:25 +0000 Subject: [PATCH 10/12] fix Signed-off-by: Shikugawa --- .../access_loggers/grpc/v4alpha/als.proto | 98 ------------------- .../access_loggers/grpc/v4alpha/als.proto | 98 ------------------- 2 files changed, 196 deletions(-) delete mode 100644 api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto delete mode 100644 generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto deleted file mode 100644 index 3499689d4450d..0000000000000 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ /dev/null @@ -1,98 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.access_loggers.grpc.v4alpha; - -import "envoy/config/core/v4alpha/base.proto"; -import "envoy/config/core/v4alpha/config_source.proto"; -import "envoy/config/core/v4alpha/grpc_service.proto"; - -import "google/protobuf/duration.proto"; -import "google/protobuf/wrappers.proto"; - -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.access_loggers.grpc.v4alpha"; -option java_outer_classname = "AlsProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; - -// [#protodoc-title: gRPC Access Log Service (ALS)] - -// Configuration for the built-in *envoy.access_loggers.http_grpc* -// :ref:`AccessLog `. This configuration will -// populate :ref:`StreamAccessLogsMessage.http_logs -// `. -// [#extension: envoy.access_loggers.http_grpc] -message HttpGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig"; - - CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}]; - - // Additional request headers to log in :ref:`HTTPRequestProperties.request_headers - // `. - repeated string additional_request_headers_to_log = 2; - - // Additional response headers to log in :ref:`HTTPResponseProperties.response_headers - // `. - repeated string additional_response_headers_to_log = 3; - - // Additional response trailers to log in :ref:`HTTPResponseProperties.response_trailers - // `. - repeated string additional_response_trailers_to_log = 4; -} - -// Configuration for the built-in *envoy.access_loggers.tcp_grpc* type. This configuration will -// populate *StreamAccessLogsMessage.tcp_logs*. -// [#extension: envoy.access_loggers.tcp_grpc] -message TcpGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.access_loggers.grpc.v3.TcpGrpcAccessLogConfig"; - - CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}]; -} - -// Common configuration for gRPC access logs. -// [#next-free-field: 8] -message CommonGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; - - // The friendly name of the access log to be returned in :ref:`StreamAccessLogsMessage.Identifier - // `. This allows the - // access log server to differentiate between different access logs coming from the same Envoy. - string log_name = 1 [(validate.rules).string = {min_len: 1}]; - - // The gRPC service for the access log service. - config.core.v4alpha.GrpcService grpc_service = 2 [(validate.rules).message = {required: true}]; - - // API version for access logs service transport protocol. This describes the access logs service - // gRPC endpoint and version of messages used on the wire. - config.core.v4alpha.ApiVersion transport_api_version = 6 - [(validate.rules).enum = {defined_only: true}]; - - // Interval for flushing access logs to the gRPC stream. Logger will flush requests every time - // this interval is elapsed, or when batch size limit is hit, whichever comes first. Defaults to - // 1 second. - google.protobuf.Duration buffer_flush_interval = 3 [(validate.rules).duration = {gt {}}]; - - // Soft size limit in bytes for access log entries buffer. Logger will buffer requests until - // this limit it hit, or every time flush interval is elapsed, whichever comes first. Setting it - // to zero effectively disables the batching. Defaults to 16384. - google.protobuf.UInt32Value buffer_size_bytes = 4; - - // Additional filter state objects to log in :ref:`filter_state_objects - // `. - // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. - repeated string filter_state_objects_to_log = 5; - - // Sets the retry policy when the establishment of a gRPC stream fails. - // If the stream succeeds once in establishing If the stream succeeds - // at least once in establishing itself, no retry will be performed - // no matter what gRPC status is received. Note that only - // :ref:`num_retries ` - // will be used in this configuration. - config.core.v4alpha.RetryPolicy grpc_stream_retry_policy = 7; -} diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto deleted file mode 100644 index 3499689d4450d..0000000000000 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ /dev/null @@ -1,98 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.access_loggers.grpc.v4alpha; - -import "envoy/config/core/v4alpha/base.proto"; -import "envoy/config/core/v4alpha/config_source.proto"; -import "envoy/config/core/v4alpha/grpc_service.proto"; - -import "google/protobuf/duration.proto"; -import "google/protobuf/wrappers.proto"; - -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.access_loggers.grpc.v4alpha"; -option java_outer_classname = "AlsProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; - -// [#protodoc-title: gRPC Access Log Service (ALS)] - -// Configuration for the built-in *envoy.access_loggers.http_grpc* -// :ref:`AccessLog `. This configuration will -// populate :ref:`StreamAccessLogsMessage.http_logs -// `. -// [#extension: envoy.access_loggers.http_grpc] -message HttpGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig"; - - CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}]; - - // Additional request headers to log in :ref:`HTTPRequestProperties.request_headers - // `. - repeated string additional_request_headers_to_log = 2; - - // Additional response headers to log in :ref:`HTTPResponseProperties.response_headers - // `. - repeated string additional_response_headers_to_log = 3; - - // Additional response trailers to log in :ref:`HTTPResponseProperties.response_trailers - // `. - repeated string additional_response_trailers_to_log = 4; -} - -// Configuration for the built-in *envoy.access_loggers.tcp_grpc* type. This configuration will -// populate *StreamAccessLogsMessage.tcp_logs*. -// [#extension: envoy.access_loggers.tcp_grpc] -message TcpGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.access_loggers.grpc.v3.TcpGrpcAccessLogConfig"; - - CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}]; -} - -// Common configuration for gRPC access logs. -// [#next-free-field: 8] -message CommonGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; - - // The friendly name of the access log to be returned in :ref:`StreamAccessLogsMessage.Identifier - // `. This allows the - // access log server to differentiate between different access logs coming from the same Envoy. - string log_name = 1 [(validate.rules).string = {min_len: 1}]; - - // The gRPC service for the access log service. - config.core.v4alpha.GrpcService grpc_service = 2 [(validate.rules).message = {required: true}]; - - // API version for access logs service transport protocol. This describes the access logs service - // gRPC endpoint and version of messages used on the wire. - config.core.v4alpha.ApiVersion transport_api_version = 6 - [(validate.rules).enum = {defined_only: true}]; - - // Interval for flushing access logs to the gRPC stream. Logger will flush requests every time - // this interval is elapsed, or when batch size limit is hit, whichever comes first. Defaults to - // 1 second. - google.protobuf.Duration buffer_flush_interval = 3 [(validate.rules).duration = {gt {}}]; - - // Soft size limit in bytes for access log entries buffer. Logger will buffer requests until - // this limit it hit, or every time flush interval is elapsed, whichever comes first. Setting it - // to zero effectively disables the batching. Defaults to 16384. - google.protobuf.UInt32Value buffer_size_bytes = 4; - - // Additional filter state objects to log in :ref:`filter_state_objects - // `. - // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. - repeated string filter_state_objects_to_log = 5; - - // Sets the retry policy when the establishment of a gRPC stream fails. - // If the stream succeeds once in establishing If the stream succeeds - // at least once in establishing itself, no retry will be performed - // no matter what gRPC status is received. Note that only - // :ref:`num_retries ` - // will be used in this configuration. - config.core.v4alpha.RetryPolicy grpc_stream_retry_policy = 7; -} From 19136f0087941c12af2a25c9d7ae0f13dc65d338 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Fri, 8 Oct 2021 11:15:32 +0000 Subject: [PATCH 11/12] fix Signed-off-by: Shikugawa --- docs/root/version_history/current.rst | 34 --------------------------- 1 file changed, 34 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index e1b1b9fb6ec89..9733b6d71dd74 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -25,40 +25,6 @@ Removed Config or Runtime New Features ------------ * access log: added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. -* access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). -* bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. -* contrib: added new :ref:`contrib images ` which contain contrib extensions. -* dns: added :ref:`V4_PREFERRED ` option to return V6 addresses only if V4 addresses are not available. -* ext_authz: added :ref:`dynamic_metadata_from_headers ` to support emitting dynamic metadata from headers returned by an external authorization service via HTTP. -* grpc reverse bridge: added a new :ref:`option ` to support streaming response bodies when withholding gRPC frames from the upstream. -* grpc_json_transcoder: added support to unescape '+' in query parameters to space with a new config field :ref:`query_param_unescape_plus `. -* http: added cluster_header in :ref:`weighted_clusters ` to allow routing to the weighted cluster specified in the request_header. -* http: added :ref:`alternate_protocols_cache_options ` for enabling HTTP/3 connections to servers which advertise HTTP/3 support via `HTTP Alternative Services `_. -* http: added :ref:`string_match ` in the header matcher. -* http: added :ref:`x-envoy-upstream-stream-duration-ms ` that allows configuring the max stream duration via a request header. -* http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. -* http: sanitizing the referer header as documented :ref:`here `. This feature can be temporarily turned off by setting runtime guard ``envoy.reloadable_features.sanitize_http_header_referer`` to false. -* http: validating outgoing HTTP/2 CONNECT requests to ensure that if ``:path`` is set that ``:protocol`` is present. This behavior can be temporarily turned off by setting runtime guard ``envoy.reloadable_features.validate_connect`` to false. -* jwt_authn: added support for :ref:`Jwt Cache ` and its size can be specified by :ref:`jwt_cache_size `. -* jwt_authn: added support for extracting JWTs from request cookies using :ref:`from_cookies `. -* jwt_authn: added support for setting the extracted headers from a successfully verified JWT using :ref:`header_in_metadata ` to dynamic metadata. -* listener: new listener metric ``downstream_cx_transport_socket_connect_timeout`` to track transport socket timeouts. -* lua: added ``header:getAtIndex()`` and ``header:getNumValues()`` methods to :ref:`header object ` for retrieving the value of a header at certain index and get the total number of values for a given header. -* matcher: added :ref:`invert ` for inverting the match result in the metadata matcher. -* overload: add a new overload action that resets streams using a lot of memory. To enable the tracking of allocated bytes in buffers that a stream is using we need to configure the minimum threshold for tracking via:ref:`buffer_factory_config `. We have an overload action ``Envoy::Server::OverloadActionNameValues::ResetStreams`` that takes advantage of the tracking to reset the most expensive stream first. -* rbac: added :ref:`destination_port_range ` for matching range of destination ports. -* rbac: added :ref:`matcher` along with extension category ``extension_category_envoy.rbac.matchers`` for custom RBAC permission matchers. Added reference implementation for matchers :ref:`envoy.rbac.matchers.upstream_ip_port `. -* route config: added :ref:`dynamic_metadata ` for routing based on dynamic metadata. -* router: added retry options predicate extensions configured via - :ref:` `. These - extensions allow modification of requests between retries at the router level. There are not - currently any built-in extensions that implement this extension point. -* router: added :ref:`per_try_idle_timeout ` timeout configuration. -* router: added an optional :ref:`override_auto_sni_header ` to support setting SNI value from an arbitrary header other than host/authority. -* sxg_filter: added filter to transform response to SXG package to :ref:`contrib images `. This can be enabled by setting :ref:`SXG ` configuration. -* thrift_proxy: added support for :ref:`mirroring requests `. -* udp: allows updating filter chain in-place through LDS, which is supported by Quic listener. Such listener config will be rejected in other connection-less UDP listener implementations. It can be reverted by ``envoy.reloadable_features.udp_listener_updates_filter_chain_in_place``. -* udp: disallow L4 filter chain in config which configures connection-less UDP listener. It can be reverted by ``envoy.reloadable_features.udp_listener_updates_filter_chain_in_place``. * http: added support for :ref:`retriable health check status codes `. Deprecated From 2150dfb9586cafd930cb8d291d138331e2afaae2 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Tue, 2 Nov 2021 17:38:29 +0900 Subject: [PATCH 12/12] fix Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index d8f2175dc8898..a671873580f31 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -93,6 +93,7 @@ message CommonGrpcAccessLogConfig { // at least once in establishing itself, no retry will be performed // no matter what gRPC status is received. Note that only // :ref:`num_retries ` - // will be used in this configuration. + // will be used in this configuration. This feature is used only when you are using + // :ref:`Envoy gRPC client `. config.core.v3.RetryPolicy grpc_stream_retry_policy = 7; }