From 7e2fec9950c775a758269aca598513bcbe983c31 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 25 Jul 2017 14:02:12 -0400 Subject: [PATCH 01/19] Watermark callbacks for H2 flow control --- include/envoy/http/codec.h | 28 +++++++++++++++++++ include/envoy/http/filter.h | 20 +++++++++++++ source/common/http/async_client_impl.h | 2 ++ source/common/http/codec_client.h | 2 ++ source/common/http/conn_manager_impl.h | 6 ++++ source/common/http/http1/codec_impl.h | 3 ++ source/common/http/http1/conn_pool.h | 2 ++ source/common/http/http2/codec_impl.h | 3 ++ source/common/router/router.h | 2 ++ source/common/upstream/health_checker_impl.cc | 2 +- source/common/upstream/health_checker_impl.h | 21 ++++++++++---- test/integration/fake_upstream.h | 2 ++ test/integration/integration.h | 2 ++ test/integration/utility.h | 2 ++ test/mocks/http/mocks.h | 10 +++++++ 15 files changed, 101 insertions(+), 6 deletions(-) diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 4c0b53726ac61..5c9d840fe2834 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -106,6 +106,16 @@ class StreamCallbacks { * @param reason supplies the reset reason. */ virtual void onResetStream(StreamResetReason reason) PURE; + + /** + * Fires when a stream, or the connection the stream is sending to, goes over its high watermark. + */ + virtual void onAboveWriteBufferHighWatermark() PURE; + /** + * Fires when a stream, or the connection the stream is sending to, goes from over its high + * watermark to under its low watermark. + */ + virtual void onBelowWriteBufferLowWatermark() PURE; }; /** @@ -132,6 +142,14 @@ class Stream { * @param reason supplies the reset reason. */ virtual void resetStream(StreamResetReason reason) PURE; + + /** + * Enable/disable further data from this stream. + * Cessation of data may not be immediate. For example, for HTTP/2 this may stop further flow + * control window updates which will result in the peer eventually stopping sending data. + * @param disable informs if reads should be disabled (true) or re-enabled (false). + */ + virtual void readDisable(bool disable) PURE; }; /** @@ -260,6 +278,16 @@ class ClientConnection : public virtual Connection { * @return StreamEncoder& supplies the encoder to write the request into. */ virtual StreamEncoder& newStream(StreamDecoder& response_decoder) PURE; + + /** + * Called when the connection goes over its high watermark. + */ + virtual void onBelowWriteBufferLowWatermark() PURE; + + /** + * Called when the connection goes from over its high watermark to under its low watermark. + */ + virtual void onAboveWriteBufferHighWatermark() PURE; }; typedef std::unique_ptr ClientConnectionPtr; diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index b4f9722a1677a..d237cf066131d 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -198,6 +198,16 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { * @param trailers supplies the trailers to encode. */ virtual void encodeTrailers(HeaderMapPtr&& trailers) PURE; + + /** + * Called when a decoder filter goes over its high watermark. + */ + virtual void onDecoderFilterAboveWriteBufferHighWatermark() PURE; + + /** + * Called when a decoder filter goes from over its high watermark to under its low watermark. + */ + virtual void onDecoderFilterBelowWriteBufferLowWatermark() PURE; }; /** @@ -297,6 +307,16 @@ class StreamEncoderFilterCallbacks : public virtual StreamFilterCallbacks { * It is an error to call this method in any other case. */ virtual void addEncodedData(Buffer::Instance& data) PURE; + + /** + * Called when an encoder filter goes over its high watermark. + */ + virtual void onEncoderFilterAboveWriteBufferHighWatermark() PURE; + + /** + * Called when a encoder filter goes from over its high watermark to under its low watermark. + */ + virtual void onEncoderFilterBelowWriteBufferLowWatermark() PURE; }; /** diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 107f70ac60830..27048b8f9b05e 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -197,6 +197,8 @@ class AsyncStreamImpl : public AsyncClient::Stream, void encodeHeaders(HeaderMapPtr&& headers, bool end_stream) override; void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(HeaderMapPtr&& trailers) override; + void onDecoderFilterAboveWriteBufferHighWatermark() override {} + void onDecoderFilterBelowWriteBufferLowWatermark() override {} AsyncClient::StreamCallbacks& stream_callbacks_; const uint64_t stream_id_; diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index 91fcc50bbb750..250cbe44e1395 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -157,6 +157,8 @@ class CodecClient : Logger::Loggable, // StreamCallbacks void onResetStream(StreamResetReason reason) override { parent_.onReset(*this, reason); } + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} // StreamDecoderWrapper void onPreDecodeComplete() override { parent_.responseDecodeComplete(*this); } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index b8d04b1084983..cf5d50f276b2c 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -354,6 +354,8 @@ class ConnectionManagerImpl : Logger::Loggable, void encodeHeaders(HeaderMapPtr&& headers, bool end_stream) override; void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(HeaderMapPtr&& trailers) override; + void onDecoderFilterAboveWriteBufferHighWatermark() override {} + void onDecoderFilterBelowWriteBufferLowWatermark() override {} StreamDecoderFilterSharedPtr handle_; }; @@ -384,6 +386,8 @@ class ConnectionManagerImpl : Logger::Loggable, // Http::StreamEncoderFilterCallbacks void addEncodedData(Buffer::Instance& data) override; + void onEncoderFilterAboveWriteBufferHighWatermark() override {} + void onEncoderFilterBelowWriteBufferLowWatermark() override {} void continueEncoding() override; const Buffer::Instance* encodingBuffer() override { return parent_.buffered_response_data_.get(); @@ -428,6 +432,8 @@ class ConnectionManagerImpl : Logger::Loggable, // Http::StreamCallbacks void onResetStream(StreamResetReason reason) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} // Http::StreamDecoder void decodeHeaders(HeaderMapPtr&& headers, bool end_stream) override; diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 56acfaa6b3c9f..c545edab3e01e 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -42,6 +42,7 @@ class StreamEncoderImpl : public StreamEncoder, void addCallbacks(StreamCallbacks& callbacks) override { addCallbacks_(callbacks); } void removeCallbacks(StreamCallbacks& callbacks) override { removeCallbacks_(callbacks); } void resetStream(StreamResetReason reason) override; + void readDisable(bool /*disable*/) override {} protected: StreamEncoderImpl(ConnectionImpl& connection) : connection_(connection) {} @@ -282,6 +283,8 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl { // Http::ClientConnection StreamEncoder& newStream(StreamDecoder& response_decoder) override; + void onBelowWriteBufferLowWatermark() override {} + void onAboveWriteBufferHighWatermark() override {} private: struct PendingResponse { diff --git a/source/common/http/http1/conn_pool.h b/source/common/http/http1/conn_pool.h index 5fdabe5fbcfff..23e59f51ef002 100644 --- a/source/common/http/http1/conn_pool.h +++ b/source/common/http/http1/conn_pool.h @@ -57,6 +57,8 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: // Http::StreamCallbacks void onResetStream(StreamResetReason) override { parent_.parent_.onDownstreamReset(parent_); } + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} ActiveClient& parent_; bool encode_complete_{}; diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index d6c5a83d0004c..900dcad53569a 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -143,6 +143,7 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable, // Http::StreamCallbacks void onResetStream(Http::StreamResetReason reason) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} // Http::ConnectionPool::Callbacks void onPoolFailure(Http::ConnectionPool::PoolFailureReason reason, diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc index 68216d049e0a7..283ca5282b6d9 100644 --- a/source/common/upstream/health_checker_impl.cc +++ b/source/common/upstream/health_checker_impl.cc @@ -269,7 +269,7 @@ void HttpHealthCheckerImpl::HttpActiveHealthCheckSession::onInterval() { if (!client_) { Upstream::Host::CreateConnectionData conn = host_->createConnection(parent_.dispatcher_); client_.reset(parent_.createCodecClient(conn)); - client_->addConnectionCallbacks(*this); + client_->addConnectionCallbacks(connection_callback_impl_); expect_reset_ = false; } diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h index 5a8cc0f1f50fc..fcc44d52b9a45 100644 --- a/source/common/upstream/health_checker_impl.h +++ b/source/common/upstream/health_checker_impl.h @@ -150,8 +150,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { private: struct HttpActiveHealthCheckSession : public ActiveHealthCheckSession, public Http::StreamDecoder, - public Http::StreamCallbacks, - public Network::ConnectionCallbacks { + public Http::StreamCallbacks { HttpActiveHealthCheckSession(HttpHealthCheckerImpl& parent, HostSharedPtr host); ~HttpActiveHealthCheckSession(); @@ -173,12 +172,24 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { // Http::StreamCallbacks void onResetStream(Http::StreamResetReason reason) override; - - // Network::ConnectionCallbacks - void onEvent(uint32_t events) override; void onAboveWriteBufferHighWatermark() override {} void onBelowWriteBufferLowWatermark() override {} + void onEvent(uint32_t events); + + class ConnectionCallbackImpl : public Network::ConnectionCallbacks { + public: + ConnectionCallbackImpl(HttpActiveHealthCheckSession& parent) : parent_(parent) {} + // Network::ConnectionCallbacks + void onEvent(uint32_t events) override { parent_.onEvent(events); } + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} + + private: + HttpActiveHealthCheckSession& parent_; + }; + + ConnectionCallbackImpl connection_callback_impl_{*this}; HttpHealthCheckerImpl& parent_; Http::CodecClientPtr client_; Http::StreamEncoder* request_encoder_{}; diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 3e9c4abef5362..edcbf12045ed2 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -55,6 +55,8 @@ class FakeStream : public Http::StreamDecoder, public Http::StreamCallbacks { // Http::StreamCallbacks void onResetStream(Http::StreamResetReason reason) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} private: FakeHttpConnection& parent_; diff --git a/test/integration/integration.h b/test/integration/integration.h index fd4d7def9c8ef..26480b749cffe 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -45,6 +45,8 @@ class IntegrationStreamDecoder : public Http::StreamDecoder, public Http::Stream // Http::StreamCallbacks void onResetStream(Http::StreamResetReason reason) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} private: Event::Dispatcher& dispatcher_; diff --git a/test/integration/utility.h b/test/integration/utility.h index d379ded8f73bf..8f547f20de29d 100644 --- a/test/integration/utility.h +++ b/test/integration/utility.h @@ -34,6 +34,8 @@ class BufferingStreamDecoder : public Http::StreamDecoder, public Http::StreamCa // Http::StreamCallbacks void onResetStream(Http::StreamResetReason reason) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} private: void onComplete(); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 5b1786064adfe..771d4dc915f73 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -140,6 +140,8 @@ class MockStreamCallbacks : public StreamCallbacks { // Http::StreamCallbacks MOCK_METHOD1(onResetStream, void(StreamResetReason reason)); + MOCK_METHOD0(onAboveWriteBufferHighWatermark, void()); + MOCK_METHOD0(onBelowWriteBufferLowWatermark, void()); }; class MockStream : public Stream { @@ -151,6 +153,8 @@ class MockStream : public Stream { MOCK_METHOD1(addCallbacks, void(StreamCallbacks& callbacks)); MOCK_METHOD1(removeCallbacks, void(StreamCallbacks& callbacks)); MOCK_METHOD1(resetStream, void(StreamResetReason reason)); + MOCK_METHOD1(readDisable, void(bool disable)); + MOCK_METHOD2(setWriteBufferWatermarks, void(uint32_t, uint32_t)); std::list callbacks_{}; }; @@ -198,6 +202,8 @@ class MockClientConnection : public ClientConnection { // Http::ClientConnection MOCK_METHOD1(newStream, StreamEncoder&(StreamDecoder& response_decoder)); + MOCK_METHOD0(onAboveWriteBufferHighWatermark, void()); + MOCK_METHOD0(onBelowWriteBufferLowWatermark, void()); }; class MockFilterChainFactory : public FilterChainFactory { @@ -234,6 +240,8 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_METHOD0(requestInfo, Http::AccessLog::RequestInfo&()); MOCK_METHOD0(activeSpan, Tracing::Span&()); MOCK_METHOD0(downstreamAddress, const std::string&()); + MOCK_METHOD0(onDecoderFilterAboveWriteBufferHighWatermark, void()); + MOCK_METHOD0(onDecoderFilterBelowWriteBufferLowWatermark, void()); // Http::StreamDecoderFilterCallbacks void encodeHeaders(HeaderMapPtr&& headers, bool end_stream) override { @@ -268,6 +276,8 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, MOCK_METHOD0(requestInfo, Http::AccessLog::RequestInfo&()); MOCK_METHOD0(activeSpan, Tracing::Span&()); MOCK_METHOD0(downstreamAddress, const std::string&()); + MOCK_METHOD0(onEncoderFilterAboveWriteBufferHighWatermark, void()); + MOCK_METHOD0(onEncoderFilterBelowWriteBufferLowWatermark, void()); // Http::StreamEncoderFilterCallbacks MOCK_METHOD1(addEncodedData, void(Buffer::Instance& data)); From c3629cb91a3ab6a9300896bf038224c799cba4d4 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 25 Jul 2017 14:03:29 -0400 Subject: [PATCH 02/19] H2 flow control on the response path --- source/common/http/codec_client.h | 6 +- source/common/http/codec_helper.h | 12 ++ source/common/http/conn_manager_impl.cc | 12 ++ source/common/http/conn_manager_impl.h | 7 +- source/common/http/http2/BUILD | 1 + source/common/http/http2/codec_impl.cc | 59 +++++- source/common/http/http2/codec_impl.h | 59 ++++-- source/common/router/router.h | 10 +- test/common/http/codec_client_test.cc | 8 + test/common/http/conn_manager_impl_test.cc | 39 ++++ test/common/http/http2/codec_impl_test.cc | 174 +++++++++++++++--- test/config/integration/server_http2.json | 92 ++++++++- .../integration/server_http2_upstream.json | 91 +++++++++ test/integration/http2_integration_test.cc | 38 ++-- test/integration/http2_integration_test.h | 5 +- .../http2_upstream_integration_test.cc | 57 ++++-- .../http2_upstream_integration_test.h | 8 +- 17 files changed, 580 insertions(+), 98 deletions(-) diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index 250cbe44e1395..e1df89185adc9 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -182,8 +182,10 @@ class CodecClient : Logger::Loggable, // Network::ConnectionCallbacks void onEvent(uint32_t events) override; - void onAboveWriteBufferHighWatermark() override {} - void onBelowWriteBufferLowWatermark() override {} + // Pass watermark events from the connection on to the codec which will pass it to the underlying + // streams. + void onAboveWriteBufferHighWatermark() override { codec_->onAboveWriteBufferHighWatermark(); } + void onBelowWriteBufferLowWatermark() override { codec_->onBelowWriteBufferLowWatermark(); } std::list active_requests_; Http::ConnectionCallbacks* codec_callbacks_{}; diff --git a/source/common/http/codec_helper.h b/source/common/http/codec_helper.h index f73fc95ccfd9c..c6176eb25b8bb 100644 --- a/source/common/http/codec_helper.h +++ b/source/common/http/codec_helper.h @@ -7,6 +7,18 @@ namespace Http { class StreamCallbackHelper { public: + void runLowWatermarkCallbacks() { + for (StreamCallbacks* callbacks : callbacks_) { + callbacks->onBelowWriteBufferLowWatermark(); + } + } + + void runHighWatermarkCallbacks() { + for (StreamCallbacks* callbacks : callbacks_) { + callbacks->onAboveWriteBufferHighWatermark(); + } + } + void runResetCallbacks(StreamResetReason reason) { if (reset_callbacks_run_) { return; diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 728e06cd2d4af..6e8e938e88bb5 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -987,6 +987,18 @@ void ConnectionManagerImpl::ActiveStreamDecoderFilter::encodeTrailers(HeaderMapP parent_.encodeTrailers(nullptr, *parent_.response_trailers_); } +void ConnectionManagerImpl::ActiveStreamDecoderFilter:: + onDecoderFilterAboveWriteBufferHighWatermark() { + ENVOY_STREAM_LOG(debug, "Read-disabling downstream stream due to filter callbacks.", parent_); + parent_.response_encoder_->getStream().readDisable(true); +} + +void ConnectionManagerImpl::ActiveStreamDecoderFilter:: + onDecoderFilterBelowWriteBufferLowWatermark() { + ENVOY_STREAM_LOG(debug, "Read-enabling downstream stream due to filter callbacks.", parent_); + parent_.response_encoder_->getStream().readDisable(false); +} + void ConnectionManagerImpl::ActiveStreamEncoderFilter::addEncodedData(Buffer::Instance& data) { return parent_.addEncodedData(*this, data); } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index cf5d50f276b2c..c504aa39767bd 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -279,6 +279,7 @@ class ConnectionManagerImpl : Logger::Loggable, // Network::ConnectionCallbacks void onEvent(uint32_t events) override; + // TODO(alyssawilk) disable upstream reads. void onAboveWriteBufferHighWatermark() override {} void onBelowWriteBufferLowWatermark() override {} @@ -354,8 +355,8 @@ class ConnectionManagerImpl : Logger::Loggable, void encodeHeaders(HeaderMapPtr&& headers, bool end_stream) override; void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(HeaderMapPtr&& trailers) override; - void onDecoderFilterAboveWriteBufferHighWatermark() override {} - void onDecoderFilterBelowWriteBufferLowWatermark() override {} + void onDecoderFilterAboveWriteBufferHighWatermark() override; + void onDecoderFilterBelowWriteBufferLowWatermark() override; StreamDecoderFilterSharedPtr handle_; }; @@ -386,6 +387,7 @@ class ConnectionManagerImpl : Logger::Loggable, // Http::StreamEncoderFilterCallbacks void addEncodedData(Buffer::Instance& data) override; + // TODO(alysawilk) disable reads from upstream. void onEncoderFilterAboveWriteBufferHighWatermark() override {} void onEncoderFilterBelowWriteBufferLowWatermark() override {} void continueEncoding() override; @@ -432,6 +434,7 @@ class ConnectionManagerImpl : Logger::Loggable, // Http::StreamCallbacks void onResetStream(StreamResetReason reason) override; + // TODO(alyssawilk) disable upstream reads. void onAboveWriteBufferHighWatermark() override {} void onBelowWriteBufferLowWatermark() override {} diff --git a/source/common/http/http2/BUILD b/source/common/http/http2/BUILD index 247c8c06ecdb0..78e2631837be1 100644 --- a/source/common/http/http2/BUILD +++ b/source/common/http/http2/BUILD @@ -24,6 +24,7 @@ envoy_cc_library( "//include/envoy/stats:stats_interface", "//include/envoy/stats:stats_macros", "//source/common/buffer:buffer_lib", + "//source/common/buffer:watermark_buffer_lib", "//source/common/common:assert_lib", "//source/common/common:enum_to_int", "//source/common/common:linked_object", diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index ba4a12b3a447a..0802576f5809e 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -51,10 +51,14 @@ template static T* remove_const(const void* object) { return const_cast(reinterpret_cast(object)); } -ConnectionImpl::StreamImpl::StreamImpl(ConnectionImpl& parent) +ConnectionImpl::StreamImpl::StreamImpl(ConnectionImpl& parent, uint32_t buffer_limit) : parent_(parent), headers_(new HeaderMapImpl()), local_end_stream_(false), local_end_stream_sent_(false), remote_end_stream_(false), data_deferred_(false), - waiting_for_non_informational_headers_(false) {} + waiting_for_non_informational_headers_(false) { + if (buffer_limit > 0) { + setWriteBufferWatermarks(buffer_limit / 2, buffer_limit); + } +} ConnectionImpl::StreamImpl::~StreamImpl() {} @@ -118,6 +122,40 @@ void ConnectionImpl::StreamImpl::encodeTrailers(const HeaderMap& trailers) { parent_.sendPendingFrames(); } } +void ConnectionImpl::StreamImpl::readDisable(bool disable) { + ENVOY_CONN_LOG(debug, "Stream {} disabled: disable {}, unconsumed_bytes {}", parent_.connection_, + stream_id_, disable, unconsumed_bytes_); + if (disable) { + ++buffers_overrun_; + } else { + --buffers_overrun_; + if (!buffers_overrun()) { + nghttp2_session_consume(parent_.session_, stream_id_, unconsumed_bytes_); + unconsumed_bytes_ = 0; + parent_.sendPendingFrames(); + } + } +} + +void ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() { + ENVOY_CONN_LOG(debug, "recv buffer over limit ", parent_.connection_); + readDisable(true); +} + +void ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark() { + ENVOY_CONN_LOG(debug, "recv buffer under limit ", parent_.connection_); + readDisable(false); +} + +void ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() { + ENVOY_CONN_LOG(debug, "send buffer over limit ", parent_.connection_); + runHighWatermarkCallbacks(); +} + +void ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark() { + ENVOY_CONN_LOG(debug, "send buffer under limit ", parent_.connection_); + runLowWatermarkCallbacks(); +} void ConnectionImpl::StreamImpl::saveHeader(HeaderString&& name, HeaderString&& value) { if (!Utility::reconstituteCrumbledCookies(name, value, cookies_)) { @@ -260,7 +298,13 @@ ConnectionImpl::StreamImpl* ConnectionImpl::getStream(int32_t stream_id) { } int ConnectionImpl::onData(int32_t stream_id, const uint8_t* data, size_t len) { - getStream(stream_id)->pending_recv_data_.add(data, len); + StreamImpl* stream = getStream(stream_id); + stream->pending_recv_data_.add(data, len); + if (!stream->buffers_overrun()) { + nghttp2_session_consume(session_, stream_id, len); + } else { + stream->unconsumed_bytes_ += len; + } return 0; } @@ -658,13 +702,14 @@ ConnectionImpl::Http2Options::Http2Options() { // calculations. This saves a tremendous amount of memory in cases where there are a large number // of kept alive HTTP/2 connections. nghttp2_option_set_no_closed_streams(options_, 1); + nghttp2_option_set_no_auto_window_update(options_, 1); } ConnectionImpl::Http2Options::~Http2Options() { nghttp2_option_del(options_); } ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, - ConnectionCallbacks& callbacks, Stats::Scope& stats, - const Http2Settings& http2_settings) + Http::ConnectionCallbacks& callbacks, + Stats::Scope& stats, const Http2Settings& http2_settings) : ConnectionImpl(connection, stats), callbacks_(callbacks) { nghttp2_session_client_new2(&session_, http2_callbacks_.callbacks(), base(), http2_options_.options()); @@ -672,7 +717,7 @@ ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, } Http::StreamEncoder& ClientConnectionImpl::newStream(StreamDecoder& decoder) { - StreamImplPtr stream(new ClientStreamImpl(*this)); + StreamImplPtr stream(new ClientStreamImpl(*this, connection_.bufferLimit())); stream->decoder_ = &decoder; stream->moveIntoList(std::move(stream), active_streams_); return *active_streams_.front(); @@ -722,7 +767,7 @@ int ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { return 0; } - StreamImplPtr stream(new ServerStreamImpl(*this)); + StreamImplPtr stream(new ServerStreamImpl(*this, connection_.bufferLimit())); stream->decoder_ = &callbacks_.newStream(*stream); stream->stream_id_ = frame->hd.stream_id; stream->moveIntoList(std::move(stream), active_streams_); diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 900dcad53569a..608ceeb079a80 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -14,6 +14,7 @@ #include "envoy/stats/stats_macros.h" #include "common/buffer/buffer_impl.h" +#include "common/buffer/watermark_buffer.h" #include "common/common/linked_object.h" #include "common/common/logger.h" #include "common/http/codec_helper.h" @@ -120,7 +121,7 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable 0; } + ConnectionImpl& parent_; HeaderMapImplPtr headers_; StreamDecoder* decoder_{}; int32_t stream_id_{-1}; - Buffer::OwnedImpl pending_recv_data_; - Buffer::OwnedImpl pending_send_data_; + uint32_t unconsumed_bytes_{0}; + uint32_t buffers_overrun_{0}; + Buffer::WatermarkBuffer pending_recv_data_{ + Buffer::InstancePtr{new Buffer::OwnedImpl}, + [this]() -> void { this->pendingRecvBufferLowWatermark(); }, + [this]() -> void { this->pendingRecvBufferHighWatermark(); }}; + Buffer::WatermarkBuffer pending_send_data_{ + Buffer::InstancePtr{new Buffer::OwnedImpl}, + [this]() -> void { this->pendingSendBufferLowWatermark(); }, + [this]() -> void { this->pendingSendBufferHighWatermark(); }}; HeaderMapPtr pending_trailers_; Optional deferred_reset_; HeaderString cookies_; @@ -202,9 +227,10 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable active_streams_; nghttp2_session* session_{}; CodecStats stats_; + Network::Connection& connection_; private: - virtual ConnectionCallbacks& callbacks() PURE; + virtual Http::ConnectionCallbacks& callbacks() PURE; virtual int onBeginHeaders(const nghttp2_frame* frame) PURE; int onData(int32_t stream_id, const uint8_t* data, size_t len); int onFrameReceived(const nghttp2_frame* frame); @@ -216,7 +242,6 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable CONTINUE_HEADER; - Network::Connection& connection_; bool dispatching_ : 1; bool raised_goaway_ : 1; bool pending_deferred_reset_ : 1; @@ -227,21 +252,31 @@ class ConnectionImpl : public virtual Connection, Logger::LoggablerunHighWatermarkCallbacks(); + } + } + void onBelowWriteBufferLowWatermark() override { + for (auto& stream : active_streams_) { + stream->runLowWatermarkCallbacks(); + } + } private: // ConnectionImpl - ConnectionCallbacks& callbacks() override { return callbacks_; } + Http::ConnectionCallbacks& callbacks() override { return callbacks_; } int onBeginHeaders(const nghttp2_frame* frame) override; int onHeader(const nghttp2_frame* frame, HeaderString&& name, HeaderString&& value) override; - ConnectionCallbacks& callbacks_; + Http::ConnectionCallbacks& callbacks_; }; /** @@ -254,7 +289,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { private: // ConnectionImpl - ConnectionCallbacks& callbacks() override { return callbacks_; } + Http::ConnectionCallbacks& callbacks() override { return callbacks_; } int onBeginHeaders(const nghttp2_frame* frame) override; int onHeader(const nghttp2_frame* frame, HeaderString&& name, HeaderString&& value) override; diff --git a/source/common/router/router.h b/source/common/router/router.h index 3df75737ed2f1..cb184c34801e9 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -165,8 +165,14 @@ class Filter : Logger::Loggable, // Http::StreamCallbacks void onResetStream(Http::StreamResetReason reason) override; - void onAboveWriteBufferHighWatermark() override {} - void onBelowWriteBufferLowWatermark() override {} + void onAboveWriteBufferHighWatermark() override { + // Have the connection manager disable reads on the downstream stream. + parent_.callbacks_->onDecoderFilterAboveWriteBufferHighWatermark(); + } + void onBelowWriteBufferLowWatermark() override { + // Have the connection manager enable reads on the downstream stream. + parent_.callbacks_->onDecoderFilterBelowWriteBufferLowWatermark(); + } // Http::ConnectionPool::Callbacks void onPoolFailure(Http::ConnectionPool::PoolFailureReason reason, diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 0969f61056c8e..4960a13cb2514 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -158,5 +158,13 @@ TEST_F(CodecClientTest, PrematureResponse) { EXPECT_EQ(1U, cluster_->stats_.upstream_cx_protocol_error_.value()); } +TEST_F(CodecClientTest, WatermarkPassthrough) { + EXPECT_CALL(*codec_, onAboveWriteBufferHighWatermark()); + connection_cb_->onAboveWriteBufferHighWatermark(); + + EXPECT_CALL(*codec_, onBelowWriteBufferLowWatermark()); + connection_cb_->onBelowWriteBufferLowWatermark(); +} + } // namespace Http } // namespace Envoy diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index dcfb98adeab84..05f10152e6b60 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -974,6 +974,45 @@ TEST_F(HttpConnectionManagerImplTest, FilterAddBodyInline) { HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}}}, true); } +TEST_F(HttpConnectionManagerImplTest, WatermarkCallbacks) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}}}; + decoder->decodeHeaders(std::move(headers), true); + })); + + setupFilterChain(2, 2); + + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, true)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterHeadersStatus { + Buffer::OwnedImpl data("hello"); + decoder_filters_[0]->callbacks_->addDecodedData(data); + return FilterHeadersStatus::Continue; + })); + EXPECT_CALL(*decoder_filters_[1], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + EXPECT_CALL(*decoder_filters_[1], decodeData(_, true)) + .WillOnce(Return(FilterDataStatus::StopIterationAndBuffer)); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input); + + MockStream stream; + EXPECT_CALL(response_encoder_, getStream()).Times(1).WillOnce(ReturnRef(stream)); + EXPECT_CALL(stream, readDisable(true)); + ASSERT(decoder_filters_[0]->callbacks_ != nullptr); + decoder_filters_[0]->callbacks_->onDecoderFilterAboveWriteBufferHighWatermark(); + + EXPECT_CALL(response_encoder_, getStream()).Times(1).WillOnce(ReturnRef(stream)); + EXPECT_CALL(stream, readDisable(false)); + ASSERT(decoder_filters_[0]->callbacks_ != nullptr); + decoder_filters_[0]->callbacks_->onDecoderFilterBelowWriteBufferLowWatermark(); +} + TEST_F(HttpConnectionManagerImplTest, FilterAddBodyContinuation) { InSequence s; setup(false, ""); diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 9692631885eb9..a860d66b043f2 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -1,6 +1,8 @@ #include #include +#include "envoy/http/codec.h" + #include "common/http/exception.h" #include "common/http/header_map_impl.h" #include "common/http/http2/codec_impl.h" @@ -15,11 +17,13 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::AnyNumber; using testing::AtLeast; using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; using testing::NiceMock; +using testing::Return; using testing::_; namespace Envoy { @@ -40,6 +44,24 @@ Http2Settings Http2SettingsFromTuple(const Http2SettingsTuple& tp) { } } // namespace +class TestServerConnectionImpl : public ServerConnectionImpl { +public: + TestServerConnectionImpl(Network::Connection& connection, ServerConnectionCallbacks& callbacks, + Stats::Scope& scope, const Http2Settings& http2_settings) + : ServerConnectionImpl(connection, callbacks, scope, http2_settings) {} + nghttp2_session* session() { return session_; } + using ServerConnectionImpl::getStream; +}; + +class TestClientConnectionImpl : public ClientConnectionImpl { +public: + TestClientConnectionImpl(Network::Connection& connection, Http::ConnectionCallbacks& callbacks, + Stats::Scope& scope, const Http2Settings& http2_settings) + : ClientConnectionImpl(connection, callbacks, scope, http2_settings) {} + nghttp2_session* session() { return session_; } + using ClientConnectionImpl::getStream; +}; + class Http2CodecImplTest : public testing::TestWithParam { public: struct ConnectionWrapper { @@ -62,8 +84,10 @@ class Http2CodecImplTest : public testing::TestWithParam : client_http2settings_(Http2SettingsFromTuple(::testing::get<0>(GetParam()))), client_(client_connection_, client_callbacks_, stats_store_, client_http2settings_), server_http2settings_(Http2SettingsFromTuple(::testing::get<1>(GetParam()))), - server_(server_connection_, server_callbacks_, stats_store_, server_http2settings_), - request_encoder_(client_.newStream(response_decoder_)) { + server_(server_connection_, server_callbacks_, stats_store_, server_http2settings_) {} + + void initialize() { + request_encoder_ = &client_.newStream(response_decoder_); setupDefaultConnectionMocks(); EXPECT_CALL(server_callbacks_, newStream(_)) @@ -87,21 +111,23 @@ class Http2CodecImplTest : public testing::TestWithParam const Http2Settings client_http2settings_; NiceMock client_connection_; MockConnectionCallbacks client_callbacks_; - ClientConnectionImpl client_; + TestClientConnectionImpl client_; ConnectionWrapper client_wrapper_; const Http2Settings server_http2settings_; NiceMock server_connection_; MockServerConnectionCallbacks server_callbacks_; - ServerConnectionImpl server_; + TestServerConnectionImpl server_; ConnectionWrapper server_wrapper_; MockStreamDecoder response_decoder_; - StreamEncoder& request_encoder_; + StreamEncoder* request_encoder_; MockStreamDecoder request_decoder_; StreamEncoder* response_encoder_{}; MockStreamCallbacks server_stream_callbacks_; }; TEST_P(Http2CodecImplTest, ExpectContinueHeadersOnlyResponse) { + initialize(); + TestHeaderMapImpl request_headers; request_headers.addViaCopy("expect", "100-continue"); HttpTestUtility::addDefaultHeaders(request_headers); @@ -111,11 +137,11 @@ TEST_P(Http2CodecImplTest, ExpectContinueHeadersOnlyResponse) { TestHeaderMapImpl continue_headers{{":status", "100"}}; EXPECT_CALL(response_decoder_, decodeHeaders_(HeaderMapEqual(&continue_headers), false)); - request_encoder_.encodeHeaders(request_headers, false); + request_encoder_->encodeHeaders(request_headers, false); EXPECT_CALL(request_decoder_, decodeData(_, true)); Buffer::OwnedImpl hello("hello"); - request_encoder_.encodeData(hello, true); + request_encoder_->encodeData(hello, true); TestHeaderMapImpl response_headers{{":status", "200"}}; EXPECT_CALL(response_decoder_, decodeHeaders_(HeaderMapEqual(&response_headers), true)); @@ -123,6 +149,8 @@ TEST_P(Http2CodecImplTest, ExpectContinueHeadersOnlyResponse) { } TEST_P(Http2CodecImplTest, ExpectContinueTrailersResponse) { + initialize(); + TestHeaderMapImpl request_headers; request_headers.addViaCopy("expect", "100-continue"); HttpTestUtility::addDefaultHeaders(request_headers); @@ -130,11 +158,11 @@ TEST_P(Http2CodecImplTest, ExpectContinueTrailersResponse) { TestHeaderMapImpl continue_headers{{":status", "100"}}; EXPECT_CALL(response_decoder_, decodeHeaders_(HeaderMapEqual(&continue_headers), false)); - request_encoder_.encodeHeaders(request_headers, false); + request_encoder_->encodeHeaders(request_headers, false); EXPECT_CALL(request_decoder_, decodeData(_, true)); Buffer::OwnedImpl hello("hello"); - request_encoder_.encodeData(hello, true); + request_encoder_->encodeData(hello, true); TestHeaderMapImpl response_headers{{":status", "200"}}; EXPECT_CALL(response_decoder_, decodeHeaders_(HeaderMapEqual(&response_headers), false)); @@ -146,10 +174,12 @@ TEST_P(Http2CodecImplTest, ExpectContinueTrailersResponse) { } TEST_P(Http2CodecImplTest, ShutdownNotice) { + initialize(); + TestHeaderMapImpl request_headers; HttpTestUtility::addDefaultHeaders(request_headers); EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); - request_encoder_.encodeHeaders(request_headers, true); + request_encoder_->encodeHeaders(request_headers, true); EXPECT_CALL(client_callbacks_, onGoAway()); server_.shutdownNotice(); @@ -161,36 +191,42 @@ TEST_P(Http2CodecImplTest, ShutdownNotice) { } TEST_P(Http2CodecImplTest, RefusedStreamReset) { + initialize(); + TestHeaderMapImpl request_headers; HttpTestUtility::addDefaultHeaders(request_headers); EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); - request_encoder_.encodeHeaders(request_headers, false); + request_encoder_->encodeHeaders(request_headers, false); MockStreamCallbacks callbacks; - request_encoder_.getStream().addCallbacks(callbacks); + request_encoder_->getStream().addCallbacks(callbacks); EXPECT_CALL(server_stream_callbacks_, onResetStream(StreamResetReason::LocalRefusedStreamReset)); EXPECT_CALL(callbacks, onResetStream(StreamResetReason::RemoteRefusedStreamReset)); response_encoder_->getStream().resetStream(StreamResetReason::LocalRefusedStreamReset); } TEST_P(Http2CodecImplTest, InvalidFrame) { + initialize(); + ON_CALL(client_connection_, write(_)).WillByDefault(Invoke([&](Buffer::Instance& data) -> void { server_wrapper_.buffer_.add(data); })); - request_encoder_.encodeHeaders(TestHeaderMapImpl{}, true); + request_encoder_->encodeHeaders(TestHeaderMapImpl{}, true); EXPECT_THROW(server_wrapper_.dispatch(Buffer::OwnedImpl(), server_), CodecProtocolException); } TEST_P(Http2CodecImplTest, TrailingHeaders) { + initialize(); + TestHeaderMapImpl request_headers; HttpTestUtility::addDefaultHeaders(request_headers); EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); - request_encoder_.encodeHeaders(request_headers, false); + request_encoder_->encodeHeaders(request_headers, false); EXPECT_CALL(request_decoder_, decodeData(_, false)); Buffer::OwnedImpl hello("hello"); - request_encoder_.encodeData(hello, false); + request_encoder_->encodeData(hello, false); EXPECT_CALL(request_decoder_, decodeTrailers_(_)); - request_encoder_.encodeTrailers(TestHeaderMapImpl{{"trailing", "header"}}); + request_encoder_->encodeTrailers(TestHeaderMapImpl{{"trailing", "header"}}); TestHeaderMapImpl response_headers{{":status", "200"}}; EXPECT_CALL(response_decoder_, decodeHeaders_(_, false)); @@ -203,6 +239,8 @@ TEST_P(Http2CodecImplTest, TrailingHeaders) { } TEST_P(Http2CodecImplTest, TrailingHeadersLargeBody) { + initialize(); + // Buffer server data so we can make sure we don't get any window updates. ON_CALL(client_connection_, write(_)).WillByDefault(Invoke([&](Buffer::Instance& data) -> void { server_wrapper_.buffer_.add(data); @@ -211,12 +249,12 @@ TEST_P(Http2CodecImplTest, TrailingHeadersLargeBody) { TestHeaderMapImpl request_headers; HttpTestUtility::addDefaultHeaders(request_headers); EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); - request_encoder_.encodeHeaders(request_headers, false); + request_encoder_->encodeHeaders(request_headers, false); EXPECT_CALL(request_decoder_, decodeData(_, false)).Times(AtLeast(1)); Buffer::OwnedImpl body(std::string(1024 * 1024, 'a')); - request_encoder_.encodeData(body, false); + request_encoder_->encodeData(body, false); EXPECT_CALL(request_decoder_, decodeTrailers_(_)); - request_encoder_.encodeTrailers(TestHeaderMapImpl{{"trailing", "header"}}); + request_encoder_->encodeTrailers(TestHeaderMapImpl{{"trailing", "header"}}); // Flush pending data. setupDefaultConnectionMocks(); @@ -235,10 +273,12 @@ TEST_P(Http2CodecImplTest, TrailingHeadersLargeBody) { class Http2CodecImplDeferredResetTest : public Http2CodecImplTest {}; TEST_P(Http2CodecImplDeferredResetTest, DeferredResetClient) { + initialize(); + InSequence s; MockStreamCallbacks client_stream_callbacks; - request_encoder_.getStream().addCallbacks(client_stream_callbacks); + request_encoder_->getStream().addCallbacks(client_stream_callbacks); // Do a request, but pause server dispatch so we don't send window updates. This will result in a // deferred reset, followed by a pending frames flush which will cause the stream to actually @@ -248,11 +288,11 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetClient) { })); TestHeaderMapImpl request_headers; HttpTestUtility::addDefaultHeaders(request_headers); - request_encoder_.encodeHeaders(request_headers, false); + request_encoder_->encodeHeaders(request_headers, false); Buffer::OwnedImpl body(std::string(1024 * 1024, 'a')); - request_encoder_.encodeData(body, true); + request_encoder_->encodeData(body, true); EXPECT_CALL(client_stream_callbacks, onResetStream(StreamResetReason::LocalReset)); - request_encoder_.getStream().resetStream(StreamResetReason::LocalReset); + request_encoder_->getStream().resetStream(StreamResetReason::LocalReset); // Dispatch server. We expect to see some data. EXPECT_CALL(response_decoder_, decodeHeaders_(_, _)).Times(0); @@ -271,12 +311,14 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetClient) { } TEST_P(Http2CodecImplDeferredResetTest, DeferredResetServer) { + initialize(); + InSequence s; TestHeaderMapImpl request_headers; HttpTestUtility::addDefaultHeaders(request_headers); EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); - request_encoder_.encodeHeaders(request_headers, false); + request_encoder_->encodeHeaders(request_headers, false); // In this case we do the same thing as DeferredResetClient but on the server side. ON_CALL(server_connection_, write(_)).WillByDefault(Invoke([&](Buffer::Instance& data) -> void { @@ -290,7 +332,7 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetServer) { response_encoder_->getStream().resetStream(StreamResetReason::LocalReset); MockStreamCallbacks client_stream_callbacks; - request_encoder_.getStream().addCallbacks(client_stream_callbacks); + request_encoder_->getStream().addCallbacks(client_stream_callbacks); EXPECT_CALL(response_decoder_, decodeHeaders_(_, false)); EXPECT_CALL(response_decoder_, decodeData(_, false)).Times(AtLeast(1)); EXPECT_CALL(client_stream_callbacks, onResetStream(StreamResetReason::RemoteReset)); @@ -298,16 +340,85 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetServer) { client_wrapper_.dispatch(Buffer::OwnedImpl(), client_); } -// Deferred reset tests use only small windows so that we can test certain conditions. -#define HTTP2SETTINGS_DEFERRED_RESET_COMBINE \ +class Http2CodecImplFlowControlTest : public Http2CodecImplTest {}; + +// Back up the pending_sent_data_ buffer in the client connection and make sure the watermarks fire +// as expected. +// +// This also tests the readDisable logic in StreamImpl, verifying that h2 bytes are consumed +// when the stream has readDisable(true) called. +TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { + EXPECT_CALL(client_connection_, bufferLimit()).WillOnce(Return(10)); + + initialize(); + MockStreamCallbacks callbacks; + request_encoder_->getStream().addCallbacks(callbacks); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + TestHeaderMapImpl expected_headers; + HttpTestUtility::addDefaultHeaders(expected_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(HeaderMapEqual(&expected_headers), false)); + request_encoder_->encodeHeaders(request_headers, false); + + // Force the server stream to be read disabled. This will cause it to stop sending window + // updates to the client. + server_.getStream(1)->readDisable(true); + + uint32_t initial_window = + nghttp2_session_get_stream_effective_local_window_size(client_.session(), 1); + // If this limit is changed, this test will fail due to the initial large writes being divided + // into more than 4 frames. Fast fail here with this explanatory comment. + ASSERT_EQ(65535, initial_window); + // One large write gets broken into smaller frames. + EXPECT_CALL(request_decoder_, decodeData(_, false)).Times(AnyNumber()); + Buffer::OwnedImpl long_data(std::string(initial_window, 'a')); + // The one giant write will cause the buffer to go over the limit, then drain and go back under + // the limit. + EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); + EXPECT_CALL(callbacks, onBelowWriteBufferLowWatermark()); + request_encoder_->encodeData(long_data, false); + + // Verify that the window is full. The client will not send more data to the server for this + // stream. + EXPECT_EQ(0, nghttp2_session_get_stream_local_window_size(server_.session(), 1)); + EXPECT_EQ(0, nghttp2_session_get_stream_remote_window_size(client_.session(), 1)); + EXPECT_EQ(initial_window, server_.getStream(1)->unconsumed_bytes_); + + // Now that the flow control window is full, further data causes the send buffer to back up. + Buffer::OwnedImpl ten_bytes("0123456789"); + request_encoder_->encodeData(ten_bytes, false); + EXPECT_EQ(10, client_.getStream(1)->pending_send_data_.length()); + + // If we go over the limit, the stream callbacks should fire. + EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); + Buffer::OwnedImpl last_byte("!"); + request_encoder_->encodeData(last_byte, false); + EXPECT_EQ(11, client_.getStream(1)->pending_send_data_.length()); + + // Now unblock the server's stream. This will cause the bytes to be consumed, flow control + // updates to be sent, and the client to flush all queued data. + EXPECT_CALL(callbacks, onBelowWriteBufferLowWatermark()); + server_.getStream(1)->readDisable(false); + EXPECT_EQ(0, client_.getStream(1)->pending_send_data_.length()); + // The 11 bytes sent won't trigger another window update, so the final window should be the + // initial window minus the last 11 byte flush from the client to server. + EXPECT_EQ(initial_window - 11, + nghttp2_session_get_stream_local_window_size(server_.session(), 1)); + EXPECT_EQ(initial_window - 11, + nghttp2_session_get_stream_remote_window_size(client_.session(), 1)); +} + +#define HTTP2SETTINGS_SMALL_WINDOW_COMBINE \ ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ ::testing::Values(Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE), \ ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE)) +// Deferred reset tests use only small windows so that we can test certain conditions. INSTANTIATE_TEST_CASE_P(Http2CodecImplDeferredResetTest, Http2CodecImplDeferredResetTest, - ::testing::Combine(HTTP2SETTINGS_DEFERRED_RESET_COMBINE, - HTTP2SETTINGS_DEFERRED_RESET_COMBINE)); + ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, + HTTP2SETTINGS_SMALL_WINDOW_COMBINE)); // we seperate default/edge cases here to avoid combinatorial explosion #define HTTP2SETTINGS_DEFAULT_COMBINE \ @@ -333,6 +444,11 @@ INSTANTIATE_TEST_CASE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, INSTANTIATE_TEST_CASE_P(Http2CodecImplTestEdgeSettings, Http2CodecImplTest, ::testing::Combine(HTTP2SETTINGS_EDGE_COMBINE, HTTP2SETTINGS_EDGE_COMBINE)); +// Flow control tests only use only small windows so that we can test certain conditions. +INSTANTIATE_TEST_CASE_P(Http2CodecImplFlowControlTest, Http2CodecImplFlowControlTest, + ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, + HTTP2SETTINGS_SMALL_WINDOW_COMBINE)); + TEST(Http2CodecUtility, reconstituteCrumbledCookies) { { HeaderString key; diff --git a/test/config/integration/server_http2.json b/test/config/integration/server_http2.json index fd3846bbcedfa..8bb0278b5ed94 100644 --- a/test/config/integration/server_http2.json +++ b/test/config/integration/server_http2.json @@ -174,23 +174,87 @@ } }] }, - { + { "address": "tcp://{{ ip_loopback_address }}:0", + "per_connection_buffer_limit_bytes": 1024, "filters": [ - { "type": "read", "name": - "tcp_proxy", - "config": { - "stat_prefix": "test_tcp", - "route_config": { - "routes": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "http2", + "drain_timeout_ms": 5000, + "access_log": [ + { + "path": "/dev/null", + "filter" : { + "type": "logical_or", + "filters": [ + { + "type": "status_code", + "op": ">=", + "value": 500 + }, { - "cluster": "cluster_1" + "type": "duration", + "op": ">=", + "value": 1000000 } ] } - } + }, + { + "path": "/dev/null" + }], + "stat_prefix": "router", + "route_config": + { + "virtual_hosts": [ + { + "name": "redirect", + "domains": [ "www.redirect.com" ], + "require_ssl": "all", + "routes": [ + { + "prefix": "/", + "cluster": "cluster_3" + } + ] + }, + { + "name": "integration", + "domains": [ "*" ], + "routes": [ + { + "prefix": "/", + "cluster": "cluster_3", + "runtime": { + "key": "some_key", + "default": 0 + } + }, + { + "prefix": "/test/long/url", + "cluster": "cluster_3" + }, + { + "prefix": "/test/", + "cluster": "cluster_2" + } + ] + } + ] + }, + "filters": [ + { "type": "both", "name": "health_check", + "config": { + "pass_through_mode": false, "endpoint": "/healthcheck" + } + }, + { "type": "decoder", "name": "router", "config": {}} + ] } - ] + }] }], "admin": { "access_log_path": "/dev/null", "address": "tcp://{{ ip_loopback_address }}:0" }, @@ -220,6 +284,14 @@ "dns_lookup_family": "{{ dns_lookup_family }}", "hosts": [{"url": "tcp://localhost:{{ upstream_1 }}"}] }, + { + "name": "cluster_3", + "per_connection_buffer_limit_bytes": 1024, + "connect_timeout_ms": 5000, + "type": "static", + "lb_type": "round_robin", + "hosts": [{"url": "tcp://{{ ip_loopback_address }}:{{ upstream_0 }}"}] + }, { "name": "statsd", "connect_timeout_ms": 5000, diff --git a/test/config/integration/server_http2_upstream.json b/test/config/integration/server_http2_upstream.json index 2778ef91f9b2d..fe93afa35d1c6 100644 --- a/test/config/integration/server_http2_upstream.json +++ b/test/config/integration/server_http2_upstream.json @@ -253,6 +253,88 @@ ] } }] + }, + { + "address": "tcp://{{ ip_loopback_address }}:0", + "per_connection_buffer_limit_bytes": 1024, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "http2", + "drain_timeout_ms": 5000, + "access_log": [ + { + "path": "/dev/null", + "filter" : { + "type": "logical_or", + "filters": [ + { + "type": "status_code", + "op": ">=", + "value": 500 + }, + { + "type": "duration", + "op": ">=", + "value": 1000000 + } + ] + } + }, + { + "path": "/dev/null" + }], + "stat_prefix": "router", + "route_config": + { + "virtual_hosts": [ + { + "name": "redirect", + "domains": [ "www.redirect.com" ], + "require_ssl": "all", + "routes": [ + { + "prefix": "/", + "cluster": "cluster_1" + } + ] + }, + { + "name": "integration", + "domains": [ "*" ], + "routes": [ + { + "prefix": "/", + "cluster": "cluster_3", + "runtime": { + "key": "some_key", + "default": 0 + } + }, + { + "prefix": "/test/long/url", + "cluster": "cluster_3" + }, + { + "prefix": "/test/", + "cluster": "cluster_3" + } + ] + } + ] + }, + "filters": [ + { "type": "both", "name": "health_check", + "config": { + "pass_through_mode": false, "endpoint": "/healthcheck" + } + }, + { "type": "decoder", "name": "router", "config": {} } + ] + } + }] }], "admin": { "access_log_path": "/dev/null", "address": "tcp://{{ ip_loopback_address }}:0" }, @@ -275,6 +357,15 @@ "features": "http2", "dns_lookup_family": "{{ dns_lookup_family }}", "hosts": [{"url": "tcp://localhost:{{ upstream_1 }}"}] + }, + { + "name": "cluster_3", + "per_connection_buffer_limit_bytes": 1024, + "connect_timeout_ms": 5000, + "type": "static", + "lb_type": "round_robin", + "features": "http2", + "hosts": [{"url": "tcp://{{ ip_loopback_address }}:{{ upstream_0 }}"}] }] } } diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 67c231cb1cf56..f5c65e6540f32 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -59,6 +59,12 @@ TEST_P(Http2IntegrationTest, RouterRequestAndResponseWithGiantBodyBuffer) { false); } +TEST_P(Http2IntegrationTest, FlowControlOnAndGiantBody) { + testRouterRequestAndResponseWithBody(makeClientConnection(lookupPort("http_buffer_limits")), + Http::CodecClient::Type::HTTP2, 1024 * 1024, 1024 * 1024, + false); +} + TEST_P(Http2IntegrationTest, RouterHeaderOnlyRequestAndResponseNoBuffer) { testRouterHeaderOnlyRequestAndResponse(makeClientConnection(lookupPort("http")), Http::CodecClient::Type::HTTP2); @@ -181,7 +187,8 @@ TEST_P(Http2IntegrationTest, Trailers) { testTrailers(1024, 2048); } TEST_P(Http2IntegrationTest, TrailersGiantBody) { testTrailers(1024 * 1024, 1024 * 1024); } -TEST_P(Http2IntegrationTest, SimultaneousRequest) { +void Http2IntegrationTest::simultaneousRequest(uint32_t port, int32_t request1_bytes, + int32_t request2_bytes) { IntegrationCodecClientPtr codec_client; FakeHttpConnectionPtr fake_upstream_connection1; FakeHttpConnectionPtr fake_upstream_connection2; @@ -192,9 +199,7 @@ TEST_P(Http2IntegrationTest, SimultaneousRequest) { FakeStreamPtr upstream_request1; FakeStreamPtr upstream_request2; executeActions( - {[&]() -> void { - codec_client = makeHttpConnection(lookupPort("http"), Http::CodecClient::Type::HTTP2); - }, + {[&]() -> void { codec_client = makeHttpConnection(port, Http::CodecClient::Type::HTTP2); }, // Start request 1 [&]() -> void { encoder1 = &codec_client->startRequest(Http::TestHeaderMapImpl{{":method", "POST"}, @@ -224,14 +229,14 @@ TEST_P(Http2IntegrationTest, SimultaneousRequest) { // Finish request 1 [&]() -> void { - codec_client->sendData(*encoder1, 1024, true); + codec_client->sendData(*encoder1, request1_bytes, true); }, [&]() -> void { upstream_request1->waitForEndStream(*dispatcher_); }, // Finish request 2 [&]() -> void { - codec_client->sendData(*encoder2, 512, true); + codec_client->sendData(*encoder2, request2_bytes, true); }, [&]() -> void { upstream_request2->waitForEndStream(*dispatcher_); }, @@ -239,31 +244,31 @@ TEST_P(Http2IntegrationTest, SimultaneousRequest) { // Respond request 2 [&]() -> void { upstream_request2->encodeHeaders(Http::TestHeaderMapImpl{{":status", "200"}}, false); - upstream_request2->encodeData(1024, true); + upstream_request2->encodeData(request2_bytes, true); }, [&]() -> void { response2->waitForEndStream(); EXPECT_TRUE(upstream_request2->complete()); - EXPECT_EQ(512U, upstream_request2->bodyLength()); + EXPECT_EQ(request2_bytes, upstream_request2->bodyLength()); EXPECT_TRUE(response2->complete()); EXPECT_STREQ("200", response2->headers().Status()->value().c_str()); - EXPECT_EQ(1024U, response2->body().size()); + EXPECT_EQ(request2_bytes, response2->body().size()); }, // Respond request 1 [&]() -> void { upstream_request1->encodeHeaders(Http::TestHeaderMapImpl{{":status", "200"}}, false); - upstream_request1->encodeData(512, true); + upstream_request1->encodeData(request2_bytes, true); }, [&]() -> void { response1->waitForEndStream(); EXPECT_TRUE(upstream_request1->complete()); - EXPECT_EQ(1024U, upstream_request1->bodyLength()); + EXPECT_EQ(request1_bytes, upstream_request1->bodyLength()); EXPECT_TRUE(response1->complete()); EXPECT_STREQ("200", response1->headers().Status()->value().c_str()); - EXPECT_EQ(512U, response1->body().size()); + EXPECT_EQ(request2_bytes, response1->body().size()); }, // Cleanup both downstream and upstream @@ -273,4 +278,13 @@ TEST_P(Http2IntegrationTest, SimultaneousRequest) { [&]() -> void { fake_upstream_connection2->close(); }, [&]() -> void { fake_upstream_connection2->waitForDisconnect(); }}); } + +TEST_P(Http2IntegrationTest, SimultaneousRequest) { + simultaneousRequest(lookupPort("http"), 1024, 512); +} + +TEST_P(Http2IntegrationTest, SimultaneousRequestWithBufferLimits) { + simultaneousRequest(lookupPort("http_buffer_limits"), 1024 * 32, 1024 * 16); +} + } // namespace Envoy diff --git a/test/integration/http2_integration_test.h b/test/integration/http2_integration_test.h index 7daad5071fbe2..a9c624273539a 100644 --- a/test/integration/http2_integration_test.h +++ b/test/integration/http2_integration_test.h @@ -17,9 +17,12 @@ class Http2IntegrationTest : public BaseIntegrationTest, registerPort("upstream_0", fake_upstreams_.back()->localAddress()->ip()->port()); fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP1, version_)); registerPort("upstream_1", fake_upstreams_.back()->localAddress()->ip()->port()); - createTestServer("test/config/integration/server_http2.json", {"echo", "http", "http_buffer"}); + createTestServer("test/config/integration/server_http2.json", + {"echo", "http", "http_buffer", "http_buffer_limits"}); } + void simultaneousRequest(uint32_t port, int32_t request1_bytes, int32_t request2_bytes); + /** * Destructor for an individual test test. */ diff --git a/test/integration/http2_upstream_integration_test.cc b/test/integration/http2_upstream_integration_test.cc index ea583cbe01cef..bb8423adf3375 100644 --- a/test/integration/http2_upstream_integration_test.cc +++ b/test/integration/http2_upstream_integration_test.cc @@ -101,16 +101,14 @@ TEST_P(Http2UpstreamIntegrationTest, DownstreamResetBeforeResponseComplete) { TEST_P(Http2UpstreamIntegrationTest, Trailers) { testTrailers(1024, 2048); } -TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreaming) { +void Http2UpstreamIntegrationTest::bidirectionalStreaming(uint32_t port, uint32_t bytes) { IntegrationCodecClientPtr codec_client; FakeHttpConnectionPtr fake_upstream_connection; Http::StreamEncoder* encoder; IntegrationStreamDecoderPtr response(new IntegrationStreamDecoder(*dispatcher_)); FakeStreamPtr upstream_request; executeActions( - {[&]() -> void { - codec_client = makeHttpConnection(lookupPort("http"), Http::CodecClient::Type::HTTP2); - }, + {[&]() -> void { codec_client = makeHttpConnection(port, Http::CodecClient::Type::HTTP2); }, // Start request [&]() -> void { encoder = &codec_client->startRequest(Http::TestHeaderMapImpl{{":method", "POST"}, @@ -126,17 +124,17 @@ TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreaming) { // Send some data [&]() -> void { - codec_client->sendData(*encoder, 1024, false); + codec_client->sendData(*encoder, bytes, false); }, - [&]() -> void { upstream_request->waitForData(*dispatcher_, 1024); }, + [&]() -> void { upstream_request->waitForData(*dispatcher_, bytes); }, // Start response [&]() -> void { upstream_request->encodeHeaders(Http::TestHeaderMapImpl{{":status", "200"}}, false); - upstream_request->encodeData(1024, false); + upstream_request->encodeData(bytes, false); }, - [&]() -> void { response->waitForBodyData(1024); }, + [&]() -> void { response->waitForBodyData(bytes); }, // Finish request [&]() -> void { @@ -159,6 +157,14 @@ TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreaming) { EXPECT_TRUE(response->complete()); } +TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreaming) { + bidirectionalStreaming(lookupPort("http"), 1024); +} + +TEST_P(Http2UpstreamIntegrationTest, LargeBidirectionalStreamingWithBufferLimits) { + bidirectionalStreaming(lookupPort("http_with_buffer_limits"), 1024 * 32); +} + TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreamingReset) { IntegrationCodecClientPtr codec_client; FakeHttpConnectionPtr fake_upstream_connection; @@ -215,7 +221,10 @@ TEST_P(Http2UpstreamIntegrationTest, BidirectionalStreamingReset) { EXPECT_FALSE(response->complete()); } -TEST_P(Http2UpstreamIntegrationTest, SimultaneousRequest) { +void Http2UpstreamIntegrationTest::simultaneousRequest(uint32_t port, uint32_t request1_bytes, + uint32_t request2_bytes, + uint32_t response1_bytes, + uint32_t response2_bytes) { IntegrationCodecClientPtr codec_client; FakeHttpConnectionPtr fake_upstream_connection; Http::StreamEncoder* encoder1; @@ -225,9 +234,7 @@ TEST_P(Http2UpstreamIntegrationTest, SimultaneousRequest) { FakeStreamPtr upstream_request1; FakeStreamPtr upstream_request2; executeActions( - {[&]() -> void { - codec_client = makeHttpConnection(lookupPort("http"), Http::CodecClient::Type::HTTP2); - }, + {[&]() -> void { codec_client = makeHttpConnection(port, Http::CodecClient::Type::HTTP2); }, // Start request 1 [&]() -> void { encoder1 = &codec_client->startRequest(Http::TestHeaderMapImpl{{":method", "POST"}, @@ -254,14 +261,14 @@ TEST_P(Http2UpstreamIntegrationTest, SimultaneousRequest) { // Finish request 1 [&]() -> void { - codec_client->sendData(*encoder1, 1024, true); + codec_client->sendData(*encoder1, request1_bytes, true); }, [&]() -> void { upstream_request1->waitForEndStream(*dispatcher_); }, // Finish request 2 [&]() -> void { - codec_client->sendData(*encoder2, 512, true); + codec_client->sendData(*encoder2, request2_bytes, true); }, [&]() -> void { upstream_request2->waitForEndStream(*dispatcher_); }, @@ -269,31 +276,31 @@ TEST_P(Http2UpstreamIntegrationTest, SimultaneousRequest) { // Respond request 2 [&]() -> void { upstream_request2->encodeHeaders(Http::TestHeaderMapImpl{{":status", "200"}}, false); - upstream_request2->encodeData(1024, true); + upstream_request2->encodeData(response2_bytes, true); }, [&]() -> void { response2->waitForEndStream(); EXPECT_TRUE(upstream_request2->complete()); - EXPECT_EQ(512U, upstream_request2->bodyLength()); + EXPECT_EQ(request2_bytes, upstream_request2->bodyLength()); EXPECT_TRUE(response2->complete()); EXPECT_STREQ("200", response2->headers().Status()->value().c_str()); - EXPECT_EQ(1024U, response2->body().size()); + EXPECT_EQ(response2_bytes, response2->body().size()); }, // Respond request 1 [&]() -> void { upstream_request1->encodeHeaders(Http::TestHeaderMapImpl{{":status", "200"}}, false); - upstream_request1->encodeData(512, true); + upstream_request1->encodeData(response1_bytes, true); }, [&]() -> void { response1->waitForEndStream(); EXPECT_TRUE(upstream_request1->complete()); - EXPECT_EQ(1024U, upstream_request1->bodyLength()); + EXPECT_EQ(request1_bytes, upstream_request1->bodyLength()); EXPECT_TRUE(response1->complete()); EXPECT_STREQ("200", response1->headers().Status()->value().c_str()); - EXPECT_EQ(512U, response1->body().size()); + EXPECT_EQ(response1_bytes, response1->body().size()); }, // Cleanup both downstream and upstream @@ -301,4 +308,14 @@ TEST_P(Http2UpstreamIntegrationTest, SimultaneousRequest) { [&]() -> void { fake_upstream_connection->close(); }, [&]() -> void { fake_upstream_connection->waitForDisconnect(); }}); } + +TEST_P(Http2UpstreamIntegrationTest, SimultaneousRequest) { + simultaneousRequest(lookupPort("http"), 1024, 512, 1023, 513); +} + +TEST_P(Http2UpstreamIntegrationTest, LargeSimultaneousRequestWithBufferLimits) { + simultaneousRequest(lookupPort("http_with_buffer_limits"), 1024 * 20, 1024 * 14 + 2, + 1024 * 10 + 5, 1024 * 16); +} + } // namespace Envoy diff --git a/test/integration/http2_upstream_integration_test.h b/test/integration/http2_upstream_integration_test.h index fe7ca71b8b86d..a515842473a77 100644 --- a/test/integration/http2_upstream_integration_test.h +++ b/test/integration/http2_upstream_integration_test.h @@ -17,10 +17,16 @@ class Http2UpstreamIntegrationTest : public BaseIntegrationTest, registerPort("upstream_0", fake_upstreams_.back()->localAddress()->ip()->port()); fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_)); registerPort("upstream_1", fake_upstreams_.back()->localAddress()->ip()->port()); + fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_)); + registerPort("upstream_3", fake_upstreams_.back()->localAddress()->ip()->port()); createTestServer("test/config/integration/server_http2_upstream.json", - {"http", "http_buffer", "http1_buffer"}); + {"http", "http_buffer", "http1_buffer", "http_with_buffer_limits"}); } + void bidirectionalStreaming(uint32_t port, uint32_t bytes); + void simultaneousRequest(uint32_t port, uint32_t request1_bytes, uint32_t request2_bytes, + uint32_t response1_bytes, uint32_t response2_bytes); + /** * Destructor for an individual test. */ From 0437eca082e8f262d9495d2862fd82f0ceb35e5e Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 25 Jul 2017 15:53:12 -0400 Subject: [PATCH 03/19] Adding some stats --- docs/configuration/http_conn_man/stats.rst | 2 + .../http_filters/router_filter.rst | 2 + source/common/http/conn_manager_impl.cc | 2 + source/common/http/conn_manager_impl.h | 2 + source/common/router/router.h | 4 ++ test/common/http/conn_manager_impl_test.cc | 2 + test/common/router/router_test.cc | 38 +++++++++++++++++++ 7 files changed, 52 insertions(+) diff --git a/docs/configuration/http_conn_man/stats.rst b/docs/configuration/http_conn_man/stats.rst index 2395d2ce349d2..648be6bca6526 100644 --- a/docs/configuration/http_conn_man/stats.rst +++ b/docs/configuration/http_conn_man/stats.rst @@ -32,6 +32,8 @@ statistics: downstream_cx_tx_bytes_buffered, Gauge, Total sent bytes currently buffered downstream_cx_drain_close, Counter, Total connections closed due to draining downstream_cx_idle_timeout, Counter, Total connections closed due to idle timeout + downstream_flow_control_paused_reading_total, Counter, Total number of times reads were disabled due to flow control + downstream_flow_control_resumed_reading_total, Counter, Total number of times reads were enabled on the connection due to flow control downstream_rq_total, Counter, Total requests downstream_rq_http1_total, Counter, Total HTTP/1.1 requests downstream_rq_http2_total, Counter, Total HTTP/2 requests diff --git a/docs/configuration/http_filters/router_filter.rst b/docs/configuration/http_filters/router_filter.rst index be8db6bf3bfb0..d79e2f3593943 100644 --- a/docs/configuration/http_filters/router_filter.rst +++ b/docs/configuration/http_filters/router_filter.rst @@ -219,6 +219,8 @@ prefix ` comes from the owning HTTP connection no_cluster, Counter, Total requests in which the target cluster did not exist and resulted in a 404 rq_redirect, Counter, Total requests that resulted in a redirect response rq_total, Counter, Total routed requests + flow_control_paused_downstream_reads_total, Counter, Total times requests backed up enough to pause reading from downstream. + flow_control_resumed_downstream_reads_total, Counter, Total number of times requests resumed reading from downstream. Virtual cluster statistics are output in the *vhost..vcluster..* namespace and include the following diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 6e8e938e88bb5..e8d5196027b18 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -991,12 +991,14 @@ void ConnectionManagerImpl::ActiveStreamDecoderFilter:: onDecoderFilterAboveWriteBufferHighWatermark() { ENVOY_STREAM_LOG(debug, "Read-disabling downstream stream due to filter callbacks.", parent_); parent_.response_encoder_->getStream().readDisable(true); + parent_.connection_manager_.stats_.named_.downstream_flow_control_paused_reading_total_.inc(); } void ConnectionManagerImpl::ActiveStreamDecoderFilter:: onDecoderFilterBelowWriteBufferLowWatermark() { ENVOY_STREAM_LOG(debug, "Read-enabling downstream stream due to filter callbacks.", parent_); parent_.response_encoder_->getStream().readDisable(false); + parent_.connection_manager_.stats_.named_.downstream_flow_control_resumed_reading_total_.inc(); } void ConnectionManagerImpl::ActiveStreamEncoderFilter::addEncodedData(Buffer::Instance& data) { diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index c504aa39767bd..fd182459cdd61 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -58,6 +58,8 @@ namespace Http { GAUGE (downstream_cx_tx_bytes_buffered) \ COUNTER(downstream_cx_drain_close) \ COUNTER(downstream_cx_idle_timeout) \ + COUNTER(downstream_flow_control_paused_reading_total) \ + COUNTER(downstream_flow_control_resumed_reading_total) \ COUNTER(downstream_rq_total) \ COUNTER(downstream_rq_http1_total) \ COUNTER(downstream_rq_http2_total) \ diff --git a/source/common/router/router.h b/source/common/router/router.h index cb184c34801e9..f20512e2079bd 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -24,6 +24,8 @@ namespace Router { */ // clang-format off #define ALL_ROUTER_STATS(COUNTER) \ + COUNTER(flow_control_paused_downstream_reads_total) \ + COUNTER(flow_control_resumed_downstream_reads_total) \ COUNTER(no_route) \ COUNTER(no_cluster) \ COUNTER(rq_redirect) \ @@ -167,10 +169,12 @@ class Filter : Logger::Loggable, void onResetStream(Http::StreamResetReason reason) override; void onAboveWriteBufferHighWatermark() override { // Have the connection manager disable reads on the downstream stream. + parent_.config_.stats_.flow_control_paused_downstream_reads_total_.inc(); parent_.callbacks_->onDecoderFilterAboveWriteBufferHighWatermark(); } void onBelowWriteBufferLowWatermark() override { // Have the connection manager enable reads on the downstream stream. + parent_.config_.stats_.flow_control_resumed_downstream_reads_total_.inc(); parent_.callbacks_->onDecoderFilterBelowWriteBufferLowWatermark(); } diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 05f10152e6b60..ae83fdc1921fd 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -1006,11 +1006,13 @@ TEST_F(HttpConnectionManagerImplTest, WatermarkCallbacks) { EXPECT_CALL(stream, readDisable(true)); ASSERT(decoder_filters_[0]->callbacks_ != nullptr); decoder_filters_[0]->callbacks_->onDecoderFilterAboveWriteBufferHighWatermark(); + EXPECT_EQ(1U, stats_.named_.downstream_flow_control_paused_reading_total_.value()); EXPECT_CALL(response_encoder_, getStream()).Times(1).WillOnce(ReturnRef(stream)); EXPECT_CALL(stream, readDisable(false)); ASSERT(decoder_filters_[0]->callbacks_ != nullptr); decoder_filters_[0]->callbacks_->onDecoderFilterBelowWriteBufferLowWatermark(); + EXPECT_EQ(1U, stats_.named_.downstream_flow_control_resumed_reading_total_.value()); } TEST_F(HttpConnectionManagerImplTest, FilterAddBodyContinuation) { diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 19a0e18d93956..1e18f3ed70d30 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1141,5 +1141,43 @@ TEST_F(RouterTest, AutoHostRewriteDisabled) { router_.decodeHeaders(incoming_headers, true); } +TEST_F(RouterTest, Watermarks) { + EXPECT_CALL(callbacks_.route_->route_entry_, timeout()) + .WillOnce(Return(std::chrono::milliseconds(0))); + EXPECT_CALL(callbacks_.dispatcher_, createTimer_(_)).Times(0); + + NiceMock encoder; + NiceMock stream; + Http::StreamCallbacks* stream_callbacks; + EXPECT_CALL(stream, addCallbacks(_)).WillOnce(Invoke([&](Http::StreamCallbacks& callbacks) { + stream_callbacks = &callbacks; + })); + EXPECT_CALL(encoder, getStream()).WillOnce(ReturnRef(stream)); + Http::StreamDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + return nullptr; + })); + + Http::TestHeaderMapImpl headers{{"x-envoy-upstream-alt-stat-name", "alt_stat"}, + {"x-envoy-internal", "true"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + stream_callbacks->onAboveWriteBufferHighWatermark(); + EXPECT_EQ(1UL, stats_store_.counter("test.flow_control_paused_downstream_reads_total").value()); + stream_callbacks->onBelowWriteBufferLowWatermark(); + EXPECT_EQ(1UL, stats_store_.counter("test.flow_control_resumed_downstream_reads_total").value()); + + Http::HeaderMapPtr response_headers( + new Http::TestHeaderMapImpl{{":status", "200"}, + {"x-envoy-upstream-canary", "false"}, + {"x-envoy-virtual-cluster", "hello"}}); + response_decoder->decodeHeaders(std::move(response_headers), true); +} + } // namespace Router } // namespace Envoy From 2d86666a352fede309f439115c93ada6bdfe294f Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 26 Jul 2017 11:23:01 -0400 Subject: [PATCH 04/19] Taking stream buffer limits from the H2 config --- .../http_conn_man/http_conn_man.rst | 5 +++ include/envoy/http/codec.h | 5 +++ include/envoy/http/filter.h | 10 ++++- source/common/http/http2/codec_impl.cc | 9 +++-- source/common/http/http2/codec_impl.h | 8 ++-- source/common/http/utility.cc | 2 + source/common/json/config_schemas.cc | 10 +++++ test/common/http/http2/codec_impl_test.cc | 37 +++++++++++++------ test/common/http/utility_test.cc | 8 +++- test/config/integration/server_http2.json | 3 ++ .../integration/server_http2_upstream.json | 3 ++ test/test_common/utility.cc | 1 + 12 files changed, 79 insertions(+), 22 deletions(-) diff --git a/docs/configuration/http_conn_man/http_conn_man.rst b/docs/configuration/http_conn_man/http_conn_man.rst index 82fca2669610b..9b41df9b68d2c 100644 --- a/docs/configuration/http_conn_man/http_conn_man.rst +++ b/docs/configuration/http_conn_man/http_conn_man.rst @@ -141,6 +141,11 @@ http2_settings window. Currently , this has the same minimum/maximum/default as :ref:`initial_stream_window_size `. + per_stream_buffer_limit + *(optional, integer)* A soft limit on the number of bytes envoy will buffer per-stream in the + codec buffers. Once the buffer reaches this point, Watermark callbacks will fire to stop the + flow of data to the codec buffers. If this limit is zero, no buffer limits will be applied. + These are the same options available in the upstream cluster :ref:`http2_settings ` option. diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 5c9d840fe2834..5e4961214b830 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -174,6 +174,8 @@ struct Http2Settings { uint32_t max_concurrent_streams_{DEFAULT_MAX_CONCURRENT_STREAMS}; uint32_t initial_stream_window_size_{DEFAULT_INITIAL_STREAM_WINDOW_SIZE}; uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE}; + // This setting applies to the HTTP/2 codec but is not an HTTP/2 setting per the spec. + uint32_t per_stream_buffer_limit_{DEFAULT_PER_STREAM_BUFFER_LIMIT}; // disable HPACK compression static const uint32_t MIN_HPACK_TABLE_SIZE = 0; @@ -207,6 +209,9 @@ struct Http2Settings { // our default connection-level window also equals to our stream-level static const uint32_t DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE = 256 * 1024 * 1024; static const uint32_t MAX_INITIAL_CONNECTION_WINDOW_SIZE = (1U << 31) - 1; + + // By default, do not enforce stream buffer limits beyond the connection level limits. + static const uint32_t DEFAULT_PER_STREAM_BUFFER_LIMIT = 0; }; /** diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index d237cf066131d..5b1aae7e0d450 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -200,12 +200,18 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { virtual void encodeTrailers(HeaderMapPtr&& trailers) PURE; /** - * Called when a decoder filter goes over its high watermark. + * Called when the buffer for a decoder filter or any buffers the filter sends data to go over + * their high watermark. + * + * In the case of a filter such as the router filter, which spills into multiple buffers (codec, + * connection etc.) this may be called multiple times. Any such filter is responsible for calling + * the low watermark callbacks an equal number of times as the respective buffers are drained. */ virtual void onDecoderFilterAboveWriteBufferHighWatermark() PURE; /** - * Called when a decoder filter goes from over its high watermark to under its low watermark. + * Called when a decoder filter or any buffers the filter sends data to go from over its high + * watermark to under its low watermark. */ virtual void onDecoderFilterBelowWriteBufferLowWatermark() PURE; }; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 0802576f5809e..58e3490998c19 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -710,14 +710,15 @@ ConnectionImpl::Http2Options::~Http2Options() { nghttp2_option_del(options_); } ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, Http::ConnectionCallbacks& callbacks, Stats::Scope& stats, const Http2Settings& http2_settings) - : ConnectionImpl(connection, stats), callbacks_(callbacks) { + : ConnectionImpl(connection, stats, http2_settings), callbacks_(callbacks) { nghttp2_session_client_new2(&session_, http2_callbacks_.callbacks(), base(), http2_options_.options()); sendSettings(http2_settings); } Http::StreamEncoder& ClientConnectionImpl::newStream(StreamDecoder& decoder) { - StreamImplPtr stream(new ClientStreamImpl(*this, connection_.bufferLimit())); + + StreamImplPtr stream(new ClientStreamImpl(*this, per_stream_buffer_limit_)); stream->decoder_ = &decoder; stream->moveIntoList(std::move(stream), active_streams_); return *active_streams_.front(); @@ -748,7 +749,7 @@ int ClientConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, Http::ServerConnectionCallbacks& callbacks, Stats::Scope& scope, const Http2Settings& http2_settings) - : ConnectionImpl(connection, scope), callbacks_(callbacks) { + : ConnectionImpl(connection, scope, http2_settings), callbacks_(callbacks) { nghttp2_session_server_new2(&session_, http2_callbacks_.callbacks(), base(), http2_options_.options()); sendSettings(http2_settings); @@ -767,7 +768,7 @@ int ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { return 0; } - StreamImplPtr stream(new ServerStreamImpl(*this, connection_.bufferLimit())); + StreamImplPtr stream(new ServerStreamImpl(*this, per_stream_buffer_limit_)); stream->decoder_ = &callbacks_.newStream(*stream); stream->stream_id_ = frame->hd.stream_id; stream->moveIntoList(std::move(stream), active_streams_); diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 608ceeb079a80..13213279040c1 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -69,10 +69,11 @@ class Utility { */ class ConnectionImpl : public virtual Connection, Logger::Loggable { public: - ConnectionImpl(Network::Connection& connection, Stats::Scope& stats) + ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, + const Http2Settings& http2_settings) : stats_{ALL_HTTP2_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http2."))}, - connection_(connection), dispatching_(false), raised_goaway_(false), - pending_deferred_reset_(false) {} + connection_(connection), per_stream_buffer_limit_(http2_settings.per_stream_buffer_limit_), + dispatching_(false), raised_goaway_(false), pending_deferred_reset_(false) {} ~ConnectionImpl(); @@ -228,6 +229,7 @@ class ConnectionImpl : public virtual Connection, Logger::LoggablegetInteger("initial_connection_window_size", Http::Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE); + ret.per_stream_buffer_limit_ = http2_settings->getInteger( + "per_stream_buffer_limit", Http::Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT); // http_codec_options config is DEPRECATED std::string options = config.getString("http_codec_options", ""); diff --git a/source/common/json/config_schemas.cc b/source/common/json/config_schemas.cc index ff30a0d040022..edb3433a21e05 100644 --- a/source/common/json/config_schemas.cc +++ b/source/common/json/config_schemas.cc @@ -286,6 +286,11 @@ const std::string Json::Schema::HTTP_CONN_NETWORK_FILTER_SCHEMA(R"EOF( "type": "integer", "minimum": 65535, "maximum" : 2147483647 + }, + "per_stream_buffer_limit" : { + "type": "integer", + "minimum": 0, + "exclusiveMinimum" : true } } }, @@ -1380,6 +1385,11 @@ const std::string Json::Schema::CLUSTER_SCHEMA(R"EOF( "type": "integer", "minimum": 65535, "maximum" : 2147483647 + }, + "per_stream_buffer_limit" : { + "type": "integer", + "minimum": 0, + "exclusiveMinimum" : true } } }, diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index a860d66b043f2..1197357c042ec 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -30,16 +30,21 @@ namespace Envoy { namespace Http { namespace Http2 { -typedef ::testing::tuple Http2SettingsTuple; +typedef ::testing::tuple Http2SettingsTuple; typedef ::testing::tuple Http2SettingsTestParam; +const char SMALL_WINDOW = 10; + namespace { + Http2Settings Http2SettingsFromTuple(const Http2SettingsTuple& tp) { Http2Settings ret; ret.hpack_table_size_ = ::testing::get<0>(tp); ret.max_concurrent_streams_ = ::testing::get<1>(tp); ret.initial_stream_window_size_ = ::testing::get<2>(tp); ret.initial_connection_window_size_ = ::testing::get<3>(tp); + ret.per_stream_buffer_limit_ = ::testing::get<4>(tp); + return ret; } } // namespace @@ -348,8 +353,6 @@ class Http2CodecImplFlowControlTest : public Http2CodecImplTest {}; // This also tests the readDisable logic in StreamImpl, verifying that h2 bytes are consumed // when the stream has readDisable(true) called. TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { - EXPECT_CALL(client_connection_, bufferLimit()).WillOnce(Return(10)); - initialize(); MockStreamCallbacks callbacks; request_encoder_->getStream().addCallbacks(callbacks); @@ -388,13 +391,13 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { // Now that the flow control window is full, further data causes the send buffer to back up. Buffer::OwnedImpl ten_bytes("0123456789"); request_encoder_->encodeData(ten_bytes, false); - EXPECT_EQ(10, client_.getStream(1)->pending_send_data_.length()); + EXPECT_EQ(SMALL_WINDOW, client_.getStream(1)->pending_send_data_.length()); // If we go over the limit, the stream callbacks should fire. EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); Buffer::OwnedImpl last_byte("!"); request_encoder_->encodeData(last_byte, false); - EXPECT_EQ(11, client_.getStream(1)->pending_send_data_.length()); + EXPECT_EQ(SMALL_WINDOW + 1, client_.getStream(1)->pending_send_data_.length()); // Now unblock the server's stream. This will cause the bytes to be consumed, flow control // updates to be sent, and the client to flush all queued data. @@ -403,29 +406,38 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { EXPECT_EQ(0, client_.getStream(1)->pending_send_data_.length()); // The 11 bytes sent won't trigger another window update, so the final window should be the // initial window minus the last 11 byte flush from the client to server. - EXPECT_EQ(initial_window - 11, + EXPECT_EQ(initial_window - (SMALL_WINDOW + 1), nghttp2_session_get_stream_local_window_size(server_.session(), 1)); - EXPECT_EQ(initial_window - 11, + EXPECT_EQ(initial_window - (SMALL_WINDOW + 1), nghttp2_session_get_stream_remote_window_size(client_.session(), 1)); } +#define HTTP2SETTINGS_DEFERRED_RESET_COMBINE \ + ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ + ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ + ::testing::Values(Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE), \ + ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE), \ + ::testing::Values(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT)) + #define HTTP2SETTINGS_SMALL_WINDOW_COMBINE \ ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ ::testing::Values(Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE)) + ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE), \ + ::testing::Values(SMALL_WINDOW)) // Deferred reset tests use only small windows so that we can test certain conditions. INSTANTIATE_TEST_CASE_P(Http2CodecImplDeferredResetTest, Http2CodecImplDeferredResetTest, - ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, - HTTP2SETTINGS_SMALL_WINDOW_COMBINE)); + ::testing::Combine(HTTP2SETTINGS_DEFERRED_RESET_COMBINE, + HTTP2SETTINGS_DEFERRED_RESET_COMBINE)); // we seperate default/edge cases here to avoid combinatorial explosion #define HTTP2SETTINGS_DEFAULT_COMBINE \ ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ ::testing::Values(Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE)) + ::testing::Values(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE), \ + ::testing::Values(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT)) INSTANTIATE_TEST_CASE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, @@ -439,7 +451,8 @@ INSTANTIATE_TEST_CASE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, ::testing::Values(Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE, \ Http2Settings::MAX_INITIAL_STREAM_WINDOW_SIZE), \ ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE, \ - Http2Settings::MAX_INITIAL_CONNECTION_WINDOW_SIZE)) + Http2Settings::MAX_INITIAL_CONNECTION_WINDOW_SIZE), \ + ::testing::Values(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT)) INSTANTIATE_TEST_CASE_P(Http2CodecImplTestEdgeSettings, Http2CodecImplTest, ::testing::Combine(HTTP2SETTINGS_EDGE_COMBINE, HTTP2SETTINGS_EDGE_COMBINE)); diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index 9ef7169e61a75..5412198c22663 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -112,6 +112,8 @@ TEST(HttpUtility, parseHttp2Settings) { http2_settings.initial_stream_window_size_); EXPECT_EQ(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE, http2_settings.initial_connection_window_size_); + EXPECT_EQ(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT, + http2_settings.per_stream_buffer_limit_); } { @@ -120,13 +122,15 @@ TEST(HttpUtility, parseHttp2Settings) { "hpack_table_size": 1, "max_concurrent_streams": 2, "initial_stream_window_size": 3, - "initial_connection_window_size": 4 + "initial_connection_window_size": 4, + "per_stream_buffer_limit": 5 } })raw")); EXPECT_EQ(1U, http2_settings.hpack_table_size_); EXPECT_EQ(2U, http2_settings.max_concurrent_streams_); EXPECT_EQ(3U, http2_settings.initial_stream_window_size_); EXPECT_EQ(4U, http2_settings.initial_connection_window_size_); + EXPECT_EQ(5U, http2_settings.per_stream_buffer_limit_); } { @@ -140,6 +144,8 @@ TEST(HttpUtility, parseHttp2Settings) { http2_settings.initial_stream_window_size_); EXPECT_EQ(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE, http2_settings.initial_connection_window_size_); + EXPECT_EQ(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT, + http2_settings.per_stream_buffer_limit_); } { diff --git a/test/config/integration/server_http2.json b/test/config/integration/server_http2.json index 8bb0278b5ed94..49cd1be1c4fa6 100644 --- a/test/config/integration/server_http2.json +++ b/test/config/integration/server_http2.json @@ -183,6 +183,9 @@ "name": "http_connection_manager", "config": { "codec_type": "http2", + "http2_settings": { + "per_stream_buffer_limit": 1024 + }, "drain_timeout_ms": 5000, "access_log": [ { diff --git a/test/config/integration/server_http2_upstream.json b/test/config/integration/server_http2_upstream.json index fe93afa35d1c6..cf434a0fe8db7 100644 --- a/test/config/integration/server_http2_upstream.json +++ b/test/config/integration/server_http2_upstream.json @@ -263,6 +263,9 @@ "name": "http_connection_manager", "config": { "codec_type": "http2", + "http2_settings": { + "per_stream_buffer_limit": 1024 + }, "drain_timeout_ms": 5000, "access_log": [ { diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index fe353e86a814c..42ae79ceaef97 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -160,6 +160,7 @@ const uint32_t Http2Settings::DEFAULT_HPACK_TABLE_SIZE; const uint32_t Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS; const uint32_t Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE; const uint32_t Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE; +const uint32_t Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT; TestHeaderMapImpl::TestHeaderMapImpl() : HeaderMapImpl() {} From f0cb2d05ad8a1711e17d0409f03ce801922bf07f Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 26 Jul 2017 15:56:40 -0400 Subject: [PATCH 05/19] ensuring all stream data is consumed on destruction --- source/common/http/http2/codec_impl.cc | 4 +- test/common/http/http2/codec_impl_test.cc | 60 ++++++++++++++++++++--- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 58e3490998c19..aa31b4d2aa6a8 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -60,7 +60,7 @@ ConnectionImpl::StreamImpl::StreamImpl(ConnectionImpl& parent, uint32_t buffer_l } } -ConnectionImpl::StreamImpl::~StreamImpl() {} +ConnectionImpl::StreamImpl::~StreamImpl() { ASSERT(unconsumed_bytes_ == 0); } void ConnectionImpl::StreamImpl::buildHeaders(std::vector& final_headers, const HeaderMap& headers) { @@ -500,6 +500,8 @@ int ConnectionImpl::onStreamClose(int32_t stream_id, uint32_t error_code) { } connection_.dispatcher().deferredDelete(stream->removeFromList(active_streams_)); + nghttp2_session_consume(session_, stream_id, stream->unconsumed_bytes_); + stream->unconsumed_bytes_ = 0; nghttp2_session_set_stream_user_data(session_, stream->stream_id_, nullptr); } diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 1197357c042ec..465af37ad77c3 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -368,14 +368,14 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { // updates to the client. server_.getStream(1)->readDisable(true); - uint32_t initial_window = + uint32_t initial_stream_window = nghttp2_session_get_stream_effective_local_window_size(client_.session(), 1); // If this limit is changed, this test will fail due to the initial large writes being divided // into more than 4 frames. Fast fail here with this explanatory comment. - ASSERT_EQ(65535, initial_window); + ASSERT_EQ(65535, initial_stream_window); // One large write gets broken into smaller frames. EXPECT_CALL(request_decoder_, decodeData(_, false)).Times(AnyNumber()); - Buffer::OwnedImpl long_data(std::string(initial_window, 'a')); + Buffer::OwnedImpl long_data(std::string(initial_stream_window, 'a')); // The one giant write will cause the buffer to go over the limit, then drain and go back under // the limit. EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); @@ -386,7 +386,7 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { // stream. EXPECT_EQ(0, nghttp2_session_get_stream_local_window_size(server_.session(), 1)); EXPECT_EQ(0, nghttp2_session_get_stream_remote_window_size(client_.session(), 1)); - EXPECT_EQ(initial_window, server_.getStream(1)->unconsumed_bytes_); + EXPECT_EQ(initial_stream_window, server_.getStream(1)->unconsumed_bytes_); // Now that the flow control window is full, further data causes the send buffer to back up. Buffer::OwnedImpl ten_bytes("0123456789"); @@ -406,12 +406,60 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { EXPECT_EQ(0, client_.getStream(1)->pending_send_data_.length()); // The 11 bytes sent won't trigger another window update, so the final window should be the // initial window minus the last 11 byte flush from the client to server. - EXPECT_EQ(initial_window - (SMALL_WINDOW + 1), + EXPECT_EQ(initial_stream_window - (SMALL_WINDOW + 1), nghttp2_session_get_stream_local_window_size(server_.session(), 1)); - EXPECT_EQ(initial_window - (SMALL_WINDOW + 1), + EXPECT_EQ(initial_stream_window - (SMALL_WINDOW + 1), nghttp2_session_get_stream_remote_window_size(client_.session(), 1)); } +// Set up the same asTestFlowControlInPendingSendData, but tears the stream down with an early reset +// once the flow control window is full up. +TEST_P(Http2CodecImplFlowControlTest, EarlyResetRestoresWindow) { + initialize(); + MockStreamCallbacks callbacks; + request_encoder_->getStream().addCallbacks(callbacks); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + TestHeaderMapImpl expected_headers; + HttpTestUtility::addDefaultHeaders(expected_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(HeaderMapEqual(&expected_headers), false)); + request_encoder_->encodeHeaders(request_headers, false); + + // Force the server stream to be read disabled. This will cause it to stop sending window + // updates to the client. + server_.getStream(1)->readDisable(true); + + uint32_t initial_stream_window = + nghttp2_session_get_stream_effective_local_window_size(client_.session(), 1); + uint32_t initial_connection_window = nghttp2_session_get_remote_window_size(client_.session()); + // If this limit is changed, this test will fail due to the initial large writes being divided + // into more than 4 frames. Fast fail here with this explanatory comment. + ASSERT_EQ(65535, initial_stream_window); + // One large write gets broken into smaller frames. + EXPECT_CALL(request_decoder_, decodeData(_, false)).Times(AnyNumber()); + Buffer::OwnedImpl long_data(std::string(initial_stream_window, 'a')); + // The one giant write will cause the buffer to go over the limit, then drain and go back under + // the limit. + EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); + EXPECT_CALL(callbacks, onBelowWriteBufferLowWatermark()); + request_encoder_->encodeData(long_data, false); + + // Verify that the window is full. The client will not send more data to the server for this + // stream. + EXPECT_EQ(0, nghttp2_session_get_stream_local_window_size(server_.session(), 1)); + EXPECT_EQ(0, nghttp2_session_get_stream_remote_window_size(client_.session(), 1)); + EXPECT_EQ(initial_stream_window, server_.getStream(1)->unconsumed_bytes_); + EXPECT_GT(initial_connection_window, nghttp2_session_get_remote_window_size(client_.session())); + + EXPECT_CALL(server_stream_callbacks_, onResetStream(StreamResetReason::LocalRefusedStreamReset)); + EXPECT_CALL(callbacks, onResetStream(StreamResetReason::RemoteRefusedStreamReset)); + response_encoder_->getStream().resetStream(StreamResetReason::LocalRefusedStreamReset); + + // Regression test that the window is consumed even if the stream is destroyed early. + EXPECT_EQ(initial_connection_window, nghttp2_session_get_remote_window_size(client_.session())); +} + #define HTTP2SETTINGS_DEFERRED_RESET_COMBINE \ ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ From 7bf8073e2fae9ff94d928eec9eaded4b3d6a8bc9 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 31 Jul 2017 11:11:29 -0400 Subject: [PATCH 06/19] asserts and rough docs --- source/common/http/http2/codec_impl.cc | 8 ++ source/common/http/http2/codec_impl.h | 2 + source/docs/flow_control.md | 103 +++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 source/docs/flow_control.md diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 56e4da0e4d3a7..9a5dc89f18a55 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -148,21 +148,29 @@ void ConnectionImpl::StreamImpl::readDisable(bool disable) { void ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() { ENVOY_CONN_LOG(debug, "recv buffer over limit ", parent_.connection_); + ASSERT(pending_receive_buffer_high_watermark_called_ == false); + pending_receive_buffer_high_watermark_called_ = true; readDisable(true); } void ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark() { ENVOY_CONN_LOG(debug, "recv buffer under limit ", parent_.connection_); + ASSERT(pending_receive_buffer_high_watermark_called_ == true); + pending_receive_buffer_high_watermark_called_ = false; readDisable(false); } void ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() { ENVOY_CONN_LOG(debug, "send buffer over limit ", parent_.connection_); + ASSERT(pending_send_buffer_high_watermark_called_ == false); + pending_send_buffer_high_watermark_called_ = true; runHighWatermarkCallbacks(); } void ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark() { ENVOY_CONN_LOG(debug, "send buffer under limit ", parent_.connection_); + ASSERT(pending_send_buffer_high_watermark_called_ == true); + pending_send_buffer_high_watermark_called_ = false; runLowWatermarkCallbacks(); } diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index f123b47ac3ef5..8af0e130f0809 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -190,6 +190,8 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable StreamImplPtr; diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md new file mode 100644 index 0000000000000..672397d3f1a9f --- /dev/null +++ b/source/docs/flow_control.md @@ -0,0 +1,103 @@ +### Overview + +Flow control in envoy is done by having limits on each buffer, and watermark callbacks. When a +buffer contains more data than the configured limit, the high watermark callback will fire, kicking +off a chain of events which eventually informs the data source to stop sending data. This back-off +may be immediate (stop reading from a socket) or gradual (stop HTTP/2 window updates) so all +buffer limits in Envoy are considered soft limits. When the buffer eventually drains (generally to +half of of the high watermark to avoid thrashing back and forth) the low watermark callback will +fire, informing the sender it can resume sending data. + +### TCP implementation details + +TODO(alyssawilk) document the existing connection level flow control and configuration. + +### HTTP2 implementation details + +Because the various buffers in the HTTP/2 stack are fairly complicated, each path from a buffer +going over the watermark limit to disabling data from the data source is documented separately. + +TODO(alyssawilk) snag diagram from H2 flow control google doc for various components. + +For HTTP/2, when filters, streams, or connections back up, the end result is readDisable(true) +being called on the source stream. This results in the stream ceasing to consume window, and so +not sending further flow control window updates to the peer. This will result in the peer +eventually stopping sending data when the available window is consumed (or nghttp2 closing the +connection if the peer violates the flow control limit). When readDisable(false) is called, any +outstanding unconsumed data is immediately consumed, which results in resuming window updates to the +peer and the resumption of data. + +Note that readDisable() on the stream may be called by multiple entities. It is called when any +filter buffers too much, when the upstream stream backs up and has too much data buffered, or the +upstream connection has too much data buffered. Because of this, readDisable() manitains a count of +the number of times it has been disabled and only resumes reads when each caller has called the +equiavlent low watermark callback. + +## HTTP/2 codec recv buffer + +Given the HTTP/2 Envoy::Http::Http2 pending_recv_data\_ is processed immediately there's no real +need for buffer limits, but for consistency and to future-proof the implementation, it is a +WatermarkBuffer. When the high watermark triggers, it calls +ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() which calls readDisable(true) on the +stream. + +When the buffer is drained, the WatermarkBuffer calls +ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark which calls readDisable(false) on the +stream. + +## HTTP/2 filters + +TODO(alyssawilk) implement and document. + +# HTTP/2 codec upstream send buffer + +The upstream send buffer Envoy::Http::Http pending_send_data\_ is H2 stream data destined for an +Envoy backend. Data is added to this buffer after each filter in the chain is done processing, +and it backs up if there is insufficient connection or stream window to send the data. When this +buffer backs up, the Watermark buffer calls +ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() which calls +StreamCallbackHelper::runHighWatermarkCallbacks(). This results in all subscribers of the +StreamCallbacks receiving onAboveWriteBufferHighWatermark() callback. The key subscriber in this +case is Envoy::Router::Filter. When the router filter receives the high +watermark callback it in turn calls +StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark(). +This is picked up by Envoy::Http::ConnectionManagerImpl which can finally +call readDisable(true) on the downstream stream to stop data flowing from the downstream peer. + +The low watermark path is the same. When pending_send_data\_ is drained, the +Watermark buffer calls ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark, which calls +StreamCallbackHelper::runLowWatermarkCallbacks. This +kicks off StreamCallbacks::onAboveWriteBufferHighWatermark() which are picked up +the the router filter, which in turn calls +StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark which +causes the Envoy::Http::ConnectionManagerImpl to call readDisable(false) on the +downstream stream to resume the flow of data. + +# HTTP/2 network upstream network buffer + +The upstream network buffer is HTTP/2 data for all streams destined for the +Envoy backend. When the Network::Connection has too much data buffered in write_buffer\_ it calls +the Network::ConnectionCallbacks onAboveWriteBufferHighWatermark(). + +The Envoy::Http::CodecClient is subscribed to this callback. It informs the +Envoy::Http::Http2::ConnectionImpl of the event via ClientConnection::onAboveWriteBufferHighWatermark() + +When the Envoy::Http::Http2::ConnectionImpl receives the high watermark callback it calls +runHighWatermarkCallbacks() for each stream on that connection. From there on, +the code path is the same as the HTTP/2 codec upstream send buffer path above. +The stream calls the stream callbacks, the router receives them and passes them +to the connection manager which disables reads on the stream. + +As always the low watermark path is the same. The Network::Connection calls +onBelowWriteBufferLowWatermark() the Envoy::Http::CodecClient passes it to the codec via +ClientConnection::onBelowWriteBufferLowWatermark(), the Envoy::Http::Http2::ConnectionImpl passes the +change on to each stream, and so on. + +# HTTP/2 codec downstream send buffer +# HTTP/2 network upstream network buffer + +TODO(alyssawilk) implement and document. + +### HTTP implementation details + +TODO(alyssawilk) implement and document. From d6dbeda2dd723fed3d6e0d1c7d1d37da3450578f Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 31 Jul 2017 15:09:13 -0400 Subject: [PATCH 07/19] Replacing router stats with cluster stats --- docs/configuration/cluster_manager/cluster_stats.rst | 2 ++ docs/configuration/http_filters/router_filter.rst | 2 -- include/envoy/upstream/upstream.h | 2 ++ source/common/config/protocol_json.cc | 3 +-- source/common/http/utility.cc | 5 ++--- source/common/router/router.h | 6 ++---- test/common/router/router_test.cc | 8 ++++++-- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/configuration/cluster_manager/cluster_stats.rst b/docs/configuration/cluster_manager/cluster_stats.rst index 3b10bba64433a..f47d7135fb4d4 100644 --- a/docs/configuration/cluster_manager/cluster_stats.rst +++ b/docs/configuration/cluster_manager/cluster_stats.rst @@ -55,6 +55,8 @@ Every cluster has a statistics tree rooted at *cluster..* with the followi upstream_rq_retry_overflow, Counter, Total requests not retried due to circuit breaking upstream_flow_control_paused_reading_total, Counter, Total number of times flow control paused reading from upstream. upstream_flow_control_resumed_reading_total, Counter, Total number of times flow control resumed reading from upstream. + upstream_flow_control_backed_up_total, Counter, Total number of times the upstream connection backed up, pausing reads from downstream. + upstream_flow_control_drained_total, Counter, Total number of times the upstream connection drained, resuming reads from downstream. membership_change, Counter, Total cluster membership changes membership_healthy, Gauge, Current cluster healthy total (inclusive of both health checking and outlier detection) membership_total, Gauge, Current cluster membership total diff --git a/docs/configuration/http_filters/router_filter.rst b/docs/configuration/http_filters/router_filter.rst index d79e2f3593943..be8db6bf3bfb0 100644 --- a/docs/configuration/http_filters/router_filter.rst +++ b/docs/configuration/http_filters/router_filter.rst @@ -219,8 +219,6 @@ prefix ` comes from the owning HTTP connection no_cluster, Counter, Total requests in which the target cluster did not exist and resulted in a 404 rq_redirect, Counter, Total requests that resulted in a redirect response rq_total, Counter, Total routed requests - flow_control_paused_downstream_reads_total, Counter, Total times requests backed up enough to pause reading from downstream. - flow_control_resumed_downstream_reads_total, Counter, Total number of times requests resumed reading from downstream. Virtual cluster statistics are output in the *vhost..vcluster..* namespace and include the following diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index b5699dba9b8a2..f9f0aa9c2e135 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -205,6 +205,8 @@ class HostSet { COUNTER(upstream_rq_retry_overflow) \ COUNTER(upstream_flow_control_paused_reading_total) \ COUNTER(upstream_flow_control_resumed_reading_total) \ + COUNTER(upstream_flow_control_backed_up_total) \ + COUNTER(upstream_flow_control_drained_total) \ GAUGE (max_host_weight) \ COUNTER(membership_change) \ GAUGE (membership_healthy) \ diff --git a/source/common/config/protocol_json.cc b/source/common/config/protocol_json.cc index 73b4763d8e049..f5a694241c182 100644 --- a/source/common/config/protocol_json.cc +++ b/source/common/config/protocol_json.cc @@ -13,8 +13,7 @@ void ProtocolJson::translateHttp2ProtocolOptions( JSON_UTIL_SET_INTEGER(json_http2_settings, http2_protocol_options, initial_stream_window_size); JSON_UTIL_SET_INTEGER(json_http2_settings, http2_protocol_options, initial_connection_window_size); - JSON_UTIL_SET_INTEGER(json_http2_settings, http2_protocol_options, - per_stream_buffer_limit_bytes); + JSON_UTIL_SET_INTEGER(json_http2_settings, http2_protocol_options, per_stream_buffer_limit_bytes); if (json_http_codec_options == "no_compression") { if (http2_protocol_options.hpack_table_size().value() != 0) { throw EnvoyException( diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index 1940df40051ad..601b6e946393b 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -160,9 +160,8 @@ Http2Settings Utility::parseHttp2Settings(const envoy::api::v2::Http2ProtocolOpt ret.initial_connection_window_size_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, initial_connection_window_size, Http::Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE); - ret.per_stream_buffer_limit_ = - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, per_stream_buffer_limit_bytes, - Http::Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT); + ret.per_stream_buffer_limit_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT( + config, per_stream_buffer_limit_bytes, Http::Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT); return ret; } diff --git a/source/common/router/router.h b/source/common/router/router.h index f20512e2079bd..f2047eebe878f 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -24,8 +24,6 @@ namespace Router { */ // clang-format off #define ALL_ROUTER_STATS(COUNTER) \ - COUNTER(flow_control_paused_downstream_reads_total) \ - COUNTER(flow_control_resumed_downstream_reads_total) \ COUNTER(no_route) \ COUNTER(no_cluster) \ COUNTER(rq_redirect) \ @@ -169,12 +167,12 @@ class Filter : Logger::Loggable, void onResetStream(Http::StreamResetReason reason) override; void onAboveWriteBufferHighWatermark() override { // Have the connection manager disable reads on the downstream stream. - parent_.config_.stats_.flow_control_paused_downstream_reads_total_.inc(); + parent_.cluster_->stats().upstream_flow_control_backed_up_total_.inc(); parent_.callbacks_->onDecoderFilterAboveWriteBufferHighWatermark(); } void onBelowWriteBufferLowWatermark() override { // Have the connection manager enable reads on the downstream stream. - parent_.config_.stats_.flow_control_resumed_downstream_reads_total_.inc(); + parent_.cluster_->stats().upstream_flow_control_drained_total_.inc(); parent_.callbacks_->onDecoderFilterBelowWriteBufferLowWatermark(); } diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 1e18f3ed70d30..e1afe7bd91736 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1168,9 +1168,13 @@ TEST_F(RouterTest, Watermarks) { router_.decodeHeaders(headers, true); stream_callbacks->onAboveWriteBufferHighWatermark(); - EXPECT_EQ(1UL, stats_store_.counter("test.flow_control_paused_downstream_reads_total").value()); + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("upstream_flow_control_backed_up_total") + .value()); stream_callbacks->onBelowWriteBufferLowWatermark(); - EXPECT_EQ(1UL, stats_store_.counter("test.flow_control_resumed_downstream_reads_total").value()); + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("upstream_flow_control_drained_total") + .value()); Http::HeaderMapPtr response_headers( new Http::TestHeaderMapImpl{{":status", "200"}, From 1f3d7cc753b262c45892fc0657bb1e5108c34082 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 31 Jul 2017 15:21:44 -0400 Subject: [PATCH 08/19] bullet points --- source/docs/flow_control.md | 104 ++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 41 deletions(-) diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md index 672397d3f1a9f..0fb07b84a2b94 100644 --- a/source/docs/flow_control.md +++ b/source/docs/flow_control.md @@ -37,13 +37,17 @@ equiavlent low watermark callback. Given the HTTP/2 Envoy::Http::Http2 pending_recv_data\_ is processed immediately there's no real need for buffer limits, but for consistency and to future-proof the implementation, it is a -WatermarkBuffer. When the high watermark triggers, it calls -ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() which calls readDisable(true) on the -stream. +WatermarkBuffer. The high watermark path goes as follows: -When the buffer is drained, the WatermarkBuffer calls -ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark which calls readDisable(false) on the -stream. + * When pending_recv_data\_ has too much data, the WatermarkBuffer calls + ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() + * pendingRecvBufferHighWatermark calls readDisable(true) on the stream. + +The low watermark path is similar + + * When pending_recv_data\_ is drained, the WatermarkBuffer calls + ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark + * pendingRecvBufferLowWatermarkwhich calls readDisable(false) on the stream. ## HTTP/2 filters @@ -53,45 +57,63 @@ TODO(alyssawilk) implement and document. The upstream send buffer Envoy::Http::Http pending_send_data\_ is H2 stream data destined for an Envoy backend. Data is added to this buffer after each filter in the chain is done processing, -and it backs up if there is insufficient connection or stream window to send the data. When this -buffer backs up, the Watermark buffer calls -ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() which calls -StreamCallbackHelper::runHighWatermarkCallbacks(). This results in all subscribers of the -StreamCallbacks receiving onAboveWriteBufferHighWatermark() callback. The key subscriber in this -case is Envoy::Router::Filter. When the router filter receives the high -watermark callback it in turn calls -StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark(). -This is picked up by Envoy::Http::ConnectionManagerImpl which can finally -call readDisable(true) on the downstream stream to stop data flowing from the downstream peer. - -The low watermark path is the same. When pending_send_data\_ is drained, the -Watermark buffer calls ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark, which calls -StreamCallbackHelper::runLowWatermarkCallbacks. This -kicks off StreamCallbacks::onAboveWriteBufferHighWatermark() which are picked up -the the router filter, which in turn calls -StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark which -causes the Envoy::Http::ConnectionManagerImpl to call readDisable(false) on the -downstream stream to resume the flow of data. +and it backs up if there is insufficient connection or stream window to send the data. The high +watermark path goes as follows: + + * When pending_send_data\_ has too much data it calls + ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() + * pendingSendBufferHighWatermark() calls StreamCallbackHelper::runHighWatermarkCallbacks() + * runHighWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a + onAboveWriteBufferHighWatermark() callback. + * When Envoy::Router::Filter receives onAboveWriteBufferHighWatermark() it + calls StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark(). + * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterAboveWriteBufferHighWatermark() + it calls readDisable(true) on the downstream stream to pause data. + +For the low watermark path: + + * When pending_send_data\_ drains it calls + ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark() + * pendingSendBufferLowWatermark() calls StreamCallbackHelper::runLowWatermarkCallbacks() + * runLowWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a + onBelowWriteBufferLowWatermark() callback. + * When Envoy::Router::Filter receives onBelowWriteBufferLowWatermark() it + calls StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark(). + * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterBelowWriteBufferLowWatermark() + it calls readDisable(false) on the downstream stream to resume data. # HTTP/2 network upstream network buffer The upstream network buffer is HTTP/2 data for all streams destined for the -Envoy backend. When the Network::Connection has too much data buffered in write_buffer\_ it calls -the Network::ConnectionCallbacks onAboveWriteBufferHighWatermark(). - -The Envoy::Http::CodecClient is subscribed to this callback. It informs the -Envoy::Http::Http2::ConnectionImpl of the event via ClientConnection::onAboveWriteBufferHighWatermark() - -When the Envoy::Http::Http2::ConnectionImpl receives the high watermark callback it calls -runHighWatermarkCallbacks() for each stream on that connection. From there on, -the code path is the same as the HTTP/2 codec upstream send buffer path above. -The stream calls the stream callbacks, the router receives them and passes them -to the connection manager which disables reads on the stream. - -As always the low watermark path is the same. The Network::Connection calls -onBelowWriteBufferLowWatermark() the Envoy::Http::CodecClient passes it to the codec via -ClientConnection::onBelowWriteBufferLowWatermark(), the Envoy::Http::Http2::ConnectionImpl passes the -change on to each stream, and so on. +Envoy backend. The high watermark path is as follows: + + * When Envoy::Network::ConnectionImpl write_buffer\_ has too much data it calls + Network::ConnectionCallbacks::onAboveWriteBufferHighWatermark(). + * When Envoy::Http::CodecClient receives onAboveWriteBufferHighWatermark() it + calls onAboveWriteBufferHighWatermark() on codec\_. + * When Envoy::Http::Http2::ConnectionImpl receives onAboveWriteBufferHighWatermark() it calls + runHighWatermarkCallbacks() for each stream of the connection. + * runHighWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a + onAboveWriteBufferHighWatermark() callback. + * When Envoy::Router::Filter receives onAboveWriteBufferHighWatermark() it + calls StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark(). + * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterAboveWriteBufferHighWatermark() + it calls readDisable(true) on the downstream stream to pause data. + +The low watermark path is as follows: + + * When Envoy::Network::ConnectionImpl write_buffer\_ is drained it calls + Network::ConnectionCallbacks::onBelowWriteBufferLowWatermark(). + * When Envoy::Http::CodecClient receives onBelowWriteBufferLowWatermark() it + calls onBelowWriteBufferLowWatermark() on codec\_. + * When Envoy::Http::Http2::ConnectionImpl receives onBelowWriteBufferLowWatermark() it calls + runLowWatermarkCallbacks() for each stream of the connection. + * runLowWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a + onBelowWriteBufferLowWatermark() callback. + * When Envoy::Router::Filter receives onBelowWriteBufferLowWatermark() it + calls StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark(). + * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterBelowWriteBufferLowWatermark() + it calls readDisable(false) on the downstream stream to resume data. # HTTP/2 codec downstream send buffer # HTTP/2 network upstream network buffer From 0449209833be788635927db762a31e4cd12b0ac4 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 31 Jul 2017 15:33:33 -0400 Subject: [PATCH 09/19] Bullets and backticks --- source/docs/flow_control.md | 114 ++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 56 deletions(-) diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md index 0fb07b84a2b94..2b0b8cb1cc4f0 100644 --- a/source/docs/flow_control.md +++ b/source/docs/flow_control.md @@ -35,19 +35,19 @@ equiavlent low watermark callback. ## HTTP/2 codec recv buffer -Given the HTTP/2 Envoy::Http::Http2 pending_recv_data\_ is processed immediately there's no real -need for buffer limits, but for consistency and to future-proof the implementation, it is a -WatermarkBuffer. The high watermark path goes as follows: +Given the HTTP/2 `Envoy::Http::Http2::ConnectionImpl::StreamImpl::pending_recv_data_` is processed immediately +there's no real need for buffer limits, but for consistency and to future-proof the implementation, +it is a WatermarkBuffer. The high watermark path goes as follows: - * When pending_recv_data\_ has too much data, the WatermarkBuffer calls - ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() - * pendingRecvBufferHighWatermark calls readDisable(true) on the stream. + * When `pending_recv_data_` has too much data it calls + `ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark()`. + * `pendingRecvBufferHighWatermark` calls `readDisable(true)` on the stream. The low watermark path is similar - * When pending_recv_data\_ is drained, the WatermarkBuffer calls - ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark - * pendingRecvBufferLowWatermarkwhich calls readDisable(false) on the stream. + * When `pending_recv_data_` is drained, it calls + `ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark`. + * `pendingRecvBufferLowWatermarkwhich` calls `readDisable(false)` on the stream. ## HTTP/2 filters @@ -55,65 +55,67 @@ TODO(alyssawilk) implement and document. # HTTP/2 codec upstream send buffer -The upstream send buffer Envoy::Http::Http pending_send_data\_ is H2 stream data destined for an -Envoy backend. Data is added to this buffer after each filter in the chain is done processing, -and it backs up if there is insufficient connection or stream window to send the data. The high -watermark path goes as follows: - - * When pending_send_data\_ has too much data it calls - ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() - * pendingSendBufferHighWatermark() calls StreamCallbackHelper::runHighWatermarkCallbacks() - * runHighWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a - onAboveWriteBufferHighWatermark() callback. - * When Envoy::Router::Filter receives onAboveWriteBufferHighWatermark() it - calls StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark(). - * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterAboveWriteBufferHighWatermark() - it calls readDisable(true) on the downstream stream to pause data. +The upstream send buffer `Envoy::Http::Http2::ConnectionImpl::StreamImpl::pending_send_data\_` is +H2 stream data destined for an Envoy backend. Data is added to this buffer after each filter in +the chain is done processing, and it backs up if there is insufficient connection or stream window +to send the data. The high watermark path goes as follows: + + * When `pending_send_data_` has too much data it calls + `ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark()`. + * `pendingSendBufferHighWatermark()` calls `StreamCallbackHelper::runHighWatermarkCallbacks()` + * `runHighWatermarkCallbacks()` results in all subscribers of `Envoy::Http::StreamCallbacks` + receiving an `onAboveWriteBufferHighWatermark()` callback. + * When `Envoy::Router::Filter` receives `onAboveWriteBufferHighWatermark()` it + calls `StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark()`. + * When `Envoy::Http::ConnectionManagerImpl` receives + `onDecoderFilterAboveWriteBufferHighWatermark()` it calls `readDisable(true)` on the downstream + stream to pause data. For the low watermark path: - * When pending_send_data\_ drains it calls - ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark() - * pendingSendBufferLowWatermark() calls StreamCallbackHelper::runLowWatermarkCallbacks() - * runLowWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a - onBelowWriteBufferLowWatermark() callback. - * When Envoy::Router::Filter receives onBelowWriteBufferLowWatermark() it - calls StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark(). - * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterBelowWriteBufferLowWatermark() - it calls readDisable(false) on the downstream stream to resume data. + * When `pending_send_data_` drains it calls + `ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark()` + * `pendingSendBufferLowWatermark()` calls `StreamCallbackHelper::runLowWatermarkCallbacks()` + * `runLowWatermarkCallbacks()` results in all subscribers of `Envoy::Http::StreamCallbacks` + receiving a `onBelowWriteBufferLowWatermark()` callback. + * When `Envoy::Router::Filter` receives `onBelowWriteBufferLowWatermark()` it + calls `StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark()`. + * When `Envoy::Http::ConnectionManagerImpl` receives `onDecoderFilterBelowWriteBufferLowWatermark()` + it calls `readDisable(false)` on the downstream stream to resume data. # HTTP/2 network upstream network buffer The upstream network buffer is HTTP/2 data for all streams destined for the Envoy backend. The high watermark path is as follows: - * When Envoy::Network::ConnectionImpl write_buffer\_ has too much data it calls - Network::ConnectionCallbacks::onAboveWriteBufferHighWatermark(). - * When Envoy::Http::CodecClient receives onAboveWriteBufferHighWatermark() it - calls onAboveWriteBufferHighWatermark() on codec\_. - * When Envoy::Http::Http2::ConnectionImpl receives onAboveWriteBufferHighWatermark() it calls - runHighWatermarkCallbacks() for each stream of the connection. - * runHighWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a - onAboveWriteBufferHighWatermark() callback. - * When Envoy::Router::Filter receives onAboveWriteBufferHighWatermark() it - calls StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark(). - * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterAboveWriteBufferHighWatermark() - it calls readDisable(true) on the downstream stream to pause data. + * When `Envoy::Network::ConnectionImpl::write_buffer_` has too much data it calls + `Network::ConnectionCallbacks::onAboveWriteBufferHighWatermark()`. + * When `Envoy::Http::CodecClient` receives `onAboveWriteBufferHighWatermark()` it + calls `onAboveWriteBufferHighWatermark()` on `codec_`. + * When `Envoy::Http::Http2::ConnectionImpl` receives `onAboveWriteBufferHighWatermark()` it calls + `runHighWatermarkCallbacks()` for each stream of the connection. + * `runHighWatermarkCallbacks()` results in all subscribers of `Envoy::Http::StreamCallback` + receiving an `onAboveWriteBufferHighWatermark()` callback. + * When `Envoy::Router::Filter` receives `onAboveWriteBufferHighWatermark()` it + calls `StreamDecoderFilterCallback::onDecoderFilterAboveWriteBufferHighWatermark()`. + * When `Envoy::Http::ConnectionManagerImpl` receives + `onDecoderFilterAboveWriteBufferHighWatermark()` it calls `readDisable(true)` on the downstream + stream to pause data. The low watermark path is as follows: - * When Envoy::Network::ConnectionImpl write_buffer\_ is drained it calls - Network::ConnectionCallbacks::onBelowWriteBufferLowWatermark(). - * When Envoy::Http::CodecClient receives onBelowWriteBufferLowWatermark() it - calls onBelowWriteBufferLowWatermark() on codec\_. - * When Envoy::Http::Http2::ConnectionImpl receives onBelowWriteBufferLowWatermark() it calls - runLowWatermarkCallbacks() for each stream of the connection. - * runLowWatermarkCallbacks() results in all subscribers of StreamCallbacks receiving a - onBelowWriteBufferLowWatermark() callback. - * When Envoy::Router::Filter receives onBelowWriteBufferLowWatermark() it - calls StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark(). - * When Envoy::Http::ConnectionManagerImpl receives onDecoderFilterBelowWriteBufferLowWatermark() - it calls readDisable(false) on the downstream stream to resume data. + * When `Envoy::Network::ConnectionImpl::write_buffer_` is drained it calls + `Network::ConnectionCallbacks::onBelowWriteBufferLowWatermark()`. + * When `Envoy::Http::CodecClient` receives `onBelowWriteBufferLowWatermark()` it + calls `onBelowWriteBufferLowWatermark()` on `codec_`. + * When `Envoy::Http::Http2::ConnectionImpl` receives `onBelowWriteBufferLowWatermark()` it calls + `runLowWatermarkCallbacks()` for each stream of the connection. + * `runLowWatermarkCallbacks()` results in all subscribers of `Envoy::Http::StreamCallback` + receiving a `onBelowWriteBufferLowWatermark()` callback. + * When `Envoy::Router::Filter` receives `onBelowWriteBufferLowWatermark()` it + calls `StreamDecoderFilterCallback::onDecoderFilterBelowWriteBufferLowWatermark()`. + * When `Envoy::Http::ConnectionManagerImpl` receives `onDecoderFilterBelowWriteBufferLowWatermark()` + it calls `readDisable(false)` on the downstream stream to resume data. # HTTP/2 codec downstream send buffer # HTTP/2 network upstream network buffer From 76384ce4885330d5decb8e457c1e35a606315bd4 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 31 Jul 2017 16:14:21 -0400 Subject: [PATCH 10/19] iterative doc and unit test improvement --- .../common/thread_local/thread_local_impl.cc | 4 +- source/docs/flow_control.md | 41 ++++++++++++++---- source/docs/h2_buffers.png | Bin 0 -> 67245 bytes test/common/http/async_client_impl_test.cc | 13 ++++++ 4 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 source/docs/h2_buffers.png diff --git a/source/common/thread_local/thread_local_impl.cc b/source/common/thread_local/thread_local_impl.cc index 59573d1f3e17a..bce154a7811a3 100644 --- a/source/common/thread_local/thread_local_impl.cc +++ b/source/common/thread_local/thread_local_impl.cc @@ -28,13 +28,13 @@ SlotPtr InstanceImpl::allocateSlot() { if (slots_[i] == nullptr) { std::unique_ptr slot(new SlotImpl(*this, i)); slots_[i] = slot.get(); - return std::move(slot); + return slot; } } std::unique_ptr slot(new SlotImpl(*this, slots_.size())); slots_.push_back(slot.get()); - return std::move(slot); + return slot; } ThreadLocalObjectSharedPtr InstanceImpl::SlotImpl::get() { diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md index 2b0b8cb1cc4f0..21c7c94241060 100644 --- a/source/docs/flow_control.md +++ b/source/docs/flow_control.md @@ -17,7 +17,7 @@ TODO(alyssawilk) document the existing connection level flow control and configu Because the various buffers in the HTTP/2 stack are fairly complicated, each path from a buffer going over the watermark limit to disabling data from the data source is documented separately. -TODO(alyssawilk) snag diagram from H2 flow control google doc for various components. +![HTTP2 data flow diagram](h2_buffers.png) For HTTP/2, when filters, streams, or connections back up, the end result is readDisable(true) being called on the source stream. This results in the stream ceasing to consume window, and so @@ -27,11 +27,31 @@ connection if the peer violates the flow control limit). When readDisable(false outstanding unconsumed data is immediately consumed, which results in resuming window updates to the peer and the resumption of data. -Note that readDisable() on the stream may be called by multiple entities. It is called when any -filter buffers too much, when the upstream stream backs up and has too much data buffered, or the -upstream connection has too much data buffered. Because of this, readDisable() manitains a count of -the number of times it has been disabled and only resumes reads when each caller has called the -equiavlent low watermark callback. +Note that readDisable() on a stream may be called by multiple entities. It is called when any +filter buffers too much, when the stream backs up and has too much data buffered, or the +connection has too much data buffered. Because of this, readDisable() maintains a count of +the number of times it has been called to both enable and disable the stream, resumes reads when +each caller has called the equivalent low watermark callback. For example, if +the TCP window upstream fills up and results in the network buffer backing up, +all the streams associated with that connection will readDisable(true) their +downstream data sources. While the HTTP/2 flow control window fills up an +individual stream may use all of the window available and call a second +readDisable() its downstream data source. When the upstream TCP socket drains, +the connection will go below its low watermark and each stream will call +readDisable(false) to resume the flow of data. The stream which had both a +network level block and a H2 flow control block will still not be fully enabled. +Once the upstream peer sends window updates, the stream buffer will drain and +the second readDisable(false) will be called on the downstream data source, +which will finally result in data flowing from downstream again. + +The two main parties involved in flow control are the router filter (`Envoy::Router::Filter`) and +the connection manager (`Envoy::Http::ConnectionManagerImpl`). The router is +responsible for intercepting watermark events for its own buffers, the individual upstream streams +(if codec buffers fill up) and the upstream connection (if the network buffer fills up). It passes +any events to the connection manager, which the ability to call readDisable() to enable and disable +further data from downstream. + +TODO(alyssawilk) document the reverse path. ## HTTP/2 codec recv buffer @@ -55,7 +75,7 @@ TODO(alyssawilk) implement and document. # HTTP/2 codec upstream send buffer -The upstream send buffer `Envoy::Http::Http2::ConnectionImpl::StreamImpl::pending_send_data\_` is +The upstream send buffer `Envoy::Http::Http2::ConnectionImpl::StreamImpl::pending_send_data_` is H2 stream data destined for an Envoy backend. Data is added to this buffer after each filter in the chain is done processing, and it backs up if there is insufficient connection or stream window to send the data. The high watermark path goes as follows: @@ -86,7 +106,12 @@ For the low watermark path: # HTTP/2 network upstream network buffer The upstream network buffer is HTTP/2 data for all streams destined for the -Envoy backend. The high watermark path is as follows: +Envoy backend. If the network buffer fills up, all streams associated with the +underlying TCP connection will be informed of the back-up, and the data sources +(HTTP/2 streams or HTTP connections) feeding into those streams will be +readDisabled. + +The high watermark path is as follows: * When `Envoy::Network::ConnectionImpl::write_buffer_` has too much data it calls `Network::ConnectionCallbacks::onAboveWriteBufferHighWatermark()`. diff --git a/source/docs/h2_buffers.png b/source/docs/h2_buffers.png new file mode 100644 index 0000000000000000000000000000000000000000..b5c22a58a73173bfbb4dbff6c6221f6a0ff84873 GIT binary patch literal 67245 zcmdp81y>zSlpF%V-8DD_f(D16!8N$MyL)i=;0^(TySoPn?(XjHzVq!l`zMw&zsHn6ChUi-7!o`_JOBVl65_%N0PtY}{Dy%20G@gA)vX7AK{^UaD8a(QE^o?h zfG^?f#MK=E0Qv8K-w=skkO=^Q1dtFGRC3EWS@ry;cuovl6gF6mTH^(WZdsDuYbGcp zkR>Y=VmXe+lIpJ~cV$_rDtiX4XgbH?puCosKz-o9CN)lH;eyK2*Y ztxtrkTj1A*cB}od;K{a%OM3LBnlrt29HzRGwAMtTzXxJ{%hhW7{Nv&wQoV8-K6!Er zy+0Zp)Ga#L(NC=l1NH-QS)WXeP1z?UVMB17oPX<9)IF@OmhM+&FS$NGciTbvLm@+x z{diYaRIK$VpZ^@dJWaT-UsaV}L(Jwx}x~e_l?DyLn3A~a&F){PB@r&JP zg8=v#;AyD6#k``71nR;bSp$6vc)<@JAkm8WpPt7umnW+&8$b30qZXa`*fJ00 zwy<{5N>CN%iHa;Xl;mHLd>$PgosyCglv;dRTH5X>OBbk+?$1zL$v2-Zet1T-K064i zLis;s^5KQ5JV=C4R4D4UH%p0Ux9Pu9KZw#jfe}N%BoK;&-`tQ z+j5PkqtDFD{5XYv#_2hBRQ?=_{%2j6b!Ir|!Tl6r0d9X%V8V88;6rR$8a{t#rZA1V zW?@jz7P@~0@(d9q4B|h}%bUCU>r&P7uN)ljnIh}qkuN{SGt79mU9sVBvFf-jjNJZR z8j5|?MuR=leNs5S8LXg6X*h2uH}rZcni zjdgWZizgiyt;@tnxB%EYBxGd%c(5~XPs0Mb^D{=QpHQv|-!!~_i-2>t&qk_XOoLNl zSpW`;OXS{+ZEZ-)Y9 zzZ!DFF8`VvsdJ^YH_Dl9SMet#8FhGJ*pt^}&Kel<&Q4CE$zHsPJayfy-s-HeJFe(q zUKJ`3=Cn*q4|wgC=xV@c$`aGj&OP!?Hj=?#U%Sb%D8!?GT7?1Z&;t96y!qrHPeX=A zW#y3I7WO*LXEQBvmN`3r8mLM+Vmy)1QB)m=1O`DBQ1oBgesd**@;dK?(Fn02blW|# zU(-CJp)g+rgDd+RoIgq7cvhoh#%d4NQX92Xo=;U`d7l3iEN;*ZmJnwfyolEn>%c6Z zpXCd3eVw5e4nufkB&JjQn{+!bDwos2gq<_*&}VHhBAWVvx24(74!-`~6x zuW2&RF`QYpR?}763R4;5vin6L0rpA_rC-S(0ob)>uEW9J3LJcF^n*_OSv%uddu+vQ z>TfRDrUN3sY?M1olepFMP)cZrTHM5?9H$S-pTFNfT{U3t?GqtfiLF1MPfo2keKnHf zK#t^eEE|A~ja>?6)MdZKF#0Sz$HrQ& zR{i0zZ+*+VNR@*X3k$2Aflht5R(Jx-&%60D-U|7A?W&{BsVQ!%lG8_4b}W)|ZI!dC zWb3=T`d}+ZB}M+=dBi&0rX3jaUi8Ycg3Of6(Ij*R++!aC`Wjd{60zH-XIQMvZD>cv zS(uo=(9(s%@?+o9;bhWUu}@kQ$2;q|v{d9sfQ=g7fdYId)g0xSFO&8PO0K@j+Z*gf z{E1C#p~Z9tx=PEOxPtV}2jSwTjPjWG%#6d~5!Tvd1P~w#b3e{-j|c zY7yrbmU=2}>I9=f#7560sFV>p7q23=YsXoqVPQVV^EzqbaMcR{MtEF+fV+7ZILLs& zOF(PCA@%PNM>Gl^UJz{|)m=VoMD@IU*J##*xu#zkVS_zdK!u_v==1&x@+r zq1i26IgvpgKL-zggb!!81^$!U>s0eRV@3(>yB+5Q2ZbI1K(gRd8!~+J(pF}<-`h~3 z4iAKOU{&k)`IlUwjAIawrx}fi`~nBIkj4-UB8Dt@?XSz+VBGc@amx|B)4d7mduK(w z{+-%#y>+fXP99rr?d3V@H`Z}zPWanISzA6%2sbn=>6*pvJzNsh7xb~Uv8)-(zM-RH zZgp{PemJww$i_|=)$2SNwhx0F%D7Lc_+W=54G6ulE(h81nP;QIP3nHji$}&@iI( z2zcyZVp^MW6Qrc@Mn$nRG3S|?K>#MBu8t|&AAl4-!A>TAImPh{s~vK{w;*ftHQ^^9 zAcf)8RCAOpq6Noq!(sV$<*W$+xdvOwvOyZs^)LWo>jMXL1Q8uwT++>qRj#s>qhi`)RppVh*yUP2R~uPG&(ZiPDfRXX1OBe*at&x$ z7-Hj*+@IPkEjRS1FOvXYq%#BqoAoX|W~0TNR+LO=U}jz(9Pq;!c1S>Nk=sVCT5s?O z>7%H}v-`b>ugj`$wa<7@to%N)D9(fRq=O zh0JXJw3v$Zf}b}Z12P5#0Op9#_!>NSL>~b}MFl=b3bXus7x1e35E`(lZEmmt35^Ug zw>uob)TcITzUcryp~F@9;vr4JHa%LP=f#aOIGtZ74d?07`Ydk`?Npxw*OFTNr#o*Pes8d=9T_kXk>2GqEwPS?_SLYGUF$jK|A} z^{MvJoc2j|ma97g0^EtwpP=YXvm3wQhBXjACk_I$i#V(6ihlVZwEI!fQt7DuYMgN- zZxQjiS*ZH%hpl+3{Q2l$R!TuK%@xx_mDi040{Ee;I=pF@Q5f}ISBS?K9^tC)1`z@f z;(Tc-*L|#MWArY(kz@KJBwwhaIz)p=K0K78IbjAF$~ z8Js#GLU39&{JGh0{dq}>4gg@|o&H5c`wjLf<^+9o*9weaC*R#S;o@xm&678&rZ2cw z+XTa2fDcNN5Ed;FEI=SB_W4RSKvf?RI)JZVkNE&wg2vp*2P>T0=VWR-E$x8c12$83 zeXG`b*t^L|pi|F=!}grY)zi~+L2#Pf!+ahwebh#buN!Ue6-nM-IG^n;oaO%Ep=K$K z_C`?IeNlCJGio5xCku6eU(w{cL}Z>Ziwv1mzVyo^{cIv}%fenr?Vz5-usfIA zjYs2V3CMCW8c*I#oSM83wZenv215@*J;M%`V-=FSdXM(!}*Pu##Dx5L~ttZ<3b zN`olTXEno%HYy96kQ-a5ucw_2khokg7JTJTmaemJeJ$-K-eebnJ zLzq*O(Q`>6aRno1`gX(a))HDi;q`E}S~vFO2(`u0Z;nR%ExUKK2vGX^fWXfY?KRyw zll}~HPcIPhLGb>x-f74++YwJzW<|spOr$bDwYxgq$jY{9$60eS@x*QMH`P!VnmgOo z(gy!)U|^rYSSD-^$5a#SSIds&%`PIEFnbWoJz_I_vtG_iacZU&!TbJfUvb}-_l9lv zbJ-LBq1+cT;ngv{&v4R@=N+4yOXh(fr6TdGluM{H^$d6TnHdF2JAxmoe|%hS!ucV5 z+O%h2d8Ge8ox#KFypC+!s{ZbR9z#HR-Z2)Uq3urh&*vm?(9Gn++QvJwzpD)&R!ilH zm%VrwyS0}g_{@y=G>r7>dZs!GN@_1>qk{nuRxYJkmnqpkmM;fLtd+ahdA1@GM@1Zr znPCylJoe$MB&0c6^g+5Kr^m!=I57`W)}vdC@ZsbjUbi|ZZ8Y412@j+~&p9=pL|0#) zhq%_N1{m-Idw2g($1pQL=Hzu8pD(*g)0gqAhp6b5mQ>L2C_#7)nVEkgBQ0i^)C{Lf zf8aDZV4|eg?hO>X|5x+faZ1q{G16AIRlKxx)qR5koJFO1CT-&EcpR}RVlq#rRr_vI z$12#@-aE5@n_%E2$$#DxV`1T_5>}8=(mOtmOx`4zI^tAT)iv0Hq@`I)`#S~zL<|ys zQ&Y1}aB=Q~xuIxN9gwRTlaOoo-c*uujMts8hm)WopSGtI2}&L(v^KZ7LgHp1qotyw zdt$xI8sv-+)mVipHD7w_r>7sqkqb(~#Im_qy`TRl-*ycNq~Q|;BdNwx4YPAh%Jj=% zQX}VZurG{eP>UpqxDxOWK9MIAc%KM(+su7yOuP|YO1VrzLb}D*XgMI!s*3|}syd$O zW_O>Il$FzQiA!US2Z>Re)0>d>EfO6AV*rIJk}%(p)1<8>>8#$7dfVVEnmv7^X5@N$ zAjYg202us#W9?PySj*JM9&*0ueQ2>iV9F0%@Z^T>$C3Sn&m}qFb(-V{33GLKxcE*# zzHLZoDt-R+*3r?yP|9uq-M#nA+X$TRAdRQil=W?A;jc_CaMNvBSvAzOqvkRCLlr~> z|ILdxxKYG4#q2Hwd+n|y9Hm3&?1iT)MfxVGx~NJI1m4DMBZI1il;wGF;JP{q>ewM_k+!=49@!AAQ=x8lCRfN7%d|SiwQ16xY1}IhI}B z?FFIkQq*|9HWstBeq}7fy*>ak5FZ;;YO%}Yv*TR`>dJov9|t=fLnN<~vAn*Jk+Yfa z`^D?NN_XT(_3{y&cs}h^6LTN6?zPh0oHf^#NM)8KSFT>=#j3xfVWWR8WEycPV2K#a zU>iKxNi=gp92zewd8l%_Q*t7?sBBmt*dEG9&|bk4{AE16%KPfAk4;$Q>On|k&g8HV z2&db*8q`ve9PDsveY(oSwBX|i648sXQDrRSphx#^9uCb#?kpbrUmDs$-3G8wvA^S! zgK45T-JWRiSy2N;I0@MS5|GiUtJthy#(vmlv7tSz6n1S<9( z*}pZSHN0X8&Tm<{70rh0lm2ZdAb3eetcQh%p)L0r^WCgO-HV*RaFmk` zBjK+ z1Z5l^S(SpK&LP=KT1L%I>zH1`>O}nNZf7RRi89YaARS#QJ=G5gm2EhR=*&2Jb z!}d&7O*-t-ihN6Bj%#-zEgcnZiEu(^mLB&@Ng3YWU%u9IhI#g!Z5c0Bt0Ci6DUr8Q z`?1EmAoNZUmvlz=Ti){Q8g5FRSy^tS8mYaP!MFoeiFJiKgRH%GW;{a~#dBZL!jy#D~G zN$*vY_?U8_09`j+414i~IVc6=_7DJ_Msr0}Zv*QOaB$w-E*Jn`RAgeueGhl@!SVs~ z`A^^W9YA)UwPm+}#~3#z%GYAIA6AiLep3!6!^jy5|Ej8@l!ss}1mvs^k0+0jF-Yo( z8+1_NfUH|of?QBdP1YOtJ%tB!uw=c}z?-H{yyBemHM{K(W&{mSC?U-pNn(XR)x(UV zWVgDLo-lcQl4srmUsm$}^d_aNi|R$@!7Tdy=jUnnwPwt;4h6rA2RZ=(k-G9FL^=I` zl#Q0Jf5oACzyqX_;KzpaMn^^eOuW6iiyujknNFZNB8AXw*0K@QnajmGuHQj8l<|>aa5t5H-N^V*(d~G&_krfqQ_j%*q7D2CXsmU)c&Tpn6Y$Z zbLINVOmFj7&y}^j{fBLqBFK;VXgKHJnzQ4GZ|PhMuD3pMQG=3|Td<`uEFyx1x$)>H z@v46AOTfZ^=7IYUx3E*jO!(J1fjUsY9b()xmTQAJ^E_4`u}+&y!e}`*!whPjN2SO=rHfN!4>^g~%DOP|v^@;R5{z=KEg`_W7n_;?2vkRZB& z(@*=`(EM;affUpQUxp1q0c-B>jnVN82%+04?AVGC``>0}XS^O+_jU2MX_pId2W`-` zVJbP#e1$ayDykI@smxv#rHSR_8|ke7JO$#&h`)K&gS~Jq;-EE>2`bJ?Ug<$`8G~xv zw{#Uh9|i5==o!L_vT&24488&5)wpIU72D8e$5opoNBy@_c4UH4(T~7y71f+hHWQ3; zCbhEABHX37NkLiBlIR>b4gKZRF`l{BJV=pIx&i zab$VA-B^uTd`Sdjs`%0unmYCyU}XH7UkR947SuFLH@coZcJhwhO` zR9rkPRnpC(pojyI&{9OOedN?Xs}O)Ot^>{ zRcZuNiLHgvxi&hYp5H1nQEIE+JQ(JA`7Gyc*A3Yx!&SsG`2^Q{mfnW(zJsJME5gJU z;aj9h0wcZm{x|E4k`^l45aRGnhF~I5(|`~ttNdB{ysTY=Dm4awx26$JM~YB>d6}4q zsFomh#`0zSi5$Zg9Q2;e=5~9t@9Wc2kg*j4o*3Q-zha1+)ey0P9~v)XURjv~!Kax! z!Wy+^b7#G#!hL^7sh3ma8WuM?NH!;V0pE%o8e4DjwCwbGgW^ABw6qLwv|nD;SH7PU zJ%h3Y7h1!yI;hYk^OUBPl z*%yO>!GgVbhbcvozQ3*~{3UN%gDLu;VYka*>j~EG+&LtE`#&{-FUh zK`-6B@%aylP7aEA1P z=HKFuk1q+`$re*=Zbf3!O9Ba9&GUr$o6&o^Dm{Jy>(=Efk2T@l-YN_jkzf7|rx|cK z#`|1=DFS zkDXngyr5+5&~HM3PLnrvVDIoC$`^WXm^X7AxYgqb=)IYvq1+yScbhNFxF_JdveI9e z)T}@n4ywtzAm0I-xfkDZ%K>#+Qdk7VnfeJPk)T{1CABK!*<4VkwebeoxlzyWg3xfz z>mjQoouMMe+vHigN@q$08E+`qYUZpIhsnjxt)MMc8Q}^IJ`#}$tB+~HwE=^FnI`wVS0)PJ8)4@jf}4_}%}>0yQ-es<_#CgjDrb0}vyBnIMf`Na zr6jL%#bWw=ZB)S5>cji9sVs05Do?o~;-X2Hlegqu&v8k|s6D*Zk$c}dHSZYSFOuYK zA!vCEBn(&mSXg`ee(Gb#4QJ7X<+{SLN?$H7B zw&kpd!k8a3_28eY)ynE32>}5K5$Dx801(g@)Co!OIG~{WR7Za|w73m+` z)c8c2BO#$CQ+pH}KH2yLGEasqBFFuwmhx6moONW!z0-ctRtO*AXAlF^>Tou)ZuXys~4)IxkKdtQkhem5Z#y9Oi!CcIyJWYMT!`ZCE z<(biE^gF+mTUtqlXmMuYf|Z?nYmJ2tdwWYypLK3!YN@;H?+A!r>MxMUA(?TwuE!1! z;Ov6(zW}M*K!B_J^O;v-V#kc*^c-&p$Qrmo=LYn7+VFFU)*G`;vUt#H90SDuwhM## z8PjkG6zm#lX;;@ZVZxak*C`pd+U;F^ z1mFu&&hrHy)-Kai3P|JsY=l0oHi!0#oUo({eNd_2ggrs&qs3vmg9cW<$oYe%Kb2w1 z*$zBF0*e4c$X5%R2@*74>suTG*1bsyvwSl0LhkP-yu4ku>)@a^llG`gIbz%+R6hVo zZZ=NU+5ZU2!d1`%x`7T-&zF5cUG2kmFVJMHrNSeJ&CNFrrHc9|E0@VOhztBSG&A5{ zi(3;wxKQuiteWgCw<1EgzBZBl+P`^8LF3SsGfjw`em~v&{3{GSE_Z9|B?v?TVK|Am zDF^_}AdUsmn^lYKEPDs|X{8H}HJ^vC{D|2IDKoBnQW& zspFE+*eX}rgImyj^m^)AT}1O$#_BimxT%9}5P>pJ!q~A=q8nFQOyE4ee|R(J;&dd3 z2EZ(yM*t%1q|OWja;A3{s#pR^CKr0(r}Q{fQ@+LE+?*iq-Fhb^^ZIux)1IH({1zGx z{uMcJOUTTtgOvYlF+7G%c~{|p-(}o!^RD0eSI`6ch6YjM6XFUura-_IsC|Y6z}nrP zhmdl*0ub>IqUJ#I1_8KMyqeZ1ND5^nu1eRL4-7glKIJ=3N(7nN6&Nyc(?6Pr9OW>Y zAK@aZ%8RTg7o~76!P&5QMWZ4!20~(#6&QDj@8#nLLse8llcjkCd)|&++6vmzfuI~> zZaYa2&IJenTK{%k_3)P3p&e0^0zV^za> zRVRpimN3*8>% zm0y~#>#Ew)w(Z=Y1e;*DI?a_qwEA?0~GD*7oXh=>ib8d=6bJw7a~> z(SYz?{cH9uyN7H-+F>3n=z!PRJewTiDJj#@7x>jr^%t{Hptxu_f1~nt*3r7BeVMK4 z(Dci~pYrQA@Lc{VPz8!RN@3l0{2Dk>($0|7HSx-z25J;1$GeB~0ZaZ3ZG6yJg8)FA z4FEtJ2aHiVs%s-*%yaG`gRhenNgzS=($lxr{gvD|tg87d*_p~I+uvY7kOg#>wHW4I z3TjWK_ra2bOcauCc^oS8xCkmK;hIVCbO+Iz8IroR$$<_(B6MD{H*X&U4WOe1tIyYpOaequ*`W&iRDF6PYq2+ukHx2<~? zDh#Oe|6TtbI9l@Hccd*ceSMrqr04zq)(?4rBM37A+k)LLMG)de=C)t|D@yk?TAxXs z`EoF|`J+CamHJ5c9`rnw-l_pOgP1$tPgQK4z8U9;0a8IV>#GqH*JYFkO z*g%A`CK5;1IplHZb3p-`Y1a;%_OOFC8SEbaKqLl5M#i_MeQ!x`9hH{KF1`@1woIY^;t~nv=HzQr^ppH8A4$>9qJvONv{Pbigo~XTff%T;6InW7pt@4$y8l zr^b$O$M>`?j%NdW1yCC@EY`uYrrSC48mUS{wE3W1W?@t^NX|;G4)=k?WVoZw#>~y`avz1HO$xw_uWZsYqIV63A6BkLvD;a$^*w+Q3XZz>lq&r?F1>* zp^@Re#R4ha!&k|J?j{JTkkd?$?PLu9Hvo)JuElriO%r+Z=JPiUzAPJjiy{9KnOze z4LgDj*V=?{u|V#n0Aa55#A!Tp7~)~&R$WmF;Tz#6>^dZuzb__+x3m2IejpbpKRbI8 z>%`EKxWsuJ%py8S_D_zl{LDz->_(=hF6fcd-lrQr4UOcB(GCC`7`PKXqMxWuA1fW% zvbimLIKC=;|BM{Mi~Gz4FNk`UX8nwEz6P6z>W5ocd3Pf+1{RW=gixVwg_uwe4<6$~ z7!7Q*7B-oMrJ=wTSCK+|1OQ*yCUbOBFnH|SuB|tkr@D68FR)Hw4UDg=oo#(4e@%Qd zXD>Wew=nssfFH0p^tQD0{YN9IRz*#me`0f3b=F=1B}>u?Yyh~>=~n21_KJA?lZfIv zMK;PB#XvA zAY^p_ls3`9@kZiyt*ZJVMx>~-*(XL?dK7LK9eF{UhE*dh%8!Dx<;jMQO9t^@UCW*E zY@6ZrYot!abWcH57r58^Xg5w?@#O#;zv2(=t2Ay92h%a?wA{x zMiM2>?Ai`Ip_d_(dkCvuhp}-8DV1>cwR^wzgCgP~-l&Dw;ery<)zsFMp}9Yl5mfyH zlVC)hw9~%O>4o&td7}*uTKT6jn8pfJCz0Qk0rX_q%<1@)l%Z{f^3`x$kLNx94G2|| zUSmD#?A%8_r*}Fv^bd$91)G8o*q0@?zaarVtuF!$wN~2S8joe|gawE@ynTe0&V@7h zi0bNl@4_uCl&lQimJJqSUMMYORy9;TH4FQyh?^Vzd?LsB9G2Z3v-@#sQELx@Cb>s)h+<~VEi((=6U3BTU_i0KOv{Buk; z%0i8#Z{p$fNMv1lsnmpWPY&`|M}){*){oi|Z(~aaR!`S%g7aI2e<;3XS=sb>4(Zjj zy&#XI+qBz0Yc%-^P+xG8!brvc+x9jnH-6@z4|x684rPUQ_?iBA7MDD;lFIorEqy22 zZK#G+KtzKN7Ip+08dcLs8vDAp!BpPo(nBH^QvukXAl|1B)QnOU#TD32^xi|%f1>UQ zg7!|47HXbK@veVYI%FE>Eog-1d5f^P(|9Oz?*pp-kNa z)7ED_JbLULG6 zI~tMy07i4BRaRHiD`xzNNJpvxIs^qX37lq6k^78}h;?FAe@v>ltj&(l*bC|DkFnIl z{}kk6W@qTA%NVQ4MRIn8jG7xVshj=PS{J}UquQ_+0+|kMa%p9Lbf1B&_S5pbB_21E z%V@)^xq9UVXAXE%EfYDz+GSJ*A!S=^HR+eO46rKp$GH6i@Eg)>;$N_aDhgniYTau1+Sxq>!Oytn_2C>6jIo_g z`T~x|1IyBFd!d^>Z1&djvRrsCoX(T+*)PIIP!B z39;c@L|^d{g}Jw~t>A!*RUa}svt{CU$GPnprGyubiiNXwqvu#lrm!D4ne83}i<24( z{%*VHDSeJaM+67#*Qz|@Rnc;iju|G8FUIWc#raKjE0d$o&IW%Tof_xEVgbN&zW$y( zx~cp+K9(vhBJ7Nd(EGk8hgp;xBz1TrDvIB&o!(TzAQ8#mZGJp|K?#HaoJ8eK9GAc< ziM>PB6+dc&&+}4Fjgc4!qbl`s-H58&O1q(FquQvp{_p(gflc-^ZqNf5+p~L%4&&e0 z{|Gugl|sjnF0WQg&PBN`E^Wnc4(Hmjn=pYvFgz{pt)q?$)$dJ}^H)#M9t(Y%(bIGs zXWUNT><}dU59)r#A6qRi6w(|>WT-y<8sz_=skA;$CWa^JU|*JhO#3%lNWxZVU0A@uN?voFFdc~t+&?H=WqwLUr!*Hd2+#-UFkn)TL01^pZXMqA_V)|b>lXOK0!9_xNaTiv(P~~nZk_dL?zShp#|bCJLCXyU zBGniAV`aC{btZRpaGrpPOzZL4FTOo-Kdzafws5zTz9IW-PF)+kb~lfqb z&Ob|4ODj=Xcq>IEw;XD9kL6M>DlKtqK5IfdtrfL=k=ej%8&dnuTp` z?6qDV(;AOA_Oa2z!>2&Z0074OkMW~2sDbH^M>3bs4&cjc6X{HSYk75<)eWD|cT4sX zaJdaFN8MG!m{^oCBC-s%1yRg1I_I7w@7DG**1Q_-F=eXY1OxzLISw!DTM%L>{}@fB zYdj!q3G$)AQEX-{unl;v4#3kzB* z@~>gf#WfAsf=#_fMrJZH(kfpbZs1nqN6okEz%oa;KLiki5f@9LA5?YaU?eex8u#bH zj`DFVI5NQZM(#c071Qyp zB9(ELgOv;xhc|;x2A`JvwV_B!UZ>$p_9L;c&jSkxCt14)CedSzE6pyO(cky=>Bvwo zzMmroEH3G9Sm*|ltcsVi*vsAtL+ch8zqm|s$=~82u-$*<-I#r zegSDgD+#)#9Os!Wgy#IlLObo8$?DTnH9A3A%@%Ck3B&WYW@UxV_Vc7rn`g@%Noy-T zn|5=I*)ao~D|64A_DywVs)*Q@+e1O%CtJ{R-+_F1n^9iIxCvUs{ZcEWXdy6nDx5$* zCc{+ClOdG=Bm3w3Wo@=QBiKzK$)t*A3lEnGmGdaB%zyh#>Ri)2m=~x7c^Zq4iFGqu z*2V#Dxd{K-Bl7TNy-xw84GcYr$&?YWC_3!x@}g-wyRN_abp!PI9U8%)`MF{mbhcPy zBlZiVH^l!T0ugk@9albaiL7>$vg)!~8K1>B5!XYC=WMR@rqIx)P*^}OB{DK>&!@(r zdUSJP|?0yXS_=%LQ~?G(2(|<{c0-tdLC~gI81CKjLE= z`jnNzP7&2vRPR>F0X*%WX(bQ-DN=sq0+Ho!wP?BvaZOsS0+?Lf?nR+C zJ**aM&kwi~H{oGk+AcV?Y`ixg4sE#cF*D^cCEyHx#}wwJbENm$yVNDcQe9enCZDZt zNAiNBmsOW|yEy!^gkJ)}md)|&g4LLrmy99rXwTNyH~p$ts1va_3FswqEcCBKQc{=+ zqOdItugGBfn+D@`9UUV(BQBwRNuM2`_eH^tl$3>t2t2^mx?bhmWR(GLI!{a4A@F2O zUYqyglBkSwEM3cb_qtkaT6Q!}!@0e9FZ82Jlj zm>yj9eDu3%1e0Iq4?rM$i&ASp)Qb3n5OI{Wcj*5oP?RSYHa8h}(Sw0+I$Ko_R>N9t?W~vA( zJZ;Io6K#SlZifUmNJrzy+{wf6`niS{5HM!L5^M_pZhUMWH8Q6HbQQ(`K$p3#VPid- zh)9<&gj)uxdK$P63v1_JoPw~#AZfUEQ{4@*j}pJb+;K)vE`*viLGl;MFO+#Cpm!TL z6CA@dOGNk0O&n*gc&$3q!sA9?W?veOS6wdmc)?s*u|CUH zc|+rY_PTqw6<1>n0b%NT^6*IengX%Wli4|GY(fg#UI#KU@6308MBIh7=zkKDNbzx` zVPVDMh-~bN=VPe(TWJ|Mw)`{l>3E1}@;xMkE(uS%hLIf(=7;=GKlvTpXN!kpXOWzU z46SFwMB|fRXIi{+4RXN)0IQGcdNR*%jeJz6qN1-5~ z$yP5l7m$eY&(3IYTcK)}kNBy-knt<48IJ~92hqNbps!;jCl ze+JBrkbsuS;vC1sm(t?qkLLhzZK8HPzH%&J+LW6ZJy*jEVOZ}>NG*%`>Dz;<4;^eW zLE_7ay~C8WOlA$E$9>LQL0#L?P5!@@rT)o0)`2g_bj*#fskL4{p;NbF5Ll+7B~fax z>rztulk?B9@wHbbWjZ7;$EHNvc0$1gPv8Ogrd?UUa8&kszu%FgKxK3D;y+R9h2Njy za%h}Ct*XV0`f=obO2t1KNoP)^5d3SvY(x{*S{=scCsw5EfcGNI4ik=Sf7e%9COvKH#8rJ%mJ zoh>S1ygOQGUm9@daf(2wq)EKR=^0sK5tIDk_CFPG%DAe-Ykilae7v6`_awE6V=^Zc z^4Z#1-Dz#WH7hl7n~aea5AN@ELlG-8Tm0HeZ{xF->@o`FL#6F+*OZnZZm->ji21m6 z)F3C4Jd)OXr;VJ8EcGYgh<$-ig8bAs3blHWQ>D za4)-nxvkt!Oeq1=;>c{M7YpRvk8Uj@!7!>S^D0V;?z*c;pAeK2Ffp-=8r@pLU@_Rw zp5(R|t&wKkSqhL76Dw?gk5^TbNT6aUvOoarNhSw&eHW#ToloU^Aw--LHVH|qpXp^i zJuU1fXL)>ZjI&44xS(7%w=SdMn`v@+4`_{DR;3z zYddpPo=V~fNfEKGx)(Xnp8jSrb-0ny>bmde)9O1V=AS>~w%4~jUQa#poWh4JAoaOL zx{=2;9=CpWvDPv-HTHgaK>|tbRn1QJH%E=RFYl20L|3n_2GbSa8EUONwea>=pKFjx z!RkBOMf~ky@WB`P`nuqCU539X;QX?`J0ep2{vXpaU6#>F;_s!t`23#YLx+t3=rL)! zD;~PI(Sw5k$_SoCxw*;T7Y%vj64kiU`|)Soh@87oQGMHX+S$MS30VJWS^sHX62CFW zh)|ZzOT3K89)p87QvNk7jkAD6Dyg1wOJGRo&Ypq5-6&|;Ap*+6(l9a>!uL+Kmt`Z zj~`;_J^ND+)?=jN5=fTRQt~ldN7DG9(F#Sc9!N%w1#mRXo5K)a3TelLPf@z1wRZjY^%=eJcFN6pIU#V#TGO+fLe`(idbSEWn&;U*!OQ?Zao12- zj3nCy=ZSGCEv=iA*;Z(cBLINfylPa?kYPeojT`}R%`qz2+qGhP}R%kkbmKz>_6zwR9Z2GJh8OL`l!FLU2&kuw>;A^pBbmc zSDblH{nFLe(j_IAK_^88Ki1gLoPA7~|Lco@41H9$k+RbK+{|3MoU5Zt9TgHO!+Mox zWNb97pI)ck>Fcl<`zd#zT{AxBZmNw=V-HH(hj(M{ znni}JWu&?pwi%_7!dCSZy!d^&Yyq#9G}g2We`{l3HE0*IWllH5Aaef3kQsC zuWzvMDU8?`r4>~T7cM@%`8{29yg0jf?_EbiK>ChxmQ(WJ0Dz8Y`@y2ZZ{m-Tv1cjP z9C#7kgov#pdNrU%RAtCO0@ThsHnv91h>(=0P|1LiRrp(>jK1v zMUX(#^M$#Ijo&bxW({j`Q(Z=Gu5fuC1VCs_mptlPR-8X59*;fl{P{~|!4(@1b80z^ ziw9bta7=cqch@*VL|;*4!$&}mdBM<+PmPX>rICb+q4?UvfGK5C1v;-Baf$QvV``9V z#X5tJZl9Tnq=eL4enIX{a$N2e6CJix9MxtAuXuK0wV$l*?|CKA@JR6a{MuzywG?}o z(`%e4O2cqr-}7?U2K|Ftk7P>bz(V3h^IP)BX%)(nYp>_g_F?}!Eh7!aIYioSSaJpY zYDGbAAk~U6F-3vNka+Ba>eIn50{E@4$jCP5W0fC2CRUmqu|mEev~a(oxoimV+{(Kg z<#C>#7VjF*pNU~9t8y@_gE)D+*{h5d{K{svfPy5S%f^;CKa(({l&GY{q+WK{(9mgQ z?DVh3Ptke|#C$tDCbjB>`1O91JujhlA>yc->g7-$^g;N_?h=!bfWxj$>@(PebEAr< zWK%+y>yjFj8nTG1FEm4>+VD9E2=FGy4ZUFwzpA-Y9BK7x(ls)I3?Pl{J^W0|o$4g5 z`I!GcJHyI^3vNG0i|sTrYSBJ(t!L#>PE2N-1oxjyQ<0OA_4M}#i&ML9?uJt$?NFwQB#YFibBT>tatk8s;j%-989)dgL`=Y4h#(c`=_R^&Wsxg?y4n= zZE`tXT6WFTOiPd3B9<{FT!vj5Kmi1#E8Q0CP&2zJdZ2ok|$z-(M75hLL!L zxieTjJpmj~A-^aoZ|Y~S5hAM95b>Eow$_&Tv7SxI9LdQtqMED+N{pj&a9gHgJ&JzR z4FIZAn#;hAPs{nq%1ZltI2J#Y&=tRG3B74#L4Jd}X9jcq zL3r6}yt;zJ&r1WK>vA9_zwa-VnHl)30ASy<|D09QFKTXYf3sIpQe7P^TWMjH@jJ(_ z*8l>@xVtNuQ!!_*Mvn3MZyN@J%MmZNlD$^-L0xCk(wu*i(P-|z-6%2-xg$K`6ufH z%$?}d+e?18k`6xxMk!Gk7tq06X*!;$Zs!9{BXGx%G%KrK{J@?7;Cqc|iDy0qB3(sk zBSxP{#z}dV8X_YjLQ4|r6QDg2()fbf(wmIJu-2w3uY6qnPeMX4x{(SE>JrRL@BT!b zKm3G00OFyccBnfZ78aslEC)==DEZeT2lB4P*Cu<%bd>b84 zrjD&BEP&s(lc7th4j)n?%A{AVEI}gjpr@pClp9DCYY|hD{+WM&cXFeI_F=8Nd-LV- z`ttHpQc{wXl(f3Kdbz>c^X1{P!FrjDjBJAH7uefGpw#SY_q-1e4@V~Au5&rnAR@}N zUT#p=&;Z|tje~P7J7&O4Kwy^s63`6m-)#^B{WwhXxh5s@B$)=>k^g1|)?s-@Qg-Rj z@(OBDSHIeqP5!NZFCfwWHt**{N&ZQvf8PfOYw9=9u^ZCyv#axHqMzTMjqqR}G#q?# zm~pdk80?mom5q$(*x0NC{IR16V1TBKjJCzKwFIF>ChIinC}^O_!fJ)h^KSa(Wogf3 zZF9Lm4hFDlc8MC?93kd$@edZ2mywB4b?EXP&wRy%&oBD#>oX~-pomD2I0ZK+r<JiOGWr>Cy2u1p@+PdqLoU0uGevZSOr;LcpI z;P{Zz3NC#C_x{G05s6wHVRm{>Z{LQHz}~WaKfKn#Zj)pLfL|e!QRx=6fdn_J-*x_) zn&C(NyuBJVLkKuOL%jV2)?RhR$;fglD%uJc{2*nNl_3ChZOxqI`7bfVtf2uU0N~>J z+~`tKRIt|cTvSv?=zD&@8xbC6d4vZpAUx8zoh3FhGIDZq5{<&gA|z~cI#d9k`PVO0 zz&9jBOhzUuF%hizwEuj(0&Dl7!dqb&)Zq0A2?+ooOlE(3G%HLt+1r~}RwgYb1`{(7 z^NAOqMf{)Z`=l37n}otn>NQTO#Ojt{|HNn=W?wYuGW7 zcZa0J6J=p$a(fx`cIrU{B_s{3s6SpE`ET?KjZ8+(ndK zA4}#c)l}7Z8P)5`bl$v3CXka`(9q(C1jnXvc_HfVXN>wGX%n62;+Ev(>1Q!;N>QBX zyW8jv1T5|NilO>+!44C!Iuw5*;t;C37Yx}c(H{z-cVOg0#zZq!k&4LWbX8O`j{*IjS2XgTPGG)<*e&D4IEj4gdCDUPT4Rt1Yjzbmu6TNK9Dx z%}IV_q&)bSail0X7~ZRG6<>x7wUUJuB(06*Y9V;trbTA;3JAQ^tosK7!1|@p9~aS& zRQ&0u`QbAhr_m@AYQWO+@Y5{{WEBf%Z)Z1NMLzQG*^Mg%#D6xt?A0~(ci`>lZ*-Un z34uVgM0>(vkP=nsz9~%2Z*SKsD8R-wx?CS3vCT*LAvMlM!d%JazGe*cNAVW+!753^ zbD*^96eACe?H^)wz&qGeXK)xQ(YJCyeP8iTi$bq>p;7>r(`Y17P@SQ+zC1f*LT2oF zV7|}#VExy2H>L=yDL8tBFuS^9_P4w2sM6ohS*;z2A8w1G;UDG1U^3CsWq8~_L`7B3 zEkCP01T&9+`Bpxp?y@?d#H?JD{OG~((v)i@H#0ai)cEy=kZDCf-Qlm?@`Ow~(f4n< zP(fE*q4_1yK-02(YH?4&fR(QJ(&+jpxrwBDd!+-sg^HN0+`Lm}DF8I*rSt%CBXFQ0 ze*vvPkOb8?{-Tmt*TF`GlmD`5N)j00f3elwyb;ke0=y*$d{Ccf9pD8@S50VXzszJje1h|G4lIpxaqM-5{95?L zuPIx<@TnE>Xhg8Jf{8Ev`>Ng#CoIQRZ$B|Ue8YjjQ@jt+{kr%xZM?hSkSczF$20djA}m(T=3j=epT&#lL;)MMm-(f=$lINKO&^ zO_3tFqkLfjI8ERY+OnFj{2|#J{jAWc8(EwLvijIoXkqhW$mRCv@cy?uKd2cr zdg9xq^GW^LgMxD~?4DQq>(oqa%ERZ(H%pB@#ugSonDqQWigU8GST8lLU~cq99s>S+ zrw$Os@VHzUBy{ZVEJj){n3cc&RrNI zW@5VgPRGZ@NyS>zU0Yd<2>9k=y39EE_KqnrHAZ73&-9Kwn!Xmu3dY9NCsx+h)*bxK z?)b>$WI(K2+A@KZr z4nxz7oS*8X@H^uyl(fB*N7C)U`liTA3C^fRxOfHG{T0>*{RkUK(vVb!-DcL|%Jb@F zCJzPjOB#mf6~ttoMr7kERfIAKVK1Rc^y|oP;32V_@5tbjlY0-CnVTWl*$Fo;0N=m> z7q2lC@1`{|k+i)0#MIQs>Lc(n`9($HB3>#TcC47qU!eYm8JOfG7^j)cM`uwL2WIr* zq6kMPkqPO=xnwd3tVdcNRV*)%4^(3a2<20e2#Alja85}X>?hnmoz-0&85U2{T{*Qj zH5sHjrP}bxvX8iKCtoBBs$I=!1+w9~qcI|X`z`LZP50P;Bo?PsDgui_MmC9ojx5l~ z!<)9b-7q)%gNFb(#(#1s1dKMSjD_F=TI1-PQsX$MXO%U+q!W4(OQsWfhtjL}uJ|Lf z94#e@5uLKeOv9q0A~6tZ;Z@vd?^jMA5T9k|>BQ5b?k` ziX-9mQ@Vl8V3axB|5E)?%?WL}iOcfAK0Iik5{b}v?_gzU@Di8rgN1OIgv|7J`F6j2 z#Y$ymn~R0<@sYRXuSZ4Fs$Id{@ZH@zOI!E&?kFG?9gdw<7-X>%CT7-*LOdxdyv1!7 zAuhj`^vF$pnkF9}r*=v_GdNCh+^tWkgrw~^ffiy#z}$`VNnZ$y4(-k8YKvQ%+wCvp zKna=avg)IrA3S|SPu8`SeSH&t@HqNYQU^UDyAXh%ivn-@y!wydGu! z64Xl>Nw!CLGAWHQGnv=hZ=IBQY+z7Pzi5J4*i=$-r6jw!H95)I)wP(Db3Q*`Gs5}i zmm{3&+6CK|Cn$zgTx9zzvkt<1<3#EmdUV@ZQPohC zCm^Y`WKb1zWi-%pGf^tMyyyyTK!*+$?(Zs)(_b-*@W@{3^17(L{!c;R(3Y=OE8oEM zb@#>2KzG;;HU~WMCMu@l!sw{;@c?CgeFqHuUr|3gKKDmyD=QDXwvbZjrIGF+EJ+g+ zFv&oN{G$GLTAdQs*`-jae9OCbs|tOmpzwqZAEw;_hIh&J`Efsr@+k4R2#R2_o_Tq! zoalttt4+CFM_5S6J3oI9L=T#+?r!^&`7yLg=F3e^_m_K!h=`!uU=a{N=?^~VJO}lg0T-t z*exoVb1}61tatguz{^rYYT8*+MIa8JjYoQFX z1St-7c5>7JY~PsJSQc#fPbqwstAE?u1tcXUKc%!7jb(7zY+HW(KoEBGky%soOJAps z<@EuU(ugRZq~u)7i=L*;bP2uZ45M&0LsLshtlz)cc<spL*VK&I2b+I@nl3JQr_P&ewK27a60%F(+ufnPmw0iG^^R;tzgAvK>l?AKRvqf=_d z_2FH$o5NjX3$KerZO;ZfsrRwE`!8hYi2MDvD>4Tytk<5S&S@?)sCf0#zPS-Uh zIW1Sb?4O~(keu@~Xa>!kd}27Hg6vwAZ5w!h-R$t2qMHVbS=^g9S`;L5eSKWn*)ZlS zEf44V;X#r!OSRQzD}8uaGT^=sc6TpNR#qk_qopWvb8`W>!_UN2=JE0jOU%f~`0d-b z$FtdT-Qpf;g^WUMY_|sX=+9&#BK(fAvFodBQ=^?31sPHvznC>twu~n2yuS)fUR6a4 zJwFTjhWwbCLBQ2nC?Uct$AsTN3GgO7nGZR2Y@!UnDZuRyS{Q=z|Xn;q7f69OR>+ zxux1`eR@O|Xiek3Vd8GIBmE$-+jcN>Dl6c(S;phR2&K`nMrF1lRf(D8l zVW};DT!3h(sbS^hByAe$ojb>Fg&%>5qt8`DtoEd z8b^}yWEEwC$i{+~m-a%Uq~4QTTY^hVE3vV|52YA-*QaYE{H~v2<;2NcZN;plL;Q|% zIiL(B2Y@MQhf~#?4Nk(Wr!3ncBbm;mFBW|{k>Oo>{?zbfciLm?n~}MI?Hyk9JGi4N z0+#qq1J5&rm`pg*_QnSeTNs~=B<?mC)|Zn;GJ?`DF~_sxIB0cO1$Q zq~wxuk#g>-3FL>LA-m9EksqFIr}?z64EW;P605)@)`9F+PxI5h_lv4E(rGRK7W6ot zBa&(agpq#9W4&gQJ~^zxgfT7I+=G48_s(y9!&q04IQB^oKYzYqbCX*oe08x!b)-i3 z7J>pD(5pmuilq6_)4=Z)7x+*`GOY;kx15-%L4bA;NVzM2>moZ?zdF5f?^roC!@yNi8y{XmY8XAN-~+l9%&w| zdXF+WLTjX!wpX_J)R9#q4Q2jAl3a`3*CiU1Cb;#|JOn3|+*;6?N{nQ}-XmX#y|n%+ zQBfBTpRH67{Y|3@XlSD|v=w%`!9iOxlSS@L1|$X6>*gybA1bz zZ?&8na?jk=tDsw`8koP?~KU(4?d9#CLvCGIq98+RyYF1x( z!V99izKL zX?|r`kdu>Hb9E&-N z)7!R3cDqKI_oa7EimZw zVTldQo<^|bRc68yzACZ7pK8v zTAa_?HJChqMfiwqQXg?e^K3T#q13pTU$v!Ehx2R8f`(XTfV3e=gitx&PQ}6L%KEY; zx7qqGHRAP)W*GO=gE2w6ep$0mO?hZVto6>cyu#P3wXTQ3$r+Z&O>tenmxhN6IJ0bO zor}T~86h?l;-JCeFamc|rJwLCwZdf6XjoZkttIT*L+_z(zyvlw@yuelJ>tA@Owlw@ zyDfrWtaA3uDY}f>DSexOdwrA{bGMx$_tGK&C-OH(8kbkU>8X|3K~bxL%9A||=i_K{ z`k3ExA0fTdl){KaP~>uJRAjo8HMW&b^$Ze{Q~3F0=Xz2h$Mnopf3N?9P_jX$dI?sG zM^qHq@l&r5iFWs<6)Q#9h9w>$8NYadn-x7}Z9|QWCqwCitlBpv4QbIB(a-0$^_YF7 zws&qnSH-^dHainrGm3j72dRvdXe-5hkK2jE@}Sys-z3qr@o47b!0di5VQslzVxcVk ztSMYPFoNd#MF*Nj=oheR{G=)<9V#qv)zWP1 zPIPSk>hZ)UcqnC9dT#Qtz9e3=68sg}Je8JT9}iG#oTn)xZpih)8EIn>*6*NDm5*XHpOr zbeUfdIgUC^5n1rRatN)N`2Ou^7OUudzrltx%7Q%W?%rb5#3NTqU%0-4rMX%3Y4oor z)QUk45>xH*Wu%9{K=#`cz8Z&t(jX{-Fh2{|)PyAja~q>l3%9DpejvRZlUJLW^dH06 zR6#&sd-sN&lN1h4L0_1|Z{Yd%tTHT?+1_AY)$HErPTtrF1IzSFw8e8sv(wqn@Nz4d zk#CMXvCze8A3IEazui2vkTKGdTIl^S(K}6XOEol@lTL8i9_0{xLbQiwJ=kYm#^kzt z*GgCqgE}Up7awBR9ff|dmg&({_!*uFr)ulRsxuugTbvlALx!MtOa-D8(=kjUK82DbT z%u$}1A3FM74-v{{jwOCPOnYBE5o~Qsz-87p+0|xP(*3l_#CT@SYpn&|?+BQhnuEN4 zlkG9q8J;JHVb$`vNfRR%+TtRr+zIY~L%lyWPVy-9Tv#F^YPg9mn|K{m2DO(y9*JgR zS|G|TgQAyxR;#l~p0qufI)&k5^YH`Y$(`xT$4LyITC6w?!lrST`&;eGTv>E`I$nX! zghIFx8>P1=>p_zp9udi;g6_3sfs z3dJro@qr?=`4W9EZf)ref0;wvZ>g|4w{*#XU2K%a6VF#YNJpuyk1o>re}eJ~jGLrn zG4w(lEYuS|b>$yaAbW?$#m(4qIzUBz8Ap$(BOaJr8W`)MCuXo;_9dln$`KhB)fSy} zL-zN^BqNs2Rtdwtak*_jVe?)Em1i~ioa&Z6tXya_#*BuPNs&6-mEY{j+v+R4nwg#K z^`Xo9=6}iE)4upPSpObGvauZY?V;H{#1Uc$KijTcl@y}MSDM|oPrhlWzK;-tN0_T1 zw@rE3=G9P@TM&=v&BV`zt$Ppe>g%WNQd7ih{llUMF>Qa{m2>-XmYVlohAMsI4Ej?J z)3ai5^Aqfs>DiEg-5&}qqeE>DC8_ki0~q#1yrGd5Ci=n&KlazPXP=J_UKa7EvxkW5 zo}V6EhLcg8sEP_{U+d`I#8%^ZX!ZQCwIrwnq!6Eszv-s+Cr+AC27Pkj-$lpw*_9W~ z(DOD+Apy=L0%WM9h)4XmEpMzNj@z1QrL44D!jg1VtzWnJ_-Vs9yk$I3SXc}5ytcNq z+%4?7#j{5Fg^bi;ZvkhSy7@!f{F-$Ox&+~NCohk!(z689;VEz;OI$CXpuD#d;_poI z8yR%wOEMpU6IJlb*c<2#HyOpnE82ybhg@>XLwLkGdty}L$-f4Kh)7E<%ls!A_Ey^N zlli2u=|zLV2XPsW*!Jr`ZiiQYt+#_}MWCqG8u1r!a%;b1|H?3=Zzz|bEk-LVuFaui+D)6%oBG2f(egbJh+yjxKXk9(LI<3BhsK{`Qni@TB#mGuZI zrECpHgJ}OThk167q6?SA$Ipw8NFw z7atqLqyq_oX*$;n5)G!9*XfdO*sWGwJox&c#F|{nEU_3FDiZ#zgN_C4Cdm>8QpfXR zi&grFI=!@=cF%aRzck73%FNeCtN*73C}V1MCGJ<2b#pk~*Jv}1JX!I~xhfGPa!n*p z6uMu@sD3L#ksEf{jO>&=u!W1ASm#9V&r{=()j@`%!ev8?XGecTTtkWgFFncblDjhl ze)uINaZLE_4^sVS3$ae~Rl~f^=op6`k~q5h`u4g*@Bk52UQvO|q6aI`yR?)dMFCI) z*uxJ2LhFEXfBxiDbUKxZe-Qft;k|omEPS!~k}OnG$~}|2MejU-bgL)hH2njUkN4;I zs)1#ipbSmntvE1#cS-8N8KOQM`zyP0F-Hb>rKR4X4JqK#+hC5hpyyQHNh z`;FV?Uq4aGphmk1=XkSkx@bv50RI$;4g1ptrs z_V%rTxP!etjV8yl;S}zTp0G3)VWXQ0Ai59{?a^y;96ODeko^61WdFGk&)x#4V&o&m zKn+(BUyk0zpW|8L=jZ1d8q~hpUk*4caL0bGU@#G8{`K~QE{0zm7xvj{9^m^>qu)~# z?kNJ*kQbOL6^w>5I;B{mUa#AO_-ax9{pXttwx_x6@UM~!V;i}Hzv_pfS1lcP+ha_* zn`iGE=FuJ;sQTogpr&Dof{qeak6yKiYihQDcP1R49Ab-yxor@~Afyo#x{Tk{fF;-8 zz1%iA?kj*1K<(B@ZXm5d8LOzJ^l)l^SlJW6@_n!Rnf76C&rWA;m35Tlb&K)K^Ai9z zolg;=S9i*kiw-y~_*~=Fdvw@2Gs6hA8D56t?y34r#2S@GpWq|JqQiH|r6~sb`-2HN z?X#X>jZiTmVnTm8Q$E>AsHrV1FKf<%x=4SKfEFR`qm{s{+^3wUH0RiS@(K@F~WWBEf`7yP0D_wN`z7CIe7aMb(%|0vL0 zI~|Z|kBR0^_~5_ZCHi-2c%PmT`YVV(AGQi(%fDtjelV?0yFz{`|6oaIM|@@L-77tu z79a2UaGg>SC=uT#fR4WD5l#~ET>*z*1u;P?cGt&uezvZ$u??aO`K%4;wlQWYY=Z_2}v1{FmG>bbr3zCnnU);;&M#&uqKK5U@rms8`7B~ zMn;0|d-sA%Bsc>LyCnMy16>%-M`G*jliA(d_om$MO&ja*lz5u9g+DAaB1n^h0)!g@ zow?8F3eR4mGHlgV#?8#U>~~f=Je4^Go(OJi0uz4?uv~kkC1rLyS@iX`R0!**1&?$} zG^Qsb;7Ty`)A zI?84k(_EYxv30a7^al(`*t}jrKFgACB7v&IHeC14tJhGs-w1qay-w$bLQl6*z`ay@-C+nEM zDJCV+Gqn8m;%la%-g^_&XI?q>gsM-+}M>pig*q5Bcyakkgebds*4{o`gXM}4Gk>FAGQ0_HLo&32tM(qP*(ZbM6S&EQ(Xk2%;Qm7btpvp z>|A4Ra`K~dJNm21>)m&>Lr`Fu+H91F07N`zZRrFGnJZE=nXjRyj`gwBZcuS#uUrRUOC`75&4HE!mnSFV}U z+3N9>aq4EyI!)!Hs@F?~pue>Y@B20+XRMJw;`8p%?O6p2c9@d7T$}Mvy26r@;ekP^ zfJOzCxlwQSY9bO9)k0Z*Ufs%9iXESl6Qxp_A8a=h-dr?s+kXTP$V|c#0DIjklx24S zd+YQ@C83t#Q9(%0gt zBJAru-PzocD!EK#fqG$K!>~iD0z)S{r0-MHW-eSqOg}InqXfWASk45;qxJzNPAChF zOjN5^RvPw3O^6KH8^f!_x4VH$Q-5lT)phZ)_L#lu=YiQ*b%tC?C@6?6D>(Omr|n7N zD~%nKNv&4rVN@N=_S}LzM+Uab$MUCkL}i-t!239elPINMorPzRyWB3ivpqdvHJATiJS*mDkC9C7qHY7N0M+{3KuIAw1oA8Xw@}stp0OU=crPsRWla?u_3?H7`8mbw z2ukAgygzv*1sVAn^J)qT0RoCjism!FP-S#=Rivez@M&9%it_&bG}V;E&JL2+vf&jL zHtrsnCM4O)dzEHg+=j{Z)cR7@tFBO#toOO-b;5cXX@a%Q}j zo{=ykm-1hgV+fHuXeOP6kk2(XZpU2rf`|V_PG%S|W@@df>fg}rPS@tgPcSKzsoPhz_;Gk`EOrc7V?%CPJ;SuLZ z{8`1HKZ#vBHuXt{gV=^T=c`wy-lG*~8%sZbGGt}JK9GTN_XlR!nd-IYUo(?-!ky^Y zYdOW7NHJ^lcjEGqt@uhaqeTB${_mTcv+5-zA->2Y5@M%`NrX*KHXsc(qeSNP3<~ns z5KLWQjCWyo6Zg^|yF)dQQPzwMt`9r>`1gj36Fpx z5JgAVUaFPGV>FtSK|)WSJ?+y(Pmd37v?b#=(d&y^VBL{L0nPuFG$dH6x>^eh3+Z^F zHZ1U6i&axTb!mNlYr`i2svqZic5i$_CUU69jX+yn?CiOi^a8ME$s7p8Y<=yoi8p^5 zCtN>20h*rI7a=P9>uDT#?q4W@>aGU6%q~bC zahh8!`Bw|v24r4lu-pFwB2vo7$vn@Y?z9|o%G)|Fce{~-z={Dw*%v8czYf9HN z`))G!ZW{CgRJ`nSfJDpebbQRF3`6ys4gJ}C(f>Dm3@sT&fD-ZWUM8XNw}4J0%bCZ* zb;?)$Ji=DT!YjmxvS0L|Cys?5Q6u&$x0jj#3d@xaN`i54{|LCbEEnXV2obBp3qpRU zyzaJhzoo#(nj3~c-n&KQEihf|e`^CHkOv|ti|gR;lf{$B{FrFSM?X*q8*Um*6`r@uMetz)xe{8xmd&;%M8KVP#Q64E8G}96+)G z8`p8z)2-23(LFN(u82_RCi?+iw#9?;M*yb)ti`5SgNI?w1=S5)hvMB}Q32J%euAQ6 zbg{zTadm=_*ZIq;%3Wd;gRzWQbuP*s5nyKKw$ap{35CEj;FaDg{JR zk9(O#U=WG`ShVKU8IlI*^9r2RRGk@FqZTGNnSag#R@PKD4d_#rwh}-Q^{{+2N@NXsK4_MNWsuM*{e>XO@mepv1YrB7$qryAn3pb&7qnIhw27VsZoFN;IgN z_s=W~sGlUlEjBFZPZBx#$ktl3v6WTZ+Mb2I)Vmi34;9~xL~9#fvI zkb0Z*creh?r>K1WqoX%TtpHa0k;0;)!~K5=BnCfz{6g;U?@#2k&{R^I5%kv&f)x7GzWqLe&^0PD7Mk;-`w!X5owRf@HF>u32Blf;o0ywAGxBPM?M);a)LKFV$ueXY> zWclUw3IdDLkEdC;3lx}&X$smp=GAmRbh{_MyN<$DHoPND?DzPE%u-pw<+6Xu^uP>D z{5rzOT;JxoR9Db<&V$mLz!xV;4FhX4EOqs8bAL?91JWv-6>cDJg?*1R436N7_*)4W(gDakIgAR-e~foV-E@2~eYK)J{TDlIsIvP}h1c zHMXytDdKc#OXr#;Fm+7ug0P?fdyC5r(2uU}!M5-}qqdfmaD^R9siUEIj~ew*llDlr z6P7MCxLEAov$v#Hrie!M>ItH~ZwNFQe_FUb4O+{j1PtO7y9*MI0jOulZJ)s^0CuP= z7!1yJugtw4N(FXft^UVcoPtg@=#YuSD-3@ouEUgHhdoI&jL z?F6wsX(3I2aBHveZ7_Y&A|-s3($_>j)6@s=bg=MF-#Uk;9nN=u<6vS*;IsE{h`sL2 zKPHIsg(*ITQ|0qGw+$mCoO=t5r(1Lk%)uVQ#0;Uno;PxPE(Wo<#I-Kd+q}Y7YMOC5 z#qPwUTJ?tL*rY-d${Fgb*C3*WMMNZE`tkR2Zz3clL_=Nu{^}quGBOgN)}>ml39N>= zcz8;Riit@{AnmBd=|V(Wy11fZ988*?o}T{xz%<#-tt|v>rXNT1)lc_V8xBuAuD7QE zfRBlZ>F(}!l#`;E07v%q_07)S0t}~}+qitPFgsh=e@Zrq>*0nUk;ofhmRmQ!s@xat zdEkMTE%$5wKe0zF2u?*9{9Y~#PYIF{kxqXy68Of%U>34W!U^fPX_&cb6gT~rKY0Fg z0>ES%+I}@c870NLX`{$u&2QI@jdq%c{H9~AN?xIaQuGfDfcpo~lQy=2 zfdMA^zfYC^K&;!t^-{e}F6fK7Ij7-duHoU~nwpx6?V*e7>zdkHQP{4Y9yL|f?w+1B zeh)r!a$%s?k(8tb=`No>sfvnH^6;n|@xNgvE728d5h6kbYmAyjL`00>F3igOo7ms0 zutFprhXuh!$-zG#a51vPHq*F(JK>B@5+Rm5fg+&C@-K%(EnbR3z`tV+IY0;rDeTXm zKb@VO-rnA#VFY7F|M*LC$yjV`>?WrRUy)2>BO|a|BRVD~fOG*y7uVYQ5{Ni{6+fnvvKW2q=LHa7d5kOkCpD$I;1&*jx3aRktS; z0gq$!M_=(Z+D2Q!pUNgWx~8~%}HG)A?5rA)MvAam*EQtd{$6 zha3shW_~?ld?@07e7VC#t4Vvhv^70BBlM+-kfu@41KCx?^ zgB`!l?Lpd#IYwxnN4L9ep%y5DLRmRd^|Xl(PeJH|fr z|7&~qr@VZ75+_h8VgG9+%A(cr%t2?}@FCvVbcUS`#4yVqPl*eYWYHhNZQHU65ReL( zUZ`vSY5q7wuWZ9Ax875~{aJcQBQ^D;zyI3BC5nvv#>IuQ)eVoyi6usg0wng9*88BMutb;2ULQqkMhECp+xLX1^H=vb0FEhHRCi& zp7nO{;xsa6|kgUt&I{NcH^m2`6NtmC5K;lYqVCx@13I zkJQO{miN$>fubViCf4T6G;zTpv{Kkr_-_!e6YwQ*BXbML2yN==)4?QAE`|uhej${` z!ug~#IEtc+EZ}rfagt3&;1a$3Sf1{4Ar~Ma7Z>#{E@mn=Mvh1#kD97V|1%t6ZGJw8 zhBHtGN=q4w3K_Lo5RW*)hTIlzygnm&M0#FPq)*|qsYy+T zYYls=D%#*Lqve*;a{rhMRIPwpBNu*iBM}I;uT!?TI50Kc%$vNQSEZ&#O_5<_WY|1& z+={`VvDHTn$jWP{qNUwBb7XV!`J~v~6dA}SEutVTm|a?4Z(srAE3y!W@=QzW*{Y0yiY^W-MM=SwAcH9XHh}T&3!$7E9#uN&-{cm%lL} zZbN<-AqG5C)MjU40{#yk8}iGIKO5yXO0@z}?S)p++HRtGo3Oa&-RpkQg1l}bbbw{8 z%7}dUWMg|1WM~Yeub>aw5n+hPPPgzh^?9oCO(w1t*udoTXCIRg(le^c>zxQ;@0!}` z(oUK_|M7+tAn@th2X58JdW2*t)00uif)H zoJKISr9=^eamGYP=O}%8V%n!y0V((g1%b^?{{xq)C`e$5aYL?qYKkkDatRh|LKpsG z+JEFtfuKgrCl(xi8Bn~XXJY0!-fw_FmQZc?0;waOtEH1ThI)IAI(#t%e;$d-#Lr7g zsLrBX2?4Qfh`P*7%t(G|z1@}i@Bvn1jfD>*NOn$>Q&xtJLn)v5XIW>NAC+}<)`2EE zUdlih&9{?y^>}ls(SZ$DL7}=>a~qg0@f8UDp;79&Mq#x*c{fIy@0m5y>$eZ5z%jB zZ!ZA({Q4S*tXU0*2mAU0LPM8Z+_+ow_9k*|J|=@td_49uAoMs{5~L_8bDNr)Zf|dS zLCgYlzbh?n9}s~PC4Rc|9(3@J|Jq$9`K>(*a0=kj{>#Y>#VYb2uGkag17;fHYfKS(%@Ao-R`P@&#am&RQ}tN%8Tw zGg=;N6Rac2Txi76QBhC`#bB{+26T)2p&Dn`8a+9f4HnBy#g(9l($LTt^UxInE?TAAgZ(49 zlha+b=?qX@FU`y-`BVPmk_rQQ_8ULfxw^WB67bUePFnZJW8&MPXf!ZZ8StEwp{4ps*u zI0xPAY-%6`NKvCh>aecVT4``DNiW&1)q0W+hOsxlgTN9*^})j}DT0y-o#w$2cgUa} zc~xrxak-YLLec*ipyVQ@+LBb=nJMU^Gfr0a=A z#Z(jJ<)c^jx={T)-F>^C;6w0c56d*vs@T7CnB6u{n7;b_BUD6a(IuR1!tjlHeghf^ z^?vB-{LmZ!YvAW&HvJ>d+gx)+a#qfcGJ{V9x}z+cR#AzAtrX!2ZF!+xAYHz(;W4Jw z&jdHVapb&xhwjzqrXScf*3};5jaxI}&6e5wB~;({_pcUgL+Elwl820cqS}94^2@9B zr_(5RKFt%~R^(_$%o2@K%2HB{cCe%O@|&X_S5RN#Wk?R$WQ>IQrL0^(ztZoX5kRh< z?;RjS^8OFzqXhfV(8hSZ7`3?gtI64;0NbG6yy!g+UB(~76)8&YC#{dhJ>vUfN z2aEn$17aI1=!a;G7cT(*E|WAK&be zrTo|kG9G??pJLlTFnRMWuG|O1lf>iZn78da%wJg=dU|OYX~GRQ4Zs6@+fk-BWZff+?wh=!I{4OU*F|FwxB6hB$17?!dIP_-B2$Y^S&{xyOA zqYDmjLCHej1)Us_C8l?^+)!Gv0`9n+G9#cjdmzqCl=~g`yHozbeTgrMhFp`+iXSeA zA#x62!EIy)C)NAHSGKxw0?nO2FK927$Iala?swq;A~j&jfzYFmn)-JU$}dF6>%)Wy zpX#zqhyd7E;~Bf;JHNEhe>@$aZ*M)I7fA$&Q1$gTp;bsB_0UAYUf82L56sx!(i{*_v;7hBN-W1(DOaPIl=N`K-!lMY}7PW~{=$(d&@P7c| ze`gRNG1NgI5ftR%2?}3w!JAtZKqF{r1MJf09wyTQ0E8QHiV1yI11lZ`*$UxjEO_OQ z0Qqf%robK|QbH7)^v`?E1P=AQ_h+*2?<|k?fQ{pYCL{SHt*5sIZXUP{=CG(JqZDj7 zbCDUYU?jjI(@6Y|8$zTCWL@VM*6;Z@0rUsPA26q&kGXIXYE=@7?(a8UYGI*CJjIN@ zpzf>Avshc7zmrg%Zvg7(^FWFAP<+VfL>qq7IxK2LDsZZ<=;*hM$K7Pyiocg?+H&Ct zf`WZqICi>j4ar3rm$?ph?)gJqjSfoB=K~(Hk|+XqQ|!`BIS$4*C#eBcC3!|B=3__v zqm%RYqsVISVrZ(qy@p(Vw^gpxfd4`=F34`p#re$pKyueY6T&jTIB7Re#x;*bVYS1? zaZY$+z;yc(#>OF}E4sE|6BL-bV{p`LnYgsbqVpIX5tW1Df6Dn5f{YyREccFVS0LD`kIcR+JKJmQhj9`Wxre7xhDsyJA)SbhDr%O_*>)LELy{{Ev z{!xLygtB@9TAhy%x25qvp8C0QYLTGAQcjoNEHo0Lq5ueNLt7chy4w5}_s``qD|1>( zYMQG2XC|}${ORprQq+(Z|MDEWwfxf_Dn{Z9tT=M^cN`NoC*j1RrpN+ZK}j`@N=|qq z40gb6e7t+}<{21i&FTUTQ7#6pP^7O*HG1L;VeO7hUP_s!h1ypzkk6vgG3d?C=et)I zY#fyh_4#s%Vw2i=*K+9xz|VspO_z`?q@i&?P`>U7Q>dbM%961svAnrOO9a>5#SiC#;_fCD~Ghtd3ovYRnB=^pm#r9KprQO+z2 zk!EZ%>a9vSCD6< z`?Kk9m(SDL#?a;_>A@qJK89fl?00#Eg9hieIKcSTzb(i|`2KL|uFJu(J;~W*INd}I zrntW13=1A!y&koEX{-V0U7^>WWaQ+6=|8#b;fb@Pl(spn6Xmk?FfvLmSvlEg#_lC0pNy zHh8H`r!6>Z%J#?OH%hc{^vq~W^Da~JTG$-s>i}L348**G(qB}Iu==0{JCSpF`B(|? zv!eDj$90klLuJlh`1NPm?vAqN1y0RRHjzCYlkC-T#Y=je{c8hDEib+ypX@d-*LZmd zkUE%3eFk{2=nlr?Z9pwcy~+^!qIo+UI3zfEg>NtV$Lsb}7!FSoBF_8$f9W6TnIll- z4xrv7`&8wuG~NWzryqo!sI~5Zk`ue0oz}$(;dZoiW`_jL#g3;Lql^6pfEg=2R#d;e zs+M!a-acSf5m9`09v^TSId@emxkx%XFHYeeD4{<9(mvSq)iOad{(_<`|DKKN+7h7N zXNa3v&UA;ZK!(h3$xrS95+FWn(63a|s{`@vt&^KwY|u%QHtF*sipy2|Sx~JxpEDYc z+sg`TE`yN$5zMBR==JuN0jYNrR`KN6gvEy3;C}!8{QULw)aYLwu#MKTyz=+ma)$7( zy~hD1JV1dT^V?Nqz$@ZnQI-Ehp{Au~sjLFo)Bc@v?_(py$h^OP(L7#TN(YhSC$}p% zr4ogEi;R^fhq<`nR85>Z9~BJ@0#xh|_EdXcKE!pb<*o^e2CHCHReKHzz;c=hcg8o*&Ovns1@U(K0edP=PpRL`T&uFi_nr+E1tRZkE zrH%&9W#CtHtIyG_Eg1lr4okXE@Ph$*x&34@Z-sV^B{%pIp*6P2K0b#Et9JB#C)kAq zjOb>Tet?|R@==-wyM^XzIwCfbs^}}PS5wKTA{GRuJUI=4aS<1x>uB|Lo-g;*g5Z19?@DP4=$;>G6h3~%zdqVJ}&O+{{)p>R?bX0;x@$0 zsb`6u;f|8yyDz%2q?oQWdop<#A5qvt6A}h^!fiao%^>4PSr4_ZflCh!p_4#Ks&?UoZ`uA~#(>6#pD`Xay%^ z)Cxb*djyZ=uA@Mee&JZNDY%7*Q+%?obBLhoy-C}t>c~j`aJZ!Ev?{tq|7w;6&dby$8wV85zy%qTw9uh<` zgBl691iaQEe0Xd(r>*g#Vq-Po+qqk;hqAcO>2}&~L1T{|8!fBbp|^>{cpCoQ79$a# zhb1c(zJ}YRM*UB&_1P~z*ILW}Il5#dTw}@eHap_udMNZ7gTnC1ZLjMvMM=Hm{J94E zgyRDkm~VT9@fYm2XDapv#I<&Q>E{+1Z;LttzwK=8ElQiQbAK1mz!L_o@M;K;LmU^Q zxV*B0Y;AS(85JI9s_5mF!BNWMN@^y3km26(&;+_$RJZFNgZz2Kg^5c)8D@-w&R%mOIY>|MdBarjQ{1DaJ@Y8Az)!& zo?~Lh^SB!1JJG`VQe$Rcx1#z?({8>?c_=8dfj>>Lr_p?c!?3{od4Fv`|+UC1ph^_x`@Fa)`68-c!Tc!7|m<60JkD8#fqfngt|2+v(abf!+W? z@S|&2*=L%4O_qu8(u(WS@!mwdGhutK1i6(g%y~**)+rFIu%GDjaZBbx z?|pTQqDJerHCc2-OiL*}{(V;Gyfm=2nKFM%8L=0O`i#!<2!b{@#m-k}w_w{8v20$; zz|Gy!`ZP=`!R=(%#d-s_gdvNV{p{e;&?pQlHl_|XwzAcs?7Bj`cwfCFTjbS*G8dO3 zG50q+nhLs3&9UX(vUK~uzi$kxujhqi$7Lu@gp$q)-Iqk$<5W4J&&&UYX1{&ev-r#X z_7h`30&%6R`EAIABnl$DaKz>0pO1wJUcpRyl0xzC!sTpz<6tCP#S)MX2!V#UIIwhd zKSd)+)Sj*-RMRlz_ZbzrNOwD!#`L&D4_wM45u(o(@sK1Nc__ed6Yo2@vqU#Ha6>!X4wWEJw z{(b=^d5kovy_L}HFae@#*=Y{8rbdVVYK~3kDdyOnblClMA|vah!;`AVv0~2ouA=rr z675h54Kh*(2c=?Ze#b=1v9P!+DsppkySkRETYjR6$z|Z=eExYLLnfS?6ltcH1|;`z zAw6O6QaFsmu8_jnKy(?FEBftNon1ZJL?!1=!#Oa42wbiUcr`g)>7B{1>(<*F){lG2 zUlb5SMRhaRn8>bd{%7Hqo4YM3QLLvcQPle+6mWaHqbR5`U5t+hW+i5}hyDhgp)vj2Oxvwcn~O@(ZV2ggs226K3j z^n2#W(UZ5txQ$J@C^U~uX7*t>=0uGd4ecb^Qms}$!hRk7+%XDv zwH1{wqlLKg^4{2792(!A`q}Xpk|R#q%q^LX>hV|PT}%lnu$|we);lasp30A@Yesjn zGOdviFW~NT*_@42W7-^tw`*U@l!E;+8VmjZ^}&YlypnhZae2LHsD@bmxn}^ zIcw%DcfRV?c#3ha593~`kvx8k7X;6OCP*3`s(hcXDoW1Q=JycI3F(80I$7Og#NJ-8 z4?6i59&*rbMOtrU`t2VV26gl?N0L-#1x4l_Icfj{OnmpX3Jt293L-z{MiaqL=7JVPno3-6vY5@U@q=L#(15tOG!jt+r%l9nPg6(sH}FV+ zVS$gte^HEx=~s}q7OwfUz7^jM!xNkQQC88_+v=gVdD)iwaqGaV>d{zth}@bYCwWgl zWUQ$I;$^WQ(bw8{?#HCtg8FetcFq%<=c?b;VJIXwH5SY@V2fLszLfbH`8a_@xm!Ls z`2Ih5<_mWE+q?ZH=bqQ#--(AbElD3@?N1dPsn4pB`y{E<&bMO!5oiES)Bs{V?K^feTj$+> z%l@K(PYKQ|8R>W+O;Hys%i6(+5x-jikW{^SM4ET}%O0gbw_{5crsZTz>RzI2oeK-` z{c8Slb#b|hM-YknfN5_QtY8T@4^24vA9H@CLJFcQv5K`ydf58_!I3SYv;O`I362Y& z{D|&?B4a>Wg#;CAeIKuz%iW%NRumii;8puBVWYd}WsU2#+nevXOu%6c4*cZiuUt6x zsq1n-<2ZLUx-)^&8{ZYP;qyzb*ni|CfE zp=@vAO2Rc}QnvzPJm&NkC6%Ve3&_&Q8`7fWd@#?BbB7Wm1XivvLm7R`&b2&Pjx*9! zJ)on1p;#&~Td%9JhuEA!h-e&NBkj6(W@>5)akY3hhP(%PV#OuX>=~KLx>kN~^7QPS9UZ$vgIVv+iO){WB>Uz#{N6ZS z{k=H5;WNA9+UJq#Y;^1cKPJ)V6}@?zOt(kI9uPhV87AYi=2woQiauPNe6}9SO>C_0 zdB`oM#N48CB}+0jEx^_w=%DVh^1!;8?qN^pzBNOZSuRo6_fI;T6=P$^%p3ePqV?4U zwnardF~V-mSmbi|&=Y^X%$Jj&+Su@#U+%r0sSCo#Eh)vPE^|5BEHm=mOW>0~?p%q{ z{APZ^$iBhs?492~ia405oD=3cAg)JJx4q=v1=@aENN=7@WLYd;%<3&`(Fz(F zRVO4Ow6a#4b#NIS%wlpIPY+({WzPy5Cewhc4i+WZX(p#TomvX!(?EP51@x`Py+KK} zH`y{!Ip>eDe=Pq~4-=f321~9sG_CXtYqiA>M)&vm?%nDqy>uOTf;O+byQ!ein@G{& zpiWNbu_onf?aV`Q@7`Eeoy`Nv+fOU)tt{0P;uoX4(>*0xoRu$~+@9YJQa-5L%W|&# zKvBVGGI8ZL=dqcXZ*B8SvUIMmM;z|qq4SQBLuNuLnp*+6@IeV?%i}lSoFuo6Md@+(6cv7rebldZX6%@l z=xk1BIj|y4Bn_6_xq?m?#F{!`R8M4tc zp;4E$PSp}YEQz?;Y<#k@=Wm9KOS7m5j@0}0^&%(dWHvVL;7^6q)jYwyNAY6y>4h9y z7aj(-TN-wDBbb;chtCTkC;H&o+Nz0uT~$Y4VPWy!GuQI!piZsoYP2-$UX9rvWUaUwaTvwK1{mf4+pTh1QQDiS)g$7#I2oY6&%kr`x3k{Yw-MdH z)oo|{YCbUf>(`R5E>6Um<(GHJLqGpSs;pX>#(12Tlqcx?`kgTq$3`tI_pd>t zl7+<&k`T@>}2M4j;MatPR zP8e5lk*|S@TPo?26D3b}zz*-&X_%^Uc%@@%>SMgvzqABTyd7~#I44~h-d|*AK_x6y z!Ko8@GNfEd7g<@5FOsFK<#vMfHcj^Nv~XJ5yINlTg8XQ$QL^<`W0DI~Ox^$=+n z{j@rNb54X};`gKVh!6ddR2=!nWxp#iu#?AGzA$@Q!!&>b8+4Cy64CcO4zM@ zAtVHYGS#^-3thv00fLOyN19aHC?Cgd-<_l3B!JC_Ku|6lGdWh-TV^_o3 zYdZjPXQH|r`?|^OEd=GBw-2?M?akKnFetd6-R5|!D-kXPjnpKq{M>w_+28B(LUn&% zUA&T)kKFv_Zd&O-vgPIAt<8q6-eg-lv7jbFk5g``j>#U3lF#q%c|s5r2}2CDoBgt~ zvI0OtfKdv~sA#lz1D|mr%+L=fEoWkm`hChf>Ax{Up8_J1lP^6zn9BuHJ7J$bV$R3S zhy&Z{jaev3>H43hxx75evN9Le(ym_HV{CC4n+4e!#!V)<^hyKxpFh?)vCSry&i_KV zvG1-0dce7g;|l8@Azq00+(+8|By;~BH2?xaLS=#2&(X;5-&bo8t9}ES1mf9a!#}C$ znN}s`?{DM2`^SCz0Ub@zmphb&V&?qROdIjcrd8d)_A2lh4qEl?csrD49t2VFR&#nH zCKYNM;_o2+iFs;a!Bp$2VI;o(TiE)i#OW+!ph4I z;;%tyPm|KOaRS7ggL9T2sO@cJwcVweSKQl5%H)eVF4T{ji4s#UQ$@Sw>RM4&frn;k zjaihRmuc~06&2O^*#@gnE$PyJ1nqPFiV9Ltgr&3=@~q3unQ%W3S{ACQsmGiBh)e64 zNr7~R<$vpW?(FcMiK+4hO6P}6;n^6 zxv{4R+QYpKsO#3M4(7Trvcg4d)Ix}nYXyi=cmpsuHrG|%2%h}f2rHBXIsqAZz3VA- zUGeLsX1#0AqQE7}3T7%?506B>3Y!;h7~wCd@ycB-1~5a1$F9`94f#7h`&~)S#Mha6SkY4-UQF zT~u7hLJ=_p4Cu`@{bQ9!hA?qGMDs&kMvPRT$BaVc$PuBW&1q?PVT1~bN! zqxeUt>bY3~sJ^;_QcMSbB&TGfmbLZ7k6v(_n42p;Y55TQVDIug{cfn+wm_M+1(Q%M zY<@B7>0YzGBv+jKb7x_bs?$^UP@1Ph%5W5E3)1vwsywX68om);S3aG2_tv~h%jmpU z>ztR5FV1&2qAA``P{(=wq_RSdv06yD-dh9dfGv(@x5>Oin4vlKugzIMHOZqB3pWV!?ld{*ba_(uT-q@SBTrt^v}g|8=2?E;p;~K6F|B#OU+U|dQ=eAnrYG0)h0{l>mh%nqz?*C!| zb^(Ye_tQ}jyrGQ3Sc~YFKIe)?kjd}c;|WU%ArNYJD_msaHMDtpOMVle0XJJ zB2uB{splnsSJqC4Xx~&npLs}nrWoVSZBbmy$Zm18vkuc)sZ}BpX_lvMk;guE&AeVa zAHqFT!pe9B7+as+scZbz)Rd;&a{KF8=hFVyaLwIijxYBvX~Hcm?FyrA?jLSGm-YI4 zjJl;29yvLLExu9o`aQOLX-1|DY|WS5(`7{CbqddpqH8<1G{%86%G3|X(gVp-dl_PV zT>g*KOpQldV@8R33WqvpJLV}7gwa=T{^p6M?u;Li1`dhOBM)rw`;=rFG8jELV4=vN zNV5GSzP73w8lo6NY>Y|55YWZ|jXgrP-d*LpwLV{T#Hho6$DTMMy=yZ=R`#uRsRDHWnigbH4J|#dl_PfAfMOEsTNM0m zf|3>(PlRZnjrUh10p)$@C@_P}B@@)T5S5TXKh`BAT&s&Z6_J~fkx>qL^qHB}IK9Z+#y$-zG!RiYJ`~db_AHrqm zJUuHOz1h@$5Zr3QJzt5e0=V&8&jUL0&i;OLE@FCmdLReA#Sm>FBsTm>bh2Wp%8`Ge zybB17Dln9mLC%mgy8ru@c`!hp)zFaQg5#kJ0s2#Q^HeBmmw{ex~#I+YteL-Pf;F zAa82%4%B7upr8;C5>oqZZfz;2iW$6kF~79rf1Rts^k#c{ZGBx=M`vwmX=!m$4cI}T z2mgZsg6xE2@%QiF0jvZ3pPrEs>Nt=bd%$`2+(pdIS3p1&ZlYvjb9HqUr0a^cLsrT` zJc1)GDJn`$R~J1`p^2O5!|pI27OlYk7J3 z?CflHbv2!AUm_nXGjots8p@qJ%$#^VU0qVJ)fh7|IVmJ8eBZyB_Ia2JQ#2u~CYpcq z{=O5SvS}*m85!6VA|G)``K+QtLlx!ZJ}EFH3A+`-`sJ&Fi61^#rVSkD~m6h@OOW_g_@cLey0P7Ep#K*^ni}L98uaCg^0iy?P292HepX85E zy=M3*)HrB2g*r?+@ff1k78cS~m@4e&wxM}eR$v0drWDiH*0uoDa$|#xg2MN7-!wwT zk2U<$Cn>Z$ckZ;dwLwV_B0@qwUS3$U)&j>^0XYRw55r%o*nMvSPBVwNzAPUz{I|RN zxqC?}%>k}luo1|>NorLLInSoRDRCJa>Ly@ghgGSSe`c!^ItDNm~i3mqLQvU~~-)~omA1Ks^ScLMY~*Jhb2OjV74hjY{-A|mKyQ$#%Q zsQ+$l`JC*|!^rbff&0?dCf>gO`rD#Pq=DvFcX#*D5YVWPsi>%EXx?`(0rv{ID2eg){cnf7T^Byu`(pS$jYV-tpV~H78Ep5?Q9N1TuSPezmyIf{-H^Qi6AX4 zZK~2fduUB6jfdCMefzfC_OxvFkdc|rH`N^ODR_!qN_^iUZU68v zf3`*1r3391WME+F>F7STF2G}NP!NKiha+`+OBHE1lERBtf$~K!|F74@(DTU2P$U7t z9hchqNzR_ZkmCzO6~+Ih0|@L8Ysd* zHHGwb2Z%USqkw4K*K?)T?0X+8J-rjxK>BYnh z?dwzW8`z!Ktx?Yn3+wxcdoz17MdPkxJ;1<9CjaXfw2AQWnC$yfxUC)V6~t$l2tN7! zy7X@}Ouh=pxYXM&`tz^+U9Q~kWbprCW7HFzrmv6cN(>_c`#!wOsK{bDR*3z2Pzkwe zr#-2{3Nw2c^x$&g21(~Nk6u}OaJgkA*BB$;?tm@RIp537dqX|f8C>kPRh4fnJWVNu zuu=1|B1IDRLaP?c7G`H?%L4O+zfixHV59*PHggtwU1?hsR#7cpSz?Nw^<*DbSf^cf)^{Ah@C#DQJoepab)cet6%qfgf9~)865knmv zaL>nW`4#^ds|1rvTfgy5OsXpzpJh=Sm!SN=5>z{y5?5u9*ym*sfnZy2f@85wz04!*VH>a&^V666&L> zby}-7&i2GLaOk|`gg#K2z_Uu}&;W^uP&6RDKy-c}bAzy;iEJ^@y78rvF|08lg+fTy z)j(0hR?$^&YHlq*Vo#ga7=N)Cadb2P2mHW_oG#(``?uKs9_Go?KUGu<8ldtcD(1`9 z-h{1VnaEV1{5x*kS93#f$7N+_Y-^usK4_%l$sQj{7M|FDvl`YtkqUDW#k)nAzfzK zTiSvmAkjU0eNNUB4rb4kn%iqy4fi)=J$r?m_Ele=q-9OxDLG2RY} zGAh50FeN{nA$Q)J|KgWzu1SadvOHgNY>hv4l5Po^*vVtJLqdIALf%jS%TPsW8GsCx z*G9U-io(ev=Nd%e8(1&gy|BL2b3$J1Rl{^h_}#U~q-#FaVD>a-qh_zqb?$cAwETYK zRB5xw>`gSzezj^{xPiX<7UfUt&&rYR4;lRfamR~me_(hZ%L4OBGU}P9r%=n;Pd z5c4pz1kwY&FAHbOo-u;816&Wy83%tlj*iSi5xAxwuVjo6-;fpE9!B-?XEa*2Cp;dB zE>!mE)cG+ZP5xppcbUCn6&q&=5xLiO#clo=)@1c_3Ra zbs}PyJFD+UqEHJuHEatn;aB@5Bq&UzOeFIVzbo}3oV zluWj^+DY{@+N;6Lgrt%r>Cs7ngqHS1hrVHB^uY2#y4AP&@$sTJzcT-d?fA`^82r<# ztU)Xc{~ien3s(~K%%og~tg`xBc(`2JG3YFd#z)6153kP+V%I}iMAb(d9hdj|XY%rP zDktD6%D;!@nx^IuV<_oU>B69zi^hg6q*W^6eWyNK!}=Epyr@TJ!~c}*_>vXL;DFPz zqD6w}xe)z-IkRYyXBU#Nw_o#bidKgrQGbemC5U0%Rh`tQ<}N>2M0&f z$;6CRK4X=@Y^bE`Rfz~OeUI$3wt_(;C+8#+#r?KeuR~0Xs7H{-tb4P$42jN714TuRb4qX4 zS1nw>zVhTzaBHY#+`e(Nd*YQ)tS)w9#X_#@*PWlOxv-Uuq(XL+*8|7#!*OYAVG_es zG1PwBv$=H|Z|+hLj}N#!r&4{u)JXG_GMD+MIG>EEIZyvQqFh6i^g2V5CGYCV;MhCJ zB)BH_4w8XQM%W*Ywl_WAAv6$~8=St_=bK|{934LSv$TPH`(0VZd>kbQjURX8<9~C* zLuhG6{RV_u!xtGLe|@|9TGoShIAh#$TitRo725_#P*V#gW+u~DJL$%XW>*8p;UXz9 z8iFr#W3yvy|Hni+7_FJwgD$4s2h_FsURHYbR zUOp}R%U$jL<8qBd%tLNrq9k7|JS3#Ejz1evQeLOy{SsyVRNZaF{tv|$>{l66JhLEE z99@*xiV(3mKPOr-?rv0l-Ov*;EIv+DFK985^lm;-I_5aV&`@{~ameIsZ3ybTT|I5M==NdmeE#003O z;U71z>fieB49n&kmL>*>TUy$MUG=ZuCK3OYNikJ@J&-kmeDY7gz0jkTCeM-w2BqzI z%?1pcYs0<0?hUXgb4|@)|0VA2nUMz8NOU zw#`-+Ou%b4BWd$aDE0Lh9($jY=uhR>TuE_^E&mT&UM(f##ljB~m}1ZE*7D7iaHFTW zN!p1Ovo&UON-D}XHDCH+J$1>8%*~!s;mpg)U!pv(-g)oA5Pm${;JegD{;p&FkU2B1 zD9*5fgQ=RnpuXri+3DA`oa$o#^e4ZEhfk;%H*;FN`P%NmjApHJ^ih4fp_-k8gqomu z@OoT14pu@nzFSszWqqrySvgc#bK1;J@~imKs1@IH5gY$X6eOYMJ>|<2xw4!O-GHD_ zj*;)X?#hY|Yx*PkqBfSiNY!5B|-dG`unLaeI2*g>KP&~pGW+E8* zHyf`oGLp4!MLBuCa}g}CsP+$RQIPQL6Ewn@kLbRH6k;#FThQjzOv=}ri3$aeAonRO z{&qK7LPbOe0Jfvp@m!HDFmoNbKJbL7@AR*fal+{aSfOTdV{Xr z@-{1GMzO*8Z`Qg~1RIN5MlPXTc&c4Y_JxIIvU-Cxmqtra$LDks>#xNdbN7=sKz1c-DHmx&ct?++q)jG_sna+_0KwI%ai?WU5ZgR*t#29fJXgP zPdF_{_BR8aYiN}Nm}H);#5;B7_QtjkDo(Z6?TpY(Pa0O~V;|zL`J6@-a=s!B*A(T; zVByIYJh>D&xA=`C1v%4QrCePY8B_P891$r+#qmvRwK6GE@?L^c2s6Si>M2L`_lkdh zsCPTRB-TXwIl#??kDD2qx~20+h?RCL&qGZ9ChlRK3-AqVjk_=0*A9Z&qstGBNCmHe zmewrH9yeR^?eBJcuH_vY{Y4$5uT!(5`iaCSKzk&+-%!}qA+xT!_j>We;Q4;fmj%9e zef(-V3yA?$fAt+h6=w6}AB}|M=GCtnB(*Sdn67~Nb3+dj#M@1|oX3y5HCwv!gM?xW-v z8U-DAm!n)2u}x^7|4v_w+*>2IwMCbbQBxc0aL-@=n}||;{_{6^YJSuVc&Mu4ry_>( zN$G6Y>+w5gqQ>i0@>;BDQqGIHZr^Jr(bClJYNfE&Rr3sz&3!yqJ0tbIa+UAXHunn4 zFZOTcl|akfK8hLU({)l{V@T4kVw)zaCngeuo42~G&3N!enAVwPtN)hGD6w}JDqM7) z%`QC+#7?*4Q*ynnjaMAFchIFe{2p^$_S_kX6kfPyGcyr6+Q1iolQeC0)TfBM`1qLe9&(F{{qI@?wwmN`io(tLuWM@q2QD~LEp5LH1_WEb z`gC<~j(Z4O^$mrlA=Y4T8^Yepd7I45UegMXRGDSSK83*i`SbSJ&;8_c9vKH?8!iD& z`x^Jp#rdII-~F%;xJ*ip`BCcZIzn-eJ(bXgH$VM+D zh^F$(z5c9!e(34=N(a%Vqo)9end4rz`tUu@zv}yM#xE%$05JtonpC z-Wr1+r0V;0px+TBhA2S}07G#}3#DhTO+2LoKYIFXPnzOtw;kUj3%@NaD?IV%qapGf zYoS**QtF@5?$uZ1UL9158#a0_!XY*4$yXu&G$+VQWqWu?%y(vUe>uYJd+AQo@4)86 z>l1xCEb9``u;y26B)us>VNTbemh6`M2WD!IY|ho+-!DlRLFF`}$;}G&Z3+bDqwhV8 zEzLIGYegJ9Tn(joEEUL|s5CT4Q3w{q{+e}Uco+&exiCesF$(sUYRxK4iEn-}SNii~ z)_WyXZ^=zve2hFtSSnD;d^LO^4Nl;ZcX5;uQVMRt8ir4)Y?ggr@x~hgOrRoEZ#>AH zvv6HM6-Cp6HI%5y+_Ns>a7xAizD}2sQEakxO8x7tHgGUn%Fn#Ku|!Nd$eU*FJ>>CI zaaJo;-;XCtQ*3*rX?yvjBG*v=#nAY`ct0!?B-UZ|6Hh8{zeIV&KFi1E%;zsY8alTINt^At5^B`|)E`;Jq(kuG{ zVO7RQXMa$_CZb8c;0h6pSP%tXY`~u(AeGCibYlZ=ZlZH%k-lpTUohYG5o@&lOkFaB zJ#WoEMRUw&WDZ2zIKoK0+I#iuylX7C>1*S{uWM&p_G$#?i=zK{*5Ut?XWbW__^CL0EL8_<4XIvx z?MQAA2Z&imuAv0)WrVDf>gv*BrtHNV6%-zM0%M^5Mp>V>L5)4C>*Ff}g_ej49tJ1lAd_Df zO3kr}?Y`gs%wnTq7es+}BdKsn&Yo?Msn3w{kA9V(d3V5G=+8XX?BWXALY<^~C{mra zFGfPsU&AI#esN|^p(C4dS$aV-6>pvuWqd*ig|$dc!<8h6S@rR(o~dW;txn*Fjw@kd zw1_?A$X{`!!Nhz`tQs0c#S4Gs?{*l)!0$_6tB<4(vL-z^2sLT|^&WSq1B&5;WnZ10 z(lbfG1;D`pWnxO$*1^4sAf>P}DH`S*-0P!Sv-- zNr0;>=0L3w1R>NKMYkzyB%#65%)pKKK7P3#wlIMz{{^WoQS7`wk$fthA%rY%TLQKX zJ*`l0!Z>TR%$Zu~BTy<3x*+z28skH=Dai+etg!R^9_Q1?afiL^p8dSf!SH8drhP_Y(7wR+F!;`&tl0(Sr#6~r*_3yC981n$n}^i2kf@?=x8}}# zrb~dLtf{F*Donb%gchIuFj2o=U+y%M+4pF=5%lwGFQB}%phbIKHM)yS@F_4bu(Y%k zKKve4k>-|!{oa3qKpOx1K%}>Ej|P-`Z}t*iqJhZ3%|EOFM7;Sqj0#EU@cNGVAFZ|j zY&!hRGldGKpsc3X;9XsF8xY60ICM0BG(fPPg@uJdAs$))K0*aLJMJS8DFUSuAucYJ znRoD}#0?>sTU=R*INz#FP{{i4j}w-e_fxnA4$wkf%yld{>Dfb!J&-&>LMnep@&8=I z&HFbSA2&ZGYyq%@pKm8|*0GS@8md6#D<&q>yB0#h*>U|{T~0185Ru+LJoJ%!6eQjJ zk}E|Z{`+^P3y9?PrHWryXSKcqwHEUS8y>w7*5)M*ZGA1OMF+|W;$NSmVBEn#ZTbEC z1sxs;)q$sC9^2p36WwW|tjrDQm9+G)`zUJK+7L;HmxZ0(Ro&3SPfykS6;~+9_fa~1 zB^288?Ct@~O>b{XfQJA~prfbv&s_v!*rcSNa8#KHVCje=N#zGxK@cp4AW(QXb_zg- zA0X;p^-Z5cSwSHrBt)Hw0Ag`eR1zF<5RDFoGjGX%Rn2k7L;V0XLxP~rgmbk%X{e^Y zK3jwgzumMu1fl;Eqn6B6WhVHf%_io$9__T>(IFE_H#0L+Utj-3O3H#a5&i)D1cYdP zeH-*SK)y#t?Kb#eajRL*O zqb^U_v@^UbO7jv8h0j#VP z_|VkR)fFk132Hb>3JM}VXJr`~v~k@&?|4YzOJDrw1;=y5aRQSEijDB?K|w*#SfJ|T zR6X6=x&-2l&?b)_MJQ$GDS_nUGL%kP@O}*fkI;!!bJW=$to;4^P{@S|4v|WOmc4WT zfz3=!j9exIBO|A7Sx0+2BR{_;N1T_ZCxxhI888$1`5{yvfBljsP9sF(`V7Jv0aTH6 zIiLrNhA>~mCPcp@iGw29vE*bbJI($MrvxzVN5-g9-}Q8NZvmnK%8Bp-El(*KEiEGI zVRi~09v(lO?c6m~BPuBzKMAUjAtBM-OM-NfbdI9vz*4BN=~b}N(?8X@m#kz|`h_fZUD zAro$ro6846-#h1SQg53*3@rdF;M`XnadMf!7|?Z=lyLJWDQ2od4@_2KlvQPjg1Q|L zzRy%mt$dEG_1>s^$>}|n1IDd)Z5ZoCXSnWd$rC?_Wlnx~K6z}Gf5+}^waK;g-gmd(x0&L~D0no)Ef&;ix5 zm4FxdF<}EmPNJiSr>A>fa!EaTf{B3vL=2}vjk<Y0gP*I^769n2o~PyL-@V4qy!xuy)BG9_;!?BCIB$r zVySeUz>@{~`&0Yfx^?RzIeDe+RDk_72-iuV>FDaF0CR+`DGnX`9(eR$K}OVph6V=> zMzi?IpWA*Qp1QkB*nwqgNE`nP@q|P7gCNs0(*!-URL#)8o*X#y0E&!*wOLLQk zz5y#s@$72+>K-s5{7Lmt>l#or5O@9j87ECeCQ}tc^=Q-r0s?@h;h;G{U6`Kf>6<2? zrj}%hq4Be^v0-CpPf%FioG4{yV{<68;Xn89--H?R8m4s&Qg?R@Kb&NxxK3ztQ#5L< zKs8m>_c$k0Nj7HQKtsqnfpNiu!=rSuce*GF6cdyiJ06##1Fu0W~ng!`4&O7|hmji#Cs^Rg0 zGWt{{v^-COf|u7hJVy}sMMXuOoSK6Aek?I8aov`@lMAiE(C_~`RoB;xro_r+mQ2_n z5C|y96&SC-{2^@^gbXBoi-W{a34d9FsE67Af%g})WVf`ye zvIV6^{SR4-d&?T)FUTkAiEewnd$b0@oY4Prh5m=7?2T#sZ(r#q(D;AZ%A4Q+Cx`Fm z{r}h5`u}+X4@o-N2RB%xs;Vl`S`_S`K?piBa?lMKG>_rMG$)ri*dFj((cQgjovxoS zZM`hK93D3#BV)1l=F$=&3CU>>`cs++eU2Ruq;REe$Q_Exe1L`mWET~Vh>wS3LbetF zj2gQ{?<79Q#Ee9#Kv(cm-F#6M3e?kq!!u4`_m&dwU%3GG3J@YZMA3XMR@Uy$PLR-J z;^iIg?%trM1*Z*qK|h(ZJ$Nn=a+&>n_v{{v&LMjZcATq&Rgj=^pEXtj5-uugh$m^3 z=a@E4MG#~HK^fUx^Q6Z(nk7L2W^>rdG6z2qMs2$xr5|zDddJhL>6RcJY9n}D; z1&*8EDR?sA@Q66)f;Y%&q{c+x?EEjK|Ap&hyAViSD~n%7Cna^iF0QbfIqN|OE1PnR zR>(q2+cG=r3nRaujRX&mCJOTLJBKP|^!2AR<>OKeV-gZbJ~aJgHKK_)*gd!JjHDN3 zKKc%h6qVnNGGVq+88{9g@nUKvBPTc6;ByXit(W$Eil`SLmjHqRtfrJAavF(iW@2(+ znSSK*`0?ZZB<00eaP_?0Kf);IB+xB-#KV*E+oIpA2gD}DJof1VTRc3M`lrLg9zI?G zIPi7rCv#08G|vw_a)9Qmb(p`k-W0J_ZDL~bMbI&wh}D}AV^o@sjg1XvgQljYzP@d6 zUBT?QI5_-VvwilgttWv8O1e2zP~hQ3Mo0VMurM*j^O!^3K^b{@m@`UBOE=ySd<+VL z87m3ikBK3?vW<(0;j^7AlZ|44`6`kwAvU(c`_u`fI)S)af!E)^-!e00GgZAmh6D%0 zNj$g8$0}^6jx2rx*UZH=(An8JGNQp}q$DpN{OOaEtE;P%6JC(CAO?tLf^Gei`Wqrf z^YimiFfIJ((HAkF$&rzf?r!-*;!uRO(BsGRPf8`Ic7d05assDWHTJRNg0z&>dwq`f z^Z9+a1~#47=H`KjR1j@mKmzBYmYfPE%ZCpiI%t4nb5i6jCJsIX1H&_hsHf7>bc~E} z%|uW&Y>Db*& zPfrIbIhIXFD)O5I)lIwQ6i6w`!jGb;I0J)1#PA(6j@?G&lS9IR%hC}-=^FXfI8wxy)G8y<}Lw! z(nIZhP#0u}Vbs+n#}W5jT3QgAL-7OR9F`Nir!e=1fHJ5Fuj&6`{zKnUF+g8B!!ONk|ex z<|!dj$UKvf5Rwdu@7Vp;w|@V8e|&qb=Xuv#`?>dhU-xyL!*Lwvd2Tr8O2UF(37fKdXx*Fq~AqrFE%)clKyk&#aa#Y(Ie*EZm= zMMOmMFK+ZwlY)&NB?v6xo$(^g%`;AN*WAFy*xP!)bqeDu(JYm!y58aT5?-`}1J*J5 zh<&o3-;UvBbYOpU21$NI1iSkpgCZ%edblWyI0)anm}gRJ8f6v9UsqQbZF^FXQS4&g zw(2{D6;3x!ydmTBUs$H&Wf zu8Aw37I`OxGBwMK7h{u>_PsjI`*8FF=_y1NN~?E~LxnmEKrr_B(IK zTx$4Zet!PzS4P;Vm}X=YVq7dN&vgVHZp?NVrz+|6jJ_LRMEG;&-z){MT9`STzMp91 z%0QlP(VG10mkV`(6BNEz%3bsOW?`}+jbB?`ef0P-ufJBx3COFpGdmF2M&DdtTLT<` zuC@sK(Yzgkj+-;xBo%#yI|He4Z@s9&8-P0)_P8|Filp1kZ6zdxT3%kBGoq)z|NHoO zR8&-f_m-Ex&bf0@DgQpNohlBFF~cx@XhmMJn@EJrqzD%PW`KtD=;n|6iH~Qa3+fmi z{&X!L^B@KW21*U>G&RGW-uh9}e~HZ0PC*-7M`h)m5sld|Un16liWeV4v0^l+Z5&6v zD9&HbrGkv7PkrmJ?IVrP`cgkPHzxy)yck<}cUxOqXQ#hDW3uk;+vPbqw&0eh&yao=8EUikNizA@_Pa}e|=qG;v#arD^T zz&Bu6@hPEs91|17Y*;F!|H=^mxyQn2E#4H&>=^6Vg`ggqSMmoB(jPpQq}6yMx&GRP z^XChTi^ngdmVH|EN0|z)4p&4*Mg~8R8^c|Fhz{GOfByWTIE@N{#f*1RBKpf%6$cZ5 z3Fw=s`-&B3A8!or3Y%!IH=pnL@qPBepFf|i&=g`sdX?@pF!A-Ppn*`rM>~7_#^vXB zeQ0kiD02P{@p@EUm+(kgNy(^QncYwscH`iia;a!sM@R(7$FF#)8xGl@nwn~tMOGL% zTIZ{`?x#WXu0N!KPR|^4nMFlX$Bz#U4_6_=J#oUT@&3Lbdai~usNSxI?)b~ifv&Eb zzi0bUR*)?3ZB+Wcq!NV%*tzTL9eyA^m>t?=hSkUTjE2U>>`#Q;Y-q#35MaX=IW0_0J8*S$Y(`XpDB#xZqmZ`gy-S}^67a6E5f}QB^cOE3Inj^xDe=szD$byx{(fsy({skg zBRR{YU8Wz9Kx>yyOH2DKrD&ILyf2S$8>kl4NqX zh1Sb@Z+hzM$6y0rzdnKCk2tzRusIwYgSJ8SUAxJB2B?DSP#A+81``8#jrP8vB%kp4 zPez7?E9T4iIO>}m|I`k^p(*zDE3`6LfATZ(#W)e9IexV!`4>@?fiuB_IO$9#3h0f( ziT%~yTYUug6yQUah^I)aHX+N%5j8k7jwia*`u zP|pFg0|W%-0VN`M(>(qtK2Qw)Xim9vCy&3SBwMl+U7%uWNl6J-Y+P%5oJO~3KYpsQ zz8=s(v)N&RsI2whtWZ+ZDKh|7Q@(rkpbL2BQso2oO)MTR2-l(MdF z2ahCf6uaQd3x7qPh{m+DSp*(t<`~>9TI9*Cp&o@narZqlP0uOH8BAL{0AE`9FY)L;{l>Y;ubODOD3iALV9-hR0 z0*Es~ohWsK$!G_^nipz=e0+RxK7KAP>UN_n38h{OVSCgXdi0nD|T1W7_>rg?Tja<(6gQa%PrwBlA zQPd@8ys?Hx;|z*`VZ#B%sq%04+1420U4{IwOU2;g#oz(XO~axQMXHnxM`!2I@bLPN z(?UQxOifKmD~Eg*?b)#|cACmXZ%N&`d-D`hb&zt*qO-W$R&f0__*Mo6pJogbp`@fN zFDpao%iX)%x?b4W+5$0s_>dvtqX5BIJsla!+qa5nx$aL4s(aWxab7sOc>=f+=Lt{u z`jEYzCyZ|9jNcH&Ab;1Z?anf`0@Z4^^KH8bmw3u+tp8Yta(B z_$_%i2M5XFB3DK|&U3FHU%vPnBw@&~os>Y7m8q!&{tg(Gr(9uCQ9YRiOW5F`Av~sR z(tRT7#5j&qC|bgfxG$p^x6MrHS+nhZ-@WTY(yu!lSCLzVazPtL2KR$RADthYFH=otfaroB8=9q(6rWNd#-@m=g zH%4{+WEgG%PYL+5ciN>4#dO8RKo4Q1;b3tLK7O2gcW)250%n4LFnbLIg((y-O03lI z1b}jM(?|7LB>#y#;KK;z(YYRzXxh_GkFTe=r~);ip`)wvT>A-6XlQ6ymd-{<4Gt!p zInxYi+1$dy!rc6XkQaxvmuR$l`P>ykDxmO4e&hh)R~BRZ&kJ8rQBKuaNWMqhO&x&h zxbwn$=S5jrS<&eo)kLsvT3T9|zoSicmtW0OLswKKlK+)D??ZI@_c__t-5tmob92n2 z*TxmKR*Hi1@+oy=&oVRRrH|?$+9kE9M~8n)$^KYzQ%=#g!9loqCF|PR z{gBXB6@fbg{y2l`CKuVE&dy)$+wb4MkMqb&xCYL~$jFG1dZPW~`bBiCZyX2w^oU=r z>yz~_eIrtn`Z{hd1k&%jyO|C;xw=ZDbm**_FZV)*r_HM;DB}F^`Sa@+FQ_eQPMkO) zqW6+p?Pq?Ek>EMAU;cU_#1}a^^?R|ZD2>{aw*SiRcP58+@7c51b4V=`4q8|9Q8a^- zOd@jCb6Mwoxd(@akbz>2AN{wef51o-z}Nw%8PrDsrNP4*A-)BSm6A*^#H=1yhwlBr zTdBymv$H3_m?5A9I)Z5I%R8zrsfAucR9Su5cL{j^_!7@@?%*o7bNu7t>>Tyi`}P!9S7-F`b-xUcu0kUz>q4spYhm5AQ{hsU2W0#i zc9K(TIo#d8>&kmU`c+}SE76n(X7t@Ze3rW|l>L)m`_kLp47Zjo~5Q>oj?olkv189{$}6$w-U zih_J$L-f1UB2jP0ulD&`zMR@!3~zvc$nH|teKceze`j4XsIIBGSwvWvt$VhCW>Qn) zl&_!Pgwq=A!Kl^FlfGwD ziKnz;yvFgH-x<-?=JqY*48X?4E0O#-mk!R(+I-XXo4UEQ=83v94EyM^4)S!^0bmb> ze#!4qZ|0Y4CVv<_6Skb&~9CQ{l-8l>l4BLI4l$Np%%F_UcaVo7SE0Yxw39TDLD$Ew%(BFS~;ZYvy zrjRi6@hNc=F$oHBEA6LH?qI;^DyL7^X|EmveBj?C?pMi(8d4BX*d^u#NFeSmsR=|W zJf!k{o|{XGUMVe3MVi5csi$8{L6lAxl!jwta;U^m%sn;vsIve!_qIngWxQ&z8u1x8 z0Th{X{8%{Lqon4{vImY26O&5hv&%U~Mb;OtUBm2?&Y5L9<8W=?)~WfS@}9ZlJTc9% zHuxOn&~kQ`@N}lXcQ$MA_4QpTtz51Xie6qPlyy-;Dew4&TrCoAJ5LOl7R}~+)5}>+ zW@m+?mEKGptlq$jKM5$o4K{&a4U16A1k;cIgBwB8g#gnpq9<(K1GF&rHnmd<=Xv%> zCVb4)N?QM(ol8_)TwDxg68--DJ1T?tFBuziv9bz9HJv?oPMMdXB>@rgtK8h6jMYb4 zl(4QK6Ys|CloQ0;1opka(8|dA*h5)Gg_?YOY}Jmkoa}5kYFtsnngl+}CIJ9LQDWvv z?&n!q)3dXk)?TpRu($^ww`NIhLI`+72jsi`{dnB9T|v3Le`qyURq_U^rjz9f(xI*;;b8KlcQkK zk!=<4Fn{(2RoGcs_m02sM}pVWL)o}8KVolfZEb6da#o&Wnz3{3U0pkO_@e;x&Ye35 zwtHS(@p}ipQ&v_s_7qY$4-b#Bva(VB7R3_hDJ>->C3~bD`pkreHnjFZQnj^theebE z-!hpzjraVXUPyvf{4G>hd1qX(=qR$4mXZ>8s})5VwSQRbXQW>szG^BptdZ*BC&Rw? z-s}-Hp+gIG^zwyo-XPrfrFrlP9y&He#(Pul90PzFo3*$| zqjJ}9UB++3LTu4}8SjrDKdy5}eq&*WVU6BC#?q`Ts*u?X!3@s0pNGqQXHVWm0Wne{ z-^;y4&dc8~5IH|7Wo$cl-R?ruSHcqK$^B0T5JgBt9=ev7ZQ>y6*B^H|khf%STzjU$ z1;u+$godlVQyxALqD!3r{@pkJ`qvE><*mO-<;<;#R`n|LGD9?QZS zj3yT35y3uWRIt9$+@bX>oeSrKlN&OOZI3ehvD_i&VGrGzkOwF?t`FG=}eNzauKYzzX>p6 zuKB=e)4o+sCL&ZLvh?xD2+bN*Xl)p8-mo=x%%P%*n>CnQ=U&m%Evl`19($kW$woP* zGm%b4*7+AbGp)(T|MYjaS4;E~*X(CqkA?Mwz2Q!asK4e|YS?;WWV;`$8io~V9AmLn zkfXR5ycxE`*Phe0e8{>;oqzYnrDYFaS_Y8D$ZgUBGbtq5$+)*MdMt~d4GP#-NX=s= z$rg0wt9ky#-u(BQ>1}a!D%>|q4FQOIR3<+ayl5?H6R&+)b3N?hfyY2BErhn4fpXpW zb+e$Am!8~k`$YMF-<(Q~o}Ho#!lWd*GlOpL>vN=y$z!)~ggxX6vX$1T{N^G{FjzTQ zYq_e-`&WJ!qRYx_*Lojh*zI(ixvTA)_0qyHX_1kIDnQuhy1?y%Bcip)2(L1~I}v~^b#ij@>T)2__a@P1^r6IcfhP=IOyCDa|=WuQ^@W@PXKTS zjLKG$@h@KoWOuKws3(12VPcGaPYW{z1>;RcOa%2Xfe5q1;{obM;VUY_sHKqK$`;A za25CH-Q-s#*?`-F0c|mtP>TeoCl-Uezi$rv^3~ZaAe<;2{|C0gJK?<&M*XrL+mo&^cS1>_NmscmBp zC&~A@#$5FfS*oIxQq0iGbj}N+VB>tR2aNdf;|81^YpwzrCbqpE73>_H+hdvo)W@7k z7oybmupNzP1WN6zDz6dY(43TEMWiXzy>OxaR~k=(AjSA4_ZO$1p<#%$bhlkGk^`|z zuQ6G7cDS$7Y_ingoT`yBvHJF zSkUeQ0;z#Luez*@F6T5)T(0!EdFM`Lmo>hYE{L?{RbgT0n8N|%s>Hv4Nj(!FI$>ok zMfUmI%^3o#T9b?AuA--w2Fu;-4Ga?0(`Av5W8Wz2PESuSyfb@q*#sIgx?Q^l5YCM+ zBjl8GosCsbKb-I}#e4IpW(UGKAeLC|ry{)moz^#T5sbUDE@D)c!ZeYI2_U5`;SK29 z5n3OKgTlmQF3DEC@$-zUt4Ag0(-eiXL3I1Bb!rj)|LGC_rEs5HTQT7N=)r^9>1QwH z=mp;>#qcq@APL)k$&xb%n4X0 zspwmR1P=ZI@iK0KI~Ch z_4Mf%uyYV~LDq2W7<zD*Pq*eM`tMD{`0~UOKZsek!py=h@f}5lvJ0E z^bx$DDX4lp8&>u(GZ*8x5fBpIHAxEhrUyS2j8+HHs;1jnc5{Y&do6<5BewnTTqgkp z9!vU!8F9Tom9o1O2_f(a-@jW!!PYa=Wi92&|D^HHvWqL}^WCKkt9=ULV+&1Y2P+GU zU{sUd%Zp^z5NaS1X?r##TnT9rI8q6T4jXBM158O;l(d0DY~{KNl5FOK!gzWdUO~p! ze|;CWVaQ_O6)SM9%O1yn1=p5a1qKE}G6p28jp7&In>&FdSMnV#*aH0hk*8;tmR{l} z){tp-z}{zf z54p_(c}YQ^=a0doP)y;>Pc2>TVL|s9A_2HVtdsy!lN&ZRozpI2F`%#y&F_UHiHh>$ z4Dv>5dH(ciN(CZ7`X)fLA3lI9hqjKJ*xB9PZzIjZ8R!oB5HcMLfJrU4pMr9MlXu4k$OboeyKk$hK@L`xLq!Iu(grj|<{O+(4 zBzh%i0x^PIECvT;em*5WUOkdug8@0}pOtx>idn%&Sj7Y1!!H9{20{$+7NC0& z>%jWqsT7C{pG@99RoaP1T4fB}wxmtQ zdgokme6K^ehq;cT3Mw>ks!X9&z9zZ5kQxQ)2tpHATwU!AD;Co%ivB&vlmQ^}JyiYN z(SgB?<8O+Do>6RWOKTTo3PnzZ{IKM8jXuM^eIGZgnD3psz49YGB!sQv{`u@KFh*c4 z?5oyb@O2okLyIA>C#WtfJKNN^9k_++AwfZcdKo%Z#c43Hu(+cQ*hAGqC?CMnA(6o8 z!V9I`{-iegat=vK1)LjcEYu0a_jF~jop8aHl3st8dkCrMMRa5lY%IZ7RaN!*ix-gG z9uyV^dN08i9;Owm($&|8&STtBO(p^lv@*wWdSc+TKAP|N+tb-ubKLXCma6I#LF%egK^mpygHg|u6b4yQt@1;PM^-bn1AiPreN|} zqBIVcfX~~7wb2{BKWoIDY27tPj}<+C9{l%(>zOGv39+C~Q8wXzA^z1Pr0jdwb@fc@Q*h>_OAtYghM}VJ#o2PdrnL!Y{(=r#za^1NbT6hOCq2&~;;n z)rG6>L-=u57nhQzZIpS+J7fm9g&N!fID;fCcGSDvXd+jDkQan2TvTs{zhdp>I|9tl z|L%{DzoQ}+lbMlWEg8Rm=fL${(|STmuC!52&pM`C%$?7lKd+-h3=@5Q&ZK!7?b5DX zIqo{!%Tv%`BFCV;FaFkU2xP@#Za_$YK#{_aOn?kR;S0Gvkem6A>`na7qw)mc*v9d> zk?Jj=%Lo+_ALQob&{9+TvhKAL0$l^85Ii36=Dm9-Tx9)xeUU!n%%BTgpWlZT&t1io zA?(malpC9EQ6KvQ0gMxZ`dce2GZi$u!P2$|hwuH-tq9Q6R#3`{2!GAh>RBN8atU5L>|U ziPPX$bCTg$*g;DnIq&L(z}UQi)pg!aH_t+YD6pH>h=3vX!-Ind7^#aci+VapJnym& zt#|s?mI}#4@H+juhk=juQgvM1-0G4a=0(J_BdsU!M5`+k{S&5%yLX>8G_=XT2xkB7 zgcFr7TCR$vw2lp zofL^~GJ_ZDj7?6a`z_gG;p9x@C28+Jnmb!r2^EGT1pz4%>-BE5`}sE4^gNLpfH{D6 znipJ3YloH>@fDZ|uq!mf!PLpy!lZkz?47Kn?Uq2a8HVOh`)pa0DWRc&uV4!8qlB4- zI*Q(VRSB#SNCY}gIl}#OkcM;{*w=bWMxsT6G3fU-bo}C!bANEW-M%i6R(@+^X$4*7 z8s#^|AYL?T7J91lEqf<%_aOJ@rT_5 zmjoUHM2T#+@Q7&xjf(jLN;9+ryMw$73SaM)@)4inTOS??EqqBibalIg!?vF4rBBD8 z&jnrKU?j+hX}S2p8^G9fR_mFK(Tq*|#M;;F+^W|DbBOgueNr%@7|sHXE9M7Yjhsq?=_RG(~ z*-f+wmR44}!qGa{c^^K{%*+IjED!|}yK@XE>&+s_g`(BNRiHK5#ES*+|0eK=P4Bi92oo;ASM}zNA)*6R z2+>m8*KNoLnL=;M4&mK_(Eixj435SdUKAY)L9am&6b^O^&x=-<$dVU+AjSnllPGt~ z${A+nRV3^Fpq_7Z7VR`wVg|Rx&2mzr$q;bru*T0L<1w|z# z<(wBrn@k8Ll4E0KoF_HzojMFP{N&mjbcqD!Tb7?M4&dJ^Ja6Y`u;hdNG!)~Gqj$}> zs#s_O5jr9q!6eC5ip@6KH+`hi^D8;f(rBVLbnBw1)CFV%4VUyO@l zlh8n-@hK#%O_az5qEL;3d$sD(FN_8S?A{G zjc6qG%2efd8u+^NfBx(2@sDvYtq-w48jVX!kd#M z9lzAn1wF6!HZd_7#aF_(=-sa5_f<56|4A<{=1S#EnEP`k#_LVrIwb|<{349dHC0ywn6=+z7O#Bi$}Y;up{JnW<%N8j zFU2&*IgDL8Vea+B&(`XVjU}m>lE~0Zk#+q1F0Q~lY=WlX zdP8o{tCjNg`Ju{{(5~N0UB+Ie9v?Sn!tcF%mTxNaI=xwc{S|7xdR4$>vb5HpqkV|_ zrsl2HMm_oJhkff!T_d?4K5aU?zHyrCXJ~307iXc_nmR=l#8Tk7e0E;>>Afg{s$U1H zs!hrklG;CaNsQc9@VG2{?2!x!4wBocEp-Y>m%4;RqukY{Uj16pAk)*1j>;Fhf+xO} zB4S~w+RE*H6kaDQXGC52*R{LbW;u0>_Rn3?#%b_dz;q?1(EmV!J#1ODIdf)yd4?}8 zZOZBO>{dgjN<`RRR2JTvXXF@2)5|_BaEt`5R-BUSA_COPHvhPO&TG?{jV-bI?{D>` z_0;k|+LUw}PO=vA!{K|DTr>na9{QT{#hHHWDRM2p!|zO5fR;10eimy1&1N&6o`uX} z{N3GJanhomALBvNYbMC8T~FWgtN*F*m-^>J^wFsOQyV|rqV{WT49IUTzTWL!@X)Iw zcJ%&M3afFTonf)(#B2Q+-RkUWwVuy91BJY*AGWAk8A#i#wa$;0!M|;_c=&A7gmCFb zcjawObLX*<&F+q#sroU`L$`lpMtA#Q>AC%y@sd~H=5^m&DjxB$AK9E65XNoTRQeRg&2jIr@bZuOfP zmmYMIt-SN+amTHt;Yzn#a+%G_a&*_F6B^6CmL}^c=>9Zlnkn395LJ46xzX6Gq;u`al@Z(ZPnE}3?0$n=(Dq)fsULHw zxqjHWo953lJM8)R{;=DU3*T&=K6kQ4t#P$n+?#JkO|RX76^T6(a-@S;Tk%H2<91~4 z6kmSp=`>LSXR5K1#2$#b?CbFd|2A@oV4EtZRA4w3!#35Dj}!-Bsy? zD`GMKaJhqGV>9=G+ECWbFB|<^6FweHe~6n23-YCX zIK?j^fh>9V{T#g+sGli}abEBv{%_vAflJ8T)zRJEu)%SbMH;avvPpPTAcBaKNI%nr zrql=_@R@2&F+O|tC$CzmmHYdO*zv0~K#~Ij1NBho_3WAF=GqkKVr4TF$+ARHu=5_= z1H7@I@0Z8%@hAt`B+wXZfLmKF$CAQ(W=2t0nZ(1y%9m7b)@?ppt~@+uS5 zAU+NV2{$%8WzcXJKkI(rBoBcHV1D=AcFSji>bX4NkCV`OHAJjX_Q0BeP@7m!gPymfx(bOq{tRJ~GDqgu7Mudmc@NET~yI1Wm7 z&?hdVHDo0Rkf(x6Myw4 z0pJf0+es(Xs`S0Seg;7d1WuNg&zmQp`B&A@K)}S8{zgC8D7R;3nJrue;h8_G3$3lc zL-ld8?g`QV!AEluLBW#*sQ#hHg1GF}r2>lM-44)z0U1^d&U%P60O)FbMyg`!daq?s zKOz}muB*(>pl>OZb)T#OeTmP&aFmyNGk9tQGqEu-`atflp$H0^$%w=Z(n)w!bNge3 z88U+Rm*>uhOl`3JOrY z_oKfE2;=9*#>|qEUXw$>-4Rs!>p*4RXCo#mIyXJ7Kmhty>;f5A%lKlV*3{_37-*g# z^#Lj#$uATF-3F4D(-EA?gz4pVkPIqeF}X<$LHQEo+sPEa)%ffHkIS#-QC$_2vu)eX z1(Fuv_HDGo_PO~skUO|8h@Q_Mfausflz_maEHCEhp|7M_mj#R<6WBYZzC=jzh}QzO)W&Y|5A2JwMYUF zYA`vqeZ+;J&Kn=*8>O-AwJtg=h5q0_9Arl26o^k8DBka+3rq?H6JQk|C>#u^kUS;3 z1omwF)@W+n)G2|e=$II&k5L>56=Y0vP{qrP3<@6fWZfUreEskxTU${@Aiw29VnhNU zy1Fq`iz$?Uxq7lnjvSSPaFOTZ)MUa)^^MJ9zv-a(#6=supPyRNK~N z!a}Q5FT_X<$Ra9g50Pk6(8nZZc+tRM*L1n&9`I0kf=y!%3+{3WA5}UeH9JQ4J0tl* z(*b;Y&mM{1(g2upd+A=jpNZs@lnELcc=>qJ^s~r7y!;=W7Y^78slOJ9>sS1 zA{vi*{1OCe7N>Ee`z5#=Bs-@FK({F=C_+v>CG~7}}h1LU! zM8%_|BqPtG2Mz>}RwoJcg z@_vYfn*;@qWK`qrwh`}{A6Fvz?T>P7Tq-a}CTEkn zkft2^?wUArE8&(n{MZ;@kikH8)T;*v9SN50gbpiI@JCZTWb2__0d)k;2W@UAEQrtjFwrzwCqf4YWFYD9OxBBXg8^+9qi$ zolst0F3J=-WS42xZ(e{h{@W|_;98kD*tseW7Qz+8?3$(FjMz^kR8&?1f-g9;i}a$0 qY5q4TBV`k$cZEOQ_P>9+x^-P(miTgiryQvWR83jy)GI|3-~R(4JCTzB literal 0 HcmV?d00001 diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index 8cccd18e5c640..a57157bc775ad 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -831,5 +831,18 @@ TEST_F(AsyncClientImplTest, MultipleDataStream) { .value()); } +TEST_F(AsyncClientImplTest, WatermarkCallbacks) { + TestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + AsyncClient::Stream* stream = + client_.start(stream_callbacks_, Optional()); + stream->sendHeaders(headers, false); + Http::StreamDecoderFilterCallbacks* filter_callbacks = + static_cast(stream); + filter_callbacks->onDecoderFilterAboveWriteBufferHighWatermark(); + filter_callbacks->onDecoderFilterBelowWriteBufferLowWatermark(); + EXPECT_CALL(stream_callbacks_, onReset()); +} + } // namespace Http } // namespace Envoy From 46fd7134eb0c0b7f09cfffe11942806d8a2711dd Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Tue, 1 Aug 2017 15:44:03 -0400 Subject: [PATCH 11/19] docs and comment updates --- source/common/http/codec_helper.h | 1 + source/common/http/http2/codec_impl.cc | 3 +++ source/docs/flow_control.md | 32 +++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/source/common/http/codec_helper.h b/source/common/http/codec_helper.h index c6176eb25b8bb..fff96ad2105f7 100644 --- a/source/common/http/codec_helper.h +++ b/source/common/http/codec_helper.h @@ -8,6 +8,7 @@ namespace Http { class StreamCallbackHelper { public: void runLowWatermarkCallbacks() { + // TODO(alyssawilk) see if we can make this safe for disconnects mid-loop for (StreamCallbacks* callbacks : callbacks_) { callbacks->onBelowWriteBufferLowWatermark(); } diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 9a5dc89f18a55..599f0ce23606e 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -521,6 +521,9 @@ int ConnectionImpl::onStreamClose(int32_t stream_id, uint32_t error_code) { } connection_.dispatcher().deferredDelete(stream->removeFromList(active_streams_)); + // Any unconsumed data must be consumed before the stream is deleted. + // nghttp2 does not appear to track this internally, and any stream deleted + // with outstanding window will contribute to a slow connection-window leak. nghttp2_session_consume(session_, stream_id, stream->unconsumed_bytes_); stream->unconsumed_bytes_ = 0; nghttp2_session_set_stream_user_data(session_, stream->stream_id_, nullptr); diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md index 21c7c94241060..5653b8ab65f98 100644 --- a/source/docs/flow_control.md +++ b/source/docs/flow_control.md @@ -10,7 +10,37 @@ fire, informing the sender it can resume sending data. ### TCP implementation details -TODO(alyssawilk) document the existing connection level flow control and configuration. +Flow control for TCP and TCP-with-TLS-termination are handled by coordination +between the `Network::ConnectionImpl` write buffer, and the `Network::TcpProxy` +filter. + +The downstream flow control goes as follows. + + * The downstream `Network::ConnectionImpl::write_buffer_` buffers too much + data. It calls + `Network::ConnectionCallbacks::onAboveWriteBufferHighWatermark()`. + * The `Network::TcpProxy::DownstreamCallbacks` receives + `onAboveWriteBufferHighWatermark()` and calls `readDisable(true)` on the upstream + connection. + * When the downstream buffer is drained, it calls + `Network::ConnectionCallbacks::onBelowWriteBufferLowWatermark()` + * The `Network::TcpProxy::DownstreamCallbacks` receives + `onBelowWriteBufferLowWatermark()` and calls `readDisable(false)` on the upstream + connection. + +Flow control for the upstream path is much the same. + + * The upstream `Network::ConnectionImpl::write_buffer_` buffers too much + data. It calls + `Network::ConnectionCallbacks::onAboveWriteBufferHighWatermark()`. + * The Network::TcpProxy::UpstreamCallbacks receives + `onAboveWriteBufferHighWatermark()` and calls `readDisable(true)` on the downstream + connection. + * When the upstream buffer is drained, it calls + `Network::ConnectionCallbacks::onBelowWriteBufferLowWatermark()` + * The `Network::TcpProxy::UpstreamCallbacks` receives + `onBelowWriteBufferLowWatermark()` and calls `readDisable(false)` on the downstream + connection. ### HTTP2 implementation details From f3346f7db8594edb2f24ff4027285862da115d3d Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 2 Aug 2017 09:47:36 -0400 Subject: [PATCH 12/19] review cleanups --- include/envoy/http/codec.h | 4 ++-- source/common/http/http2/codec_impl.cc | 8 ++++---- source/common/http/http2/codec_impl.h | 8 ++++---- source/docs/flow_control.md | 26 +++++++++++++------------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 1cf2585b19a2a..56e52ed8f2836 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -288,12 +288,12 @@ class ClientConnection : public virtual Connection { /** * Called when the connection goes over its high watermark. */ - virtual void onBelowWriteBufferLowWatermark() PURE; + virtual void onAboveWriteBufferHighWatermark() PURE; /** * Called when the connection goes from over its high watermark to under its low watermark. */ - virtual void onAboveWriteBufferHighWatermark() PURE; + virtual void onBelowWriteBufferLowWatermark() PURE; }; typedef std::unique_ptr ClientConnectionPtr; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 27e2f0a3329ad..f89121e41b586 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -148,28 +148,28 @@ void ConnectionImpl::StreamImpl::readDisable(bool disable) { void ConnectionImpl::StreamImpl::pendingRecvBufferHighWatermark() { ENVOY_CONN_LOG(debug, "recv buffer over limit ", parent_.connection_); - ASSERT(pending_receive_buffer_high_watermark_called_ == false); + ASSERT(!pending_receive_buffer_high_watermark_called_); pending_receive_buffer_high_watermark_called_ = true; readDisable(true); } void ConnectionImpl::StreamImpl::pendingRecvBufferLowWatermark() { ENVOY_CONN_LOG(debug, "recv buffer under limit ", parent_.connection_); - ASSERT(pending_receive_buffer_high_watermark_called_ == true); + ASSERT(pending_receive_buffer_high_watermark_called_); pending_receive_buffer_high_watermark_called_ = false; readDisable(false); } void ConnectionImpl::StreamImpl::pendingSendBufferHighWatermark() { ENVOY_CONN_LOG(debug, "send buffer over limit ", parent_.connection_); - ASSERT(pending_send_buffer_high_watermark_called_ == false); + ASSERT(!pending_send_buffer_high_watermark_called_); pending_send_buffer_high_watermark_called_ = true; runHighWatermarkCallbacks(); } void ConnectionImpl::StreamImpl::pendingSendBufferLowWatermark() { ENVOY_CONN_LOG(debug, "send buffer under limit ", parent_.connection_); - ASSERT(pending_send_buffer_high_watermark_called_ == true); + ASSERT(pending_send_buffer_high_watermark_called_); pending_send_buffer_high_watermark_called_ = false; runLowWatermarkCallbacks(); } diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 8af0e130f0809..62536bf48b00f 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -234,7 +234,7 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable Date: Wed, 2 Aug 2017 09:57:28 -0400 Subject: [PATCH 13/19] fixing tyop --- source/docs/flow_control.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md index 248803cc95126..b12adadfd840b 100644 --- a/source/docs/flow_control.md +++ b/source/docs/flow_control.md @@ -78,7 +78,7 @@ The two main parties involved in flow control are the router filter (`Envoy::Rou the connection manager (`Envoy::Http::ConnectionManagerImpl`). The router is responsible for intercepting watermark events for its own buffers, the individual upstream streams (if codec buffers fill up) and the upstream connection (if the network buffer fills up). It passes -any events to the connection manager, which the ability to call `readDisable()` to enable and +any events to the connection manager, which has the ability to call `readDisable()` to enable and disable further data from downstream. TODO(alyssawilk) document the reverse path. From 6a6774a26c0feeb259bfdbd93b08a25006c29d01 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 2 Aug 2017 13:30:26 -0400 Subject: [PATCH 14/19] stream flow control on by default --- include/envoy/http/codec.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 56e52ed8f2836..96b4b84159c0c 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -211,8 +211,8 @@ struct Http2Settings { static const uint32_t DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE = 256 * 1024 * 1024; static const uint32_t MAX_INITIAL_CONNECTION_WINDOW_SIZE = (1U << 31) - 1; - // By default, do not enforce stream buffer limits beyond the connection level limits. - static const uint32_t DEFAULT_PER_STREAM_BUFFER_LIMIT = 0; + // By default, limit the per-steam buffer limit to the initial window size. + static const uint32_t DEFAULT_PER_STREAM_BUFFER_LIMIT = DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE; }; /** From f3a4908636248e715c12f8bf82a665bf7cb438d2 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 2 Aug 2017 14:47:27 -0400 Subject: [PATCH 15/19] Backing out config options, clarity rename --- include/envoy/http/codec.h | 14 ++--- source/common/config/protocol_json.cc | 1 - source/common/http/codec_client.h | 8 ++- source/common/http/http1/codec_impl.h | 4 +- source/common/http/http2/codec_impl.cc | 5 +- source/common/http/http2/codec_impl.h | 11 ++-- source/common/http/utility.cc | 2 - source/common/json/config_schemas.cc | 10 ---- test/common/http/codec_client_test.cc | 4 +- test/common/http/http2/codec_impl_test.cc | 66 +++++++++-------------- test/common/http/utility_test.cc | 8 +-- test/mocks/http/mocks.h | 4 +- test/test_common/utility.cc | 1 - 13 files changed, 52 insertions(+), 86 deletions(-) diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 96b4b84159c0c..e50cea348de4f 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -175,8 +175,6 @@ struct Http2Settings { uint32_t max_concurrent_streams_{DEFAULT_MAX_CONCURRENT_STREAMS}; uint32_t initial_stream_window_size_{DEFAULT_INITIAL_STREAM_WINDOW_SIZE}; uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE}; - // This setting applies to the HTTP/2 codec but is not an HTTP/2 setting per the spec. - uint32_t per_stream_buffer_limit_{DEFAULT_PER_STREAM_BUFFER_LIMIT}; // disable HPACK compression static const uint32_t MIN_HPACK_TABLE_SIZE = 0; @@ -210,9 +208,6 @@ struct Http2Settings { // our default connection-level window also equals to our stream-level static const uint32_t DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE = 256 * 1024 * 1024; static const uint32_t MAX_INITIAL_CONNECTION_WINDOW_SIZE = (1U << 31) - 1; - - // By default, limit the per-steam buffer limit to the initial window size. - static const uint32_t DEFAULT_PER_STREAM_BUFFER_LIMIT = DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE; }; /** @@ -286,14 +281,15 @@ class ClientConnection : public virtual Connection { virtual StreamEncoder& newStream(StreamDecoder& response_decoder) PURE; /** - * Called when the connection goes over its high watermark. + * Called when the underlying Network::Connection goes over its high watermark. */ - virtual void onAboveWriteBufferHighWatermark() PURE; + virtual void onUnderlyingConnectionAboveWriteBufferHighWatermark() PURE; /** - * Called when the connection goes from over its high watermark to under its low watermark. + * Called when the underlying Network::Connection goes from over its high watermark to under its + * low watermark. */ - virtual void onBelowWriteBufferLowWatermark() PURE; + virtual void onUnderlyingConnectionBelowWriteBufferLowWatermark() PURE; }; typedef std::unique_ptr ClientConnectionPtr; diff --git a/source/common/config/protocol_json.cc b/source/common/config/protocol_json.cc index f5a694241c182..84f1df24ae397 100644 --- a/source/common/config/protocol_json.cc +++ b/source/common/config/protocol_json.cc @@ -13,7 +13,6 @@ void ProtocolJson::translateHttp2ProtocolOptions( JSON_UTIL_SET_INTEGER(json_http2_settings, http2_protocol_options, initial_stream_window_size); JSON_UTIL_SET_INTEGER(json_http2_settings, http2_protocol_options, initial_connection_window_size); - JSON_UTIL_SET_INTEGER(json_http2_settings, http2_protocol_options, per_stream_buffer_limit_bytes); if (json_http_codec_options == "no_compression") { if (http2_protocol_options.hpack_table_size().value() != 0) { throw EnvoyException( diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index 57f473138fbc4..503818e4dcad2 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -184,8 +184,12 @@ class CodecClient : Logger::Loggable, void onEvent(Network::ConnectionEvent event) override; // Pass watermark events from the connection on to the codec which will pass it to the underlying // streams. - void onAboveWriteBufferHighWatermark() override { codec_->onAboveWriteBufferHighWatermark(); } - void onBelowWriteBufferLowWatermark() override { codec_->onBelowWriteBufferLowWatermark(); } + void onAboveWriteBufferHighWatermark() override { + codec_->onUnderlyingConnectionAboveWriteBufferHighWatermark(); + } + void onBelowWriteBufferLowWatermark() override { + codec_->onUnderlyingConnectionBelowWriteBufferLowWatermark(); + } std::list active_requests_; Http::ConnectionCallbacks* codec_callbacks_{}; diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 4e6262ba2f639..5af46c243e2c6 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -283,8 +283,8 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl { // Http::ClientConnection StreamEncoder& newStream(StreamDecoder& response_decoder) override; - void onBelowWriteBufferLowWatermark() override {} - void onAboveWriteBufferHighWatermark() override {} + void onUnderlyingConnectionAboveWriteBufferHighWatermark() override {} + void onUnderlyingConnectionBelowWriteBufferLowWatermark() override {} private: struct PendingResponse { diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index f89121e41b586..ad0e4faf452d8 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -54,7 +54,9 @@ template static T* remove_const(const void* object) { ConnectionImpl::StreamImpl::StreamImpl(ConnectionImpl& parent, uint32_t buffer_limit) : parent_(parent), headers_(new HeaderMapImpl()), local_end_stream_(false), local_end_stream_sent_(false), remote_end_stream_(false), data_deferred_(false), - waiting_for_non_informational_headers_(false) { + waiting_for_non_informational_headers_(false), + pending_receive_buffer_high_watermark_called_(false), + pending_send_buffer_high_watermark_called_(false) { if (buffer_limit > 0) { setWriteBufferWatermarks(buffer_limit / 2, buffer_limit); } @@ -743,7 +745,6 @@ ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, } Http::StreamEncoder& ClientConnectionImpl::newStream(StreamDecoder& decoder) { - StreamImplPtr stream(new ClientStreamImpl(*this, per_stream_buffer_limit_)); stream->decoder_ = &decoder; stream->moveIntoList(std::move(stream), active_streams_); diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 62536bf48b00f..04fc317ddfcc6 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -72,8 +72,9 @@ class ConnectionImpl : public virtual Connection, Logger::LoggablerunHighWatermarkCallbacks(); } } - void onBelowWriteBufferLowWatermark() override { + void onUnderlyingConnectionBelowWriteBufferLowWatermark() override { for (auto& stream : active_streams_) { stream->runLowWatermarkCallbacks(); } diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index 6be17d8e079ca..1e9cc8177926d 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -158,8 +158,6 @@ Http2Settings Utility::parseHttp2Settings(const envoy::api::v2::Http2ProtocolOpt ret.initial_connection_window_size_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, initial_connection_window_size, Http::Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE); - ret.per_stream_buffer_limit_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT( - config, per_stream_buffer_limit_bytes, Http::Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT); return ret; } diff --git a/source/common/json/config_schemas.cc b/source/common/json/config_schemas.cc index 369742b0aabb5..648955e658f14 100644 --- a/source/common/json/config_schemas.cc +++ b/source/common/json/config_schemas.cc @@ -286,11 +286,6 @@ const std::string Json::Schema::HTTP_CONN_NETWORK_FILTER_SCHEMA(R"EOF( "type": "integer", "minimum": 65535, "maximum" : 2147483647 - }, - "per_stream_buffer_limit_bytes" : { - "type": "integer", - "minimum": 0, - "exclusiveMinimum" : true } } }, @@ -1399,11 +1394,6 @@ const std::string Json::Schema::CLUSTER_SCHEMA(R"EOF( "type": "integer", "minimum": 65535, "maximum" : 2147483647 - }, - "per_stream_buffer_limit_bytes" : { - "type": "integer", - "minimum": 0, - "exclusiveMinimum" : true } } }, diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 4960a13cb2514..0d0121c05dff5 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -159,10 +159,10 @@ TEST_F(CodecClientTest, PrematureResponse) { } TEST_F(CodecClientTest, WatermarkPassthrough) { - EXPECT_CALL(*codec_, onAboveWriteBufferHighWatermark()); + EXPECT_CALL(*codec_, onUnderlyingConnectionAboveWriteBufferHighWatermark()); connection_cb_->onAboveWriteBufferHighWatermark(); - EXPECT_CALL(*codec_, onBelowWriteBufferLowWatermark()); + EXPECT_CALL(*codec_, onUnderlyingConnectionBelowWriteBufferLowWatermark()); connection_cb_->onBelowWriteBufferLowWatermark(); } diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 465af37ad77c3..77ad35c18248e 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -30,21 +30,16 @@ namespace Envoy { namespace Http { namespace Http2 { -typedef ::testing::tuple Http2SettingsTuple; +typedef ::testing::tuple Http2SettingsTuple; typedef ::testing::tuple Http2SettingsTestParam; -const char SMALL_WINDOW = 10; - namespace { - Http2Settings Http2SettingsFromTuple(const Http2SettingsTuple& tp) { Http2Settings ret; ret.hpack_table_size_ = ::testing::get<0>(tp); ret.max_concurrent_streams_ = ::testing::get<1>(tp); ret.initial_stream_window_size_ = ::testing::get<2>(tp); ret.initial_connection_window_size_ = ::testing::get<3>(tp); - ret.per_stream_buffer_limit_ = ::testing::get<4>(tp); - return ret; } } // namespace @@ -295,6 +290,8 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetClient) { HttpTestUtility::addDefaultHeaders(request_headers); request_encoder_->encodeHeaders(request_headers, false); Buffer::OwnedImpl body(std::string(1024 * 1024, 'a')); + EXPECT_CALL(client_stream_callbacks, onAboveWriteBufferHighWatermark()).Times(AnyNumber()); + ; request_encoder_->encodeData(body, true); EXPECT_CALL(client_stream_callbacks, onResetStream(StreamResetReason::LocalReset)); request_encoder_->getStream().resetStream(StreamResetReason::LocalReset); @@ -332,6 +329,8 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetServer) { TestHeaderMapImpl response_headers{{":status", "200"}}; response_encoder_->encodeHeaders(response_headers, false); Buffer::OwnedImpl body(std::string(1024 * 1024, 'a')); + EXPECT_CALL(server_stream_callbacks_, onAboveWriteBufferHighWatermark()).Times(AnyNumber()); + ; response_encoder_->encodeData(body, true); EXPECT_CALL(server_stream_callbacks_, onResetStream(StreamResetReason::LocalReset)); response_encoder_->getStream().resetStream(StreamResetReason::LocalReset); @@ -376,10 +375,6 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { // One large write gets broken into smaller frames. EXPECT_CALL(request_decoder_, decodeData(_, false)).Times(AnyNumber()); Buffer::OwnedImpl long_data(std::string(initial_stream_window, 'a')); - // The one giant write will cause the buffer to go over the limit, then drain and go back under - // the limit. - EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); - EXPECT_CALL(callbacks, onBelowWriteBufferLowWatermark()); request_encoder_->encodeData(long_data, false); // Verify that the window is full. The client will not send more data to the server for this @@ -389,26 +384,27 @@ TEST_P(Http2CodecImplFlowControlTest, TestFlowControlInPendingSendData) { EXPECT_EQ(initial_stream_window, server_.getStream(1)->unconsumed_bytes_); // Now that the flow control window is full, further data causes the send buffer to back up. - Buffer::OwnedImpl ten_bytes("0123456789"); - request_encoder_->encodeData(ten_bytes, false); - EXPECT_EQ(SMALL_WINDOW, client_.getStream(1)->pending_send_data_.length()); + Buffer::OwnedImpl more_long_data(std::string(initial_stream_window, 'a')); + request_encoder_->encodeData(more_long_data, false); + EXPECT_EQ(initial_stream_window, client_.getStream(1)->pending_send_data_.length()); + EXPECT_EQ(initial_stream_window, server_.getStream(1)->unconsumed_bytes_); // If we go over the limit, the stream callbacks should fire. EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); Buffer::OwnedImpl last_byte("!"); request_encoder_->encodeData(last_byte, false); - EXPECT_EQ(SMALL_WINDOW + 1, client_.getStream(1)->pending_send_data_.length()); + EXPECT_EQ(initial_stream_window + 1, client_.getStream(1)->pending_send_data_.length()); // Now unblock the server's stream. This will cause the bytes to be consumed, flow control // updates to be sent, and the client to flush all queued data. EXPECT_CALL(callbacks, onBelowWriteBufferLowWatermark()); server_.getStream(1)->readDisable(false); EXPECT_EQ(0, client_.getStream(1)->pending_send_data_.length()); - // The 11 bytes sent won't trigger another window update, so the final window should be the - // initial window minus the last 11 byte flush from the client to server. - EXPECT_EQ(initial_stream_window - (SMALL_WINDOW + 1), + // The extra 1 byte sent won't trigger another window update, so the final window should be the + // initial window minus the last 1 byte flush from the client to server. + EXPECT_EQ(initial_stream_window - 1, nghttp2_session_get_stream_local_window_size(server_.session(), 1)); - EXPECT_EQ(initial_stream_window - (SMALL_WINDOW + 1), + EXPECT_EQ(initial_stream_window - 1, nghttp2_session_get_stream_remote_window_size(client_.session(), 1)); } @@ -436,13 +432,11 @@ TEST_P(Http2CodecImplFlowControlTest, EarlyResetRestoresWindow) { // If this limit is changed, this test will fail due to the initial large writes being divided // into more than 4 frames. Fast fail here with this explanatory comment. ASSERT_EQ(65535, initial_stream_window); - // One large write gets broken into smaller frames. + // One large write may get broken into smaller frames. EXPECT_CALL(request_decoder_, decodeData(_, false)).Times(AnyNumber()); Buffer::OwnedImpl long_data(std::string(initial_stream_window, 'a')); // The one giant write will cause the buffer to go over the limit, then drain and go back under // the limit. - EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark()); - EXPECT_CALL(callbacks, onBelowWriteBufferLowWatermark()); request_encoder_->encodeData(long_data, false); // Verify that the window is full. The client will not send more data to the server for this @@ -460,32 +454,28 @@ TEST_P(Http2CodecImplFlowControlTest, EarlyResetRestoresWindow) { EXPECT_EQ(initial_connection_window, nghttp2_session_get_remote_window_size(client_.session())); } -#define HTTP2SETTINGS_DEFERRED_RESET_COMBINE \ - ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ - ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ - ::testing::Values(Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT)) - #define HTTP2SETTINGS_SMALL_WINDOW_COMBINE \ ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ ::testing::Values(Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE), \ - ::testing::Values(SMALL_WINDOW)) + ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE)) // Deferred reset tests use only small windows so that we can test certain conditions. INSTANTIATE_TEST_CASE_P(Http2CodecImplDeferredResetTest, Http2CodecImplDeferredResetTest, - ::testing::Combine(HTTP2SETTINGS_DEFERRED_RESET_COMBINE, - HTTP2SETTINGS_DEFERRED_RESET_COMBINE)); + ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, + HTTP2SETTINGS_SMALL_WINDOW_COMBINE)); + +// Flow control tests only use only small windows so that we can test certain conditions. +INSTANTIATE_TEST_CASE_P(Http2CodecImplFlowControlTest, Http2CodecImplFlowControlTest, + ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, + HTTP2SETTINGS_SMALL_WINDOW_COMBINE)); // we seperate default/edge cases here to avoid combinatorial explosion #define HTTP2SETTINGS_DEFAULT_COMBINE \ ::testing::Combine(::testing::Values(Http2Settings::DEFAULT_HPACK_TABLE_SIZE), \ ::testing::Values(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS), \ ::testing::Values(Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT)) + ::testing::Values(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE)) INSTANTIATE_TEST_CASE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, @@ -499,17 +489,11 @@ INSTANTIATE_TEST_CASE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, ::testing::Values(Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE, \ Http2Settings::MAX_INITIAL_STREAM_WINDOW_SIZE), \ ::testing::Values(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE, \ - Http2Settings::MAX_INITIAL_CONNECTION_WINDOW_SIZE), \ - ::testing::Values(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT)) + Http2Settings::MAX_INITIAL_CONNECTION_WINDOW_SIZE)) INSTANTIATE_TEST_CASE_P(Http2CodecImplTestEdgeSettings, Http2CodecImplTest, ::testing::Combine(HTTP2SETTINGS_EDGE_COMBINE, HTTP2SETTINGS_EDGE_COMBINE)); -// Flow control tests only use only small windows so that we can test certain conditions. -INSTANTIATE_TEST_CASE_P(Http2CodecImplFlowControlTest, Http2CodecImplFlowControlTest, - ::testing::Combine(HTTP2SETTINGS_SMALL_WINDOW_COMBINE, - HTTP2SETTINGS_SMALL_WINDOW_COMBINE)); - TEST(Http2CodecUtility, reconstituteCrumbledCookies) { { HeaderString key; diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index a0d3df5d5d203..d1055f878fc4e 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -126,8 +126,6 @@ TEST(HttpUtility, parseHttp2Settings) { http2_settings.initial_stream_window_size_); EXPECT_EQ(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE, http2_settings.initial_connection_window_size_); - EXPECT_EQ(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT, - http2_settings.per_stream_buffer_limit_); } { @@ -136,15 +134,13 @@ TEST(HttpUtility, parseHttp2Settings) { "hpack_table_size": 1, "max_concurrent_streams": 2, "initial_stream_window_size": 3, - "initial_connection_window_size": 4, - "per_stream_buffer_limit_bytes": 5 + "initial_connection_window_size": 4 } })raw"); EXPECT_EQ(1U, http2_settings.hpack_table_size_); EXPECT_EQ(2U, http2_settings.max_concurrent_streams_); EXPECT_EQ(3U, http2_settings.initial_stream_window_size_); EXPECT_EQ(4U, http2_settings.initial_connection_window_size_); - EXPECT_EQ(5U, http2_settings.per_stream_buffer_limit_); } { @@ -158,8 +154,6 @@ TEST(HttpUtility, parseHttp2Settings) { http2_settings.initial_stream_window_size_); EXPECT_EQ(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE, http2_settings.initial_connection_window_size_); - EXPECT_EQ(Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT, - http2_settings.per_stream_buffer_limit_); } { diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 771d4dc915f73..efe012aa288bb 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -202,8 +202,8 @@ class MockClientConnection : public ClientConnection { // Http::ClientConnection MOCK_METHOD1(newStream, StreamEncoder&(StreamDecoder& response_decoder)); - MOCK_METHOD0(onAboveWriteBufferHighWatermark, void()); - MOCK_METHOD0(onBelowWriteBufferLowWatermark, void()); + MOCK_METHOD0(onUnderlyingConnectionAboveWriteBufferHighWatermark, void()); + MOCK_METHOD0(onUnderlyingConnectionBelowWriteBufferLowWatermark, void()); }; class MockFilterChainFactory : public FilterChainFactory { diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 42ae79ceaef97..fe353e86a814c 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -160,7 +160,6 @@ const uint32_t Http2Settings::DEFAULT_HPACK_TABLE_SIZE; const uint32_t Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS; const uint32_t Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE; const uint32_t Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE; -const uint32_t Http2Settings::DEFAULT_PER_STREAM_BUFFER_LIMIT; TestHeaderMapImpl::TestHeaderMapImpl() : HeaderMapImpl() {} From e6b5236bc3836ce558b7a5f3449394276ccbc93e Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Wed, 2 Aug 2017 16:41:06 -0400 Subject: [PATCH 16/19] removing spurious line --- test/common/http/http2/codec_impl_test.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 77ad35c18248e..339fde616f7fd 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -291,7 +291,6 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetClient) { request_encoder_->encodeHeaders(request_headers, false); Buffer::OwnedImpl body(std::string(1024 * 1024, 'a')); EXPECT_CALL(client_stream_callbacks, onAboveWriteBufferHighWatermark()).Times(AnyNumber()); - ; request_encoder_->encodeData(body, true); EXPECT_CALL(client_stream_callbacks, onResetStream(StreamResetReason::LocalReset)); request_encoder_->getStream().resetStream(StreamResetReason::LocalReset); @@ -330,7 +329,6 @@ TEST_P(Http2CodecImplDeferredResetTest, DeferredResetServer) { response_encoder_->encodeHeaders(response_headers, false); Buffer::OwnedImpl body(std::string(1024 * 1024, 'a')); EXPECT_CALL(server_stream_callbacks_, onAboveWriteBufferHighWatermark()).Times(AnyNumber()); - ; response_encoder_->encodeData(body, true); EXPECT_CALL(server_stream_callbacks_, onResetStream(StreamResetReason::LocalReset)); response_encoder_->getStream().resetStream(StreamResetReason::LocalReset); From 46d12c5d3612a3ae4fbea40966d07ca6efe9a7e5 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Thu, 3 Aug 2017 09:39:18 -0400 Subject: [PATCH 17/19] addressing review comments --- source/docs/flow_control.md | 11 ++-- test/config/integration/server_http2.json | 50 ------------------- .../integration/server_http2_upstream.json | 50 ------------------- 3 files changed, 6 insertions(+), 105 deletions(-) diff --git a/source/docs/flow_control.md b/source/docs/flow_control.md index b12adadfd840b..c451bef6eac7a 100644 --- a/source/docs/flow_control.md +++ b/source/docs/flow_control.md @@ -53,9 +53,10 @@ For HTTP/2, when filters, streams, or connections back up, the end result is `re being called on the source stream. This results in the stream ceasing to consume window, and so not sending further flow control window updates to the peer. This will result in the peer eventually stopping sending data when the available window is consumed (or nghttp2 closing the -connection if the peer violates the flow control limit). When `readDisable(false)` is called, any -outstanding unconsumed data is immediately consumed, which results in resuming window updates to the -peer and the resumption of data. +connection if the peer violates the flow control limit) and so limiting the amount of data Envoy +will buffer for each stream. When `readDisable(false)` is called, any outstanding unconsumed data +is immediately consumed, which results in resuming window updates to the peer and the resumption of +data. Note that `readDisable(true)` on a stream may be called by multiple entities. It is called when any filter buffers too much, when the stream backs up and has too much data buffered, or the @@ -146,7 +147,7 @@ The high watermark path is as follows: * When `Envoy::Network::ConnectionImpl::write_buffer_` has too much data it calls `Network::ConnectionCallbacks::onAboveWriteBufferHighWatermark()`. * When `Envoy::Http::CodecClient` receives `onAboveWriteBufferHighWatermark()` it - calls `onAboveWriteBufferHighWatermark()` on `codec_`. + calls `onUnderlyingConnectionAboveWriteBufferHighWatermark()` on `codec_`. * When `Envoy::Http::Http2::ConnectionImpl` receives `onAboveWriteBufferHighWatermark()` it calls `runHighWatermarkCallbacks()` for each stream of the connection. * `runHighWatermarkCallbacks()` results in all subscribers of `Envoy::Http::StreamCallback` @@ -162,7 +163,7 @@ The low watermark path is as follows: * When `Envoy::Network::ConnectionImpl::write_buffer_` is drained it calls `Network::ConnectionCallbacks::onBelowWriteBufferLowWatermark()`. * When `Envoy::Http::CodecClient` receives `onBelowWriteBufferLowWatermark()` it - calls `onBelowWriteBufferLowWatermark()` on `codec_`. + calls `onUnderlyingConnectionBelowWriteBufferLowWatermark()` on `codec_`. * When `Envoy::Http::Http2::ConnectionImpl` receives `onBelowWriteBufferLowWatermark()` it calls `runLowWatermarkCallbacks()` for each stream of the connection. * `runLowWatermarkCallbacks()` results in all subscribers of `Envoy::Http::StreamCallback` diff --git a/test/config/integration/server_http2.json b/test/config/integration/server_http2.json index 49cd1be1c4fa6..6d1ce16ed95d1 100644 --- a/test/config/integration/server_http2.json +++ b/test/config/integration/server_http2.json @@ -187,73 +187,23 @@ "per_stream_buffer_limit": 1024 }, "drain_timeout_ms": 5000, - "access_log": [ - { - "path": "/dev/null", - "filter" : { - "type": "logical_or", - "filters": [ - { - "type": "status_code", - "op": ">=", - "value": 500 - }, - { - "type": "duration", - "op": ">=", - "value": 1000000 - } - ] - } - }, - { - "path": "/dev/null" - }], "stat_prefix": "router", "route_config": { "virtual_hosts": [ - { - "name": "redirect", - "domains": [ "www.redirect.com" ], - "require_ssl": "all", - "routes": [ - { - "prefix": "/", - "cluster": "cluster_3" - } - ] - }, { "name": "integration", "domains": [ "*" ], "routes": [ - { - "prefix": "/", - "cluster": "cluster_3", - "runtime": { - "key": "some_key", - "default": 0 - } - }, { "prefix": "/test/long/url", "cluster": "cluster_3" - }, - { - "prefix": "/test/", - "cluster": "cluster_2" } ] } ] }, "filters": [ - { "type": "both", "name": "health_check", - "config": { - "pass_through_mode": false, "endpoint": "/healthcheck" - } - }, { "type": "decoder", "name": "router", "config": {}} ] } diff --git a/test/config/integration/server_http2_upstream.json b/test/config/integration/server_http2_upstream.json index cf434a0fe8db7..f1654de28f2eb 100644 --- a/test/config/integration/server_http2_upstream.json +++ b/test/config/integration/server_http2_upstream.json @@ -267,73 +267,23 @@ "per_stream_buffer_limit": 1024 }, "drain_timeout_ms": 5000, - "access_log": [ - { - "path": "/dev/null", - "filter" : { - "type": "logical_or", - "filters": [ - { - "type": "status_code", - "op": ">=", - "value": 500 - }, - { - "type": "duration", - "op": ">=", - "value": 1000000 - } - ] - } - }, - { - "path": "/dev/null" - }], "stat_prefix": "router", "route_config": { "virtual_hosts": [ - { - "name": "redirect", - "domains": [ "www.redirect.com" ], - "require_ssl": "all", - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1" - } - ] - }, { "name": "integration", "domains": [ "*" ], "routes": [ - { - "prefix": "/", - "cluster": "cluster_3", - "runtime": { - "key": "some_key", - "default": 0 - } - }, { "prefix": "/test/long/url", "cluster": "cluster_3" - }, - { - "prefix": "/test/", - "cluster": "cluster_3" } ] } ] }, "filters": [ - { "type": "both", "name": "health_check", - "config": { - "pass_through_mode": false, "endpoint": "/healthcheck" - } - }, { "type": "decoder", "name": "router", "config": {} } ] } From 44d9edec67624442797f7c205604ed6cdfb7a288 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Thu, 3 Aug 2017 12:26:47 -0400 Subject: [PATCH 18/19] Using less punctuation --- docs/configuration/cluster_manager/cluster_stats.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/cluster_manager/cluster_stats.rst b/docs/configuration/cluster_manager/cluster_stats.rst index f47d7135fb4d4..3ab7510fa40cb 100644 --- a/docs/configuration/cluster_manager/cluster_stats.rst +++ b/docs/configuration/cluster_manager/cluster_stats.rst @@ -55,8 +55,8 @@ Every cluster has a statistics tree rooted at *cluster..* with the followi upstream_rq_retry_overflow, Counter, Total requests not retried due to circuit breaking upstream_flow_control_paused_reading_total, Counter, Total number of times flow control paused reading from upstream. upstream_flow_control_resumed_reading_total, Counter, Total number of times flow control resumed reading from upstream. - upstream_flow_control_backed_up_total, Counter, Total number of times the upstream connection backed up, pausing reads from downstream. - upstream_flow_control_drained_total, Counter, Total number of times the upstream connection drained, resuming reads from downstream. + upstream_flow_control_backed_up_total, Counter, Total number of times the upstream connection backed up and paused reads from downstream. + upstream_flow_control_drained_total, Counter, Total number of times the upstream connection drained and resumed reads from downstream. membership_change, Counter, Total cluster membership changes membership_healthy, Gauge, Current cluster healthy total (inclusive of both health checking and outlier detection) membership_total, Gauge, Current cluster membership total From 79ff9b1751fb7a5c0ca15539ca06c1985eb6439c Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Thu, 3 Aug 2017 19:33:28 -0400 Subject: [PATCH 19/19] Docs fix --- docs/configuration/http_conn_man/http_conn_man.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/configuration/http_conn_man/http_conn_man.rst b/docs/configuration/http_conn_man/http_conn_man.rst index 296675312c65f..c5dc0d0ce39a6 100644 --- a/docs/configuration/http_conn_man/http_conn_man.rst +++ b/docs/configuration/http_conn_man/http_conn_man.rst @@ -135,17 +135,16 @@ http2_settings NOTE: 65535 is the initial window size from HTTP/2 spec. We only support increasing the default window size now, so it's also the minimum. + This field also acts as a soft limit on the number of bytes Envoy will buffer per-stream in the + HTTP/2 codec buffers. Once the buffer reaches this pointer, watermark callbacks will fire to + stop the flow of data to the codec buffers. + initial_connection_window_size *(optional, integer)* Similar to :ref:`initial_stream_window_size `, but for connection-level flow-control window. Currently , this has the same minimum/maximum/default as :ref:`initial_stream_window_size `. - per_stream_buffer_limit_bytes - *(optional, integer)* A soft limit on the number of bytes Envoy will buffer per-stream in the - codec buffers. Once the buffer reaches this point, watermark callbacks will fire to stop the - flow of data to the codec buffers. If this limit is zero, no buffer limits will be applied. - These are the same options available in the upstream cluster :ref:`http2_settings ` option.